1//===------- VectorCombine.cpp - Optimize partial vector operations -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass optimizes scalar/vector interactions using target cost models. The
10// transforms implemented here may not fit in traditional loop-based or SLP
11// vectorization passes.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Vectorize/VectorCombine.h"
16#include "VectorizeOptions.h"
17#include "llvm/ADT/DenseMap.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/ScopeExit.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/SmallVectorExtras.h"
22#include "llvm/ADT/Statistic.h"
23#include "llvm/Analysis/AssumptionCache.h"
24#include "llvm/Analysis/BasicAliasAnalysis.h"
25#include "llvm/Analysis/GlobalsModRef.h"
26#include "llvm/Analysis/InstSimplifyFolder.h"
27#include "llvm/Analysis/Loads.h"
28#include "llvm/Analysis/TargetFolder.h"
29#include "llvm/Analysis/TargetTransformInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/Analysis/VectorUtils.h"
32#include "llvm/IR/Dominators.h"
33#include "llvm/IR/Function.h"
34#include "llvm/IR/IRBuilder.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/PatternMatch.h"
37#include "llvm/IR/ProfDataUtils.h"
38#include "llvm/Support/KnownBits.h"
39#include "llvm/Support/MathExtras.h"
40#include "llvm/Transforms/Utils/Local.h"
41#include "llvm/Transforms/Utils/LoopUtils.h"
42#include <numeric>
43#include <optional>
44#include <queue>
45#include <set>
46
47#define DEBUG_TYPE "vector-combine"
48#include "llvm/Transforms/Utils/InstructionWorklist.h"
49
50using namespace llvm;
51using namespace llvm::PatternMatch;
52
53STATISTIC(NumVecLoad, "Number of vector loads formed");
54STATISTIC(NumVecCmp, "Number of vector compares formed");
55STATISTIC(NumVecBO, "Number of vector binops formed");
56STATISTIC(NumVecCmpBO, "Number of vector compare + binop formed");
57STATISTIC(NumShufOfBitcast, "Number of shuffles moved after bitcast");
58STATISTIC(NumScalarOps, "Number of scalar unary + binary ops formed");
59STATISTIC(NumScalarCmp, "Number of scalar compares formed");
60STATISTIC(NumScalarIntrinsic, "Number of scalar intrinsic calls formed");
61
62static const unsigned InvalidIndex = std::numeric_limits<unsigned>::max();
63
64namespace {
65class VectorCombine {
66public:
67 VectorCombine(const VectorizeOptions &Opts, Function &F,
68 const TargetTransformInfo &TTI, const DominatorTree &DT,
69 AAResults &AA, AssumptionCache &AC, const DataLayout *DL,
70 TTI::TargetCostKind CostKind, bool TryEarlyFoldsOnly)
71 : Opts(Opts), F(F),
72 Builder(*F.getParent(), InstSimplifyFolder(*DL),
73 IRBuilderCallbackInserter([this](Instruction *I) {
74 // Automatically add newly created instructions
75 // into the deferred queue.
76 Worklist.add(I);
77 })),
78 TTI(TTI), DT(DT), AA(AA), DL(DL), CostKind(CostKind),
79 SQ(*DL, /*TLI=*/nullptr, &DT, &AC),
80 TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
81
82 bool run();
83
84private:
85 const VectorizeOptions &Opts;
86 Function &F;
87 IRBuilder<InstSimplifyFolder, IRBuilderCallbackInserter> Builder;
88 const TargetTransformInfo &TTI;
89 const DominatorTree &DT;
90 AAResults &AA;
91 const DataLayout *DL;
92 TTI::TargetCostKind CostKind;
93 const SimplifyQuery SQ;
94
95 /// If true, only perform beneficial early IR transforms. Do not introduce new
96 /// vector operations.
97 bool TryEarlyFoldsOnly;
98
99 InstructionWorklist Worklist;
100
101 /// Next instruction to iterate. It will be updated when it is erased by
102 /// RecursivelyDeleteTriviallyDeadInstructions.
103 Instruction *NextInst;
104
105 // TODO: Direct calls from the top-level "run" loop use a plain "Instruction"
106 // parameter. That should be updated to specific sub-classes because the
107 // run loop was changed to dispatch on opcode.
108 bool vectorizeLoadInsert(Instruction &I);
109 bool widenSubvectorLoad(Instruction &I);
110 ExtractElementInst *getShuffleExtract(ExtractElementInst *Ext0,
111 ExtractElementInst *Ext1,
112 unsigned PreferredExtractIndex) const;
113 bool isExtractExtractCheap(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
114 const Instruction &I,
115 ExtractElementInst *&ConvertToShuffle,
116 unsigned PreferredExtractIndex);
117 Value *foldExtExtCmp(Value *V0, Value *V1, Value *ExtIndex, Instruction &I);
118 Value *foldExtExtBinop(Value *V0, Value *V1, Value *ExtIndex, Instruction &I);
119 bool foldExtractExtract(Instruction &I);
120 bool foldInsExtFNeg(Instruction &I);
121 bool foldInsExtBinop(Instruction &I);
122 bool foldInsExtVectorToShuffle(Instruction &I);
123 bool foldInsertScalarPartsToShuffle(Instruction &I);
124 bool foldBitOpOfCastops(Instruction &I);
125 bool foldBitOpOfCastConstant(Instruction &I);
126 bool foldBitcastShuffle(Instruction &I);
127 bool scalarizeOpOrCmp(Instruction &I);
128 bool foldExtractedCmps(Instruction &I);
129 bool foldSelectsFromBitcast(Instruction &I);
130 bool foldBinopOfReductions(Instruction &I);
131 bool foldInsertElementsToStores(Instruction &I);
132 bool scalarizeLoad(Instruction &I);
133 bool scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy, Value *Ptr);
134 bool scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy, Value *Ptr);
135 bool scalarizeExtExtract(Instruction &I);
136 bool foldConcatOfBoolMasks(Instruction &I);
137 bool foldPermuteOfBinops(Instruction &I);
138 bool foldShuffleOfBinops(Instruction &I);
139 bool foldShuffleOfSelects(Instruction &I);
140 bool foldShuffleOfCastops(Instruction &I);
141 bool foldShuffleOfShuffles(Instruction &I);
142 bool foldPermuteOfIntrinsic(Instruction &I);
143 bool foldShufflesOfLengthChangingShuffles(Instruction &I);
144 bool foldShuffleOfIntrinsics(Instruction &I);
145 bool foldShuffleToIdentity(Instruction &I);
146 bool foldShuffleFromReductions(Instruction &I);
147 bool foldShuffleChainsToReduce(Instruction &I);
148 bool foldCastFromReductions(Instruction &I);
149 bool foldSignBitReductionCmp(Instruction &I);
150 bool foldReductionZeroTest(Instruction &I);
151 bool foldICmpEqZeroVectorReduce(Instruction &I);
152 bool foldEquivalentReductionCmp(Instruction &I);
153 bool foldReduceAddCmpZero(Instruction &I);
154 bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
155 bool foldInterleaveIntrinsics(Instruction &I);
156 bool foldDeinterleaveIntrinsics(Instruction &I);
157 bool foldBitcastOfVPLoad(Instruction &I);
158 bool foldBitOrderReverseAndSwap(Instruction &I);
159 bool shrinkType(Instruction &I);
160 bool shrinkLoadForShuffles(Instruction &I);
161 bool shrinkPhiOfShuffles(Instruction &I);
162 bool foldInterleaveOfDeinterleaveChains(Instruction &I);
163
164 void replaceValue(Instruction &Old, Value &New, bool Erase = true) {
165 LLVM_DEBUG(dbgs() << "VC: Replacing: " << Old << '\n');
166 LLVM_DEBUG(dbgs() << " With: " << New << '\n');
167 Old.replaceAllUsesWith(V: &New);
168 if (auto *NewI = dyn_cast<Instruction>(Val: &New)) {
169 New.takeName(V: &Old);
170 Worklist.pushUsersToWorkList(I&: *NewI);
171 Worklist.pushValue(V: NewI);
172 }
173 if (Erase && isInstructionTriviallyDead(I: &Old)) {
174 eraseInstruction(I&: Old);
175 } else {
176 Worklist.push(I: &Old);
177 }
178 }
179
180 void eraseInstruction(Instruction &I) {
181 LLVM_DEBUG(dbgs() << "VC: Erasing: " << I << '\n');
182 SmallVector<Value *> Ops(I.operands());
183 Worklist.remove(I: &I);
184 I.eraseFromParent();
185
186 // Push remaining users of the operands and then the operand itself - allows
187 // further folds that were hindered by OneUse limits.
188 SmallPtrSet<Value *, 4> Visited;
189 for (Value *Op : Ops) {
190 if (!Visited.contains(Ptr: Op)) {
191 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
192 if (RecursivelyDeleteTriviallyDeadInstructions(
193 V: OpI, TLI: nullptr, MSSAU: nullptr, AboutToDeleteCallback: [&](Value *V) {
194 if (auto *I = dyn_cast<Instruction>(Val: V)) {
195 LLVM_DEBUG(dbgs() << "VC: Erased: " << *I << '\n');
196 Worklist.remove(I);
197 if (I == NextInst)
198 NextInst = NextInst->getNextNode();
199 Visited.insert(Ptr: I);
200 }
201 }))
202 continue;
203 Worklist.pushUsersToWorkList(I&: *OpI);
204 Worklist.pushValue(V: OpI);
205 }
206 }
207 }
208 }
209};
210} // namespace
211
212/// Return the source operand of a potentially bitcasted value. If there is no
213/// bitcast, return the input value itself.
214static Value *peekThroughBitcasts(Value *V) {
215 while (auto *BitCast = dyn_cast<BitCastInst>(Val: V))
216 V = BitCast->getOperand(i_nocapture: 0);
217 return V;
218}
219
220/// Helper to peek through bitcasts to the same value.
221static bool isEquivBitcast(Value *X, Value *Y) {
222 return X->getType() == Y->getType() &&
223 peekThroughBitcasts(V: X) == peekThroughBitcasts(V: Y);
224}
225
226static bool canWidenLoad(LoadInst *Load, const TargetTransformInfo &TTI) {
227 // Do not widen load if atomic/volatile or under asan/hwasan/memtag/tsan.
228 // The widened load may load data from dirty regions or create data races
229 // non-existent in the source.
230 if (!Load || !Load->isSimple() || !Load->hasOneUse() ||
231 Load->getFunction()->hasFnAttribute(Kind: Attribute::SanitizeMemTag) ||
232 mustSuppressSpeculation(LI: *Load))
233 return false;
234
235 // We are potentially transforming byte-sized (8-bit) memory accesses, so make
236 // sure we have all of our type-based constraints in place for this target.
237 Type *ScalarTy = Load->getType()->getScalarType();
238 uint64_t ScalarSize = ScalarTy->getPrimitiveSizeInBits();
239 unsigned MinVectorSize = TTI.getMinVectorRegisterBitWidth();
240 if (!ScalarSize || !MinVectorSize || MinVectorSize % ScalarSize != 0 ||
241 ScalarSize % 8 != 0)
242 return false;
243
244 return true;
245}
246
247bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
248 // Match insert into fixed vector of scalar value.
249 // TODO: Handle non-zero insert index.
250 Value *Scalar;
251 if (!match(V: &I,
252 P: m_InsertElt(Val: m_Poison(), Elt: m_OneUse(SubPattern: m_Value(V&: Scalar)), Idx: m_ZeroInt())))
253 return false;
254
255 // Optionally match an extract from another vector.
256 Value *X;
257 bool HasExtract = match(V: Scalar, P: m_ExtractElt(Val: m_Value(V&: X), Idx: m_ZeroInt()));
258 if (!HasExtract)
259 X = Scalar;
260
261 auto *Load = dyn_cast<LoadInst>(Val: X);
262 if (!canWidenLoad(Load, TTI))
263 return false;
264
265 Type *ScalarTy = Scalar->getType();
266 uint64_t ScalarSize = ScalarTy->getPrimitiveSizeInBits();
267 unsigned MinVectorSize = TTI.getMinVectorRegisterBitWidth();
268
269 // Check safety of replacing the scalar load with a larger vector load.
270 // We use minimal alignment (maximum flexibility) because we only care about
271 // the dereferenceable region. When calculating cost and creating a new op,
272 // we may use a larger value based on alignment attributes.
273 Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts();
274 assert(isa<PointerType>(SrcPtr->getType()) && "Expected a pointer type");
275
276 unsigned MinVecNumElts = MinVectorSize / ScalarSize;
277 auto *MinVecTy = VectorType::get(ElementType: ScalarTy, NumElements: MinVecNumElts, Scalable: false);
278 unsigned OffsetEltIndex = 0;
279 Align Alignment = Load->getAlign();
280 // Ignore frees because the existing load proves that the object has not been
281 // freed. Allocations cannot shrink, so this is sufficient.
282 if (!isSafeToLoadUnconditionally(V: SrcPtr, Ty: MinVecTy, Alignment: Align(1),
283 SQ: SQ.getWithInstruction(I: Load),
284 /*IgnoreFree=*/true)) {
285 // It is not safe to load directly from the pointer, but we can still peek
286 // through gep offsets and check if it safe to load from a base address with
287 // updated alignment. If it is, we can shuffle the element(s) into place
288 // after loading.
289 unsigned OffsetBitWidth = DL->getIndexTypeSizeInBits(Ty: SrcPtr->getType());
290 APInt Offset(OffsetBitWidth, 0);
291 SrcPtr = SrcPtr->stripAndAccumulateInBoundsConstantOffsets(DL: *DL, Offset);
292
293 // We want to shuffle the result down from a high element of a vector, so
294 // the offset must be positive.
295 if (Offset.isNegative())
296 return false;
297
298 // The offset must be a multiple of the scalar element to shuffle cleanly
299 // in the element's size.
300 uint64_t ScalarSizeInBytes = ScalarSize / 8;
301 if (Offset.urem(RHS: ScalarSizeInBytes) != 0)
302 return false;
303
304 // If we load MinVecNumElts, will our target element still be loaded?
305 APInt OffsetEltIndexAP = Offset.udiv(RHS: ScalarSizeInBytes);
306 if (OffsetEltIndexAP.uge(RHS: MinVecNumElts))
307 return false;
308 OffsetEltIndex = OffsetEltIndexAP.getZExtValue();
309
310 if (!isSafeToLoadUnconditionally(V: SrcPtr, Ty: MinVecTy, Alignment: Align(1),
311 SQ: SQ.getWithInstruction(I: Load),
312 /*IgnoreFree=*/true))
313 return false;
314
315 // Update alignment with offset value. Note that the offset could be negated
316 // to more accurately represent "(new) SrcPtr - Offset = (old) SrcPtr", but
317 // negation does not change the result of the alignment calculation.
318 Alignment = commonAlignment(A: Alignment, Offset: Offset.getZExtValue());
319 }
320
321 // Original pattern: insertelt undef, load [free casts of] PtrOp, 0
322 // Use the greater of the alignment on the load or its source pointer.
323 Alignment = std::max(a: SrcPtr->getPointerAlignment(DL: *DL), b: Alignment);
324 Type *LoadTy = Load->getType();
325 unsigned AS = Load->getPointerAddressSpace();
326 InstructionCost OldCost =
327 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: LoadTy, Alignment, AddressSpace: AS, CostKind);
328 APInt DemandedElts = APInt::getOneBitSet(numBits: MinVecNumElts, BitNo: 0);
329 OldCost +=
330 TTI.getScalarizationOverhead(Ty: MinVecTy, DemandedElts,
331 /* Insert */ true, Extract: HasExtract, CostKind);
332
333 // New pattern: load VecPtr
334 InstructionCost NewCost =
335 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: MinVecTy, Alignment, AddressSpace: AS, CostKind);
336 // Optionally, we are shuffling the loaded vector element(s) into place.
337 // For the mask set everything but element 0 to undef to prevent poison from
338 // propagating from the extra loaded memory. This will also optionally
339 // shrink/grow the vector from the loaded size to the output size.
340 // We assume this operation has no cost in codegen if there was no offset.
341 // Note that we could use freeze to avoid poison problems, but then we might
342 // still need a shuffle to change the vector size.
343 auto *Ty = cast<FixedVectorType>(Val: I.getType());
344 unsigned OutputNumElts = Ty->getNumElements();
345 SmallVector<int, 16> Mask(OutputNumElts, PoisonMaskElem);
346 assert(OffsetEltIndex < MinVecNumElts && "Address offset too big");
347 Mask[0] = OffsetEltIndex;
348 if (OffsetEltIndex)
349 NewCost += TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: Ty, SrcTy: MinVecTy,
350 CostKind, Mask);
351
352 // We can aggressively convert to the vector form because the backend can
353 // invert this transform if it does not result in a performance win.
354 if (OldCost < NewCost || !NewCost.isValid())
355 return false;
356
357 // It is safe and potentially profitable to load a vector directly:
358 // inselt undef, load Scalar, 0 --> load VecPtr
359 Builder.SetInsertPoint(Load);
360 Value *CastedPtr =
361 Builder.CreatePointerBitCastOrAddrSpaceCast(V: SrcPtr, DestTy: Builder.getPtrTy(AddrSpace: AS));
362 Value *VecLd = Builder.CreateAlignedLoad(Ty: MinVecTy, Ptr: CastedPtr, Align: Alignment);
363 VecLd = Builder.CreateShuffleVector(V: VecLd, Mask);
364
365 replaceValue(Old&: I, New&: *VecLd);
366 ++NumVecLoad;
367 return true;
368}
369
370/// If we are loading a vector and then inserting it into a larger vector with
371/// undefined elements, try to load the larger vector and eliminate the insert.
372/// This removes a shuffle in IR and may allow combining of other loaded values.
373bool VectorCombine::widenSubvectorLoad(Instruction &I) {
374 // Match subvector insert of fixed vector.
375 auto *Shuf = cast<ShuffleVectorInst>(Val: &I);
376 if (!Shuf->isIdentityWithPadding())
377 return false;
378
379 // Allow a non-canonical shuffle mask that is choosing elements from op1.
380 unsigned NumOpElts =
381 cast<FixedVectorType>(Val: Shuf->getOperand(i_nocapture: 0)->getType())->getNumElements();
382 unsigned OpIndex = any_of(Range: Shuf->getShuffleMask(), P: [&NumOpElts](int M) {
383 return M >= (int)(NumOpElts);
384 });
385
386 auto *Load = dyn_cast<LoadInst>(Val: Shuf->getOperand(i_nocapture: OpIndex));
387 if (!canWidenLoad(Load, TTI))
388 return false;
389
390 // We use minimal alignment (maximum flexibility) because we only care about
391 // the dereferenceable region. When calculating cost and creating a new op,
392 // we may use a larger value based on alignment attributes.
393 auto *Ty = cast<FixedVectorType>(Val: I.getType());
394 Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts();
395 assert(isa<PointerType>(SrcPtr->getType()) && "Expected a pointer type");
396 Align Alignment = Load->getAlign();
397 // Ignore frees because the existing load proves that the object has not been
398 // freed. Allocations cannot shrink, so this is sufficient.
399 if (!isSafeToLoadUnconditionally(V: SrcPtr, Ty, Alignment: Align(1),
400 SQ: SQ.getWithInstruction(I: Load),
401 /*IgnoreFree=*/true))
402 return false;
403
404 Alignment = std::max(a: SrcPtr->getPointerAlignment(DL: *DL), b: Alignment);
405 Type *LoadTy = Load->getType();
406 unsigned AS = Load->getPointerAddressSpace();
407
408 // Original pattern: insert_subvector (load PtrOp)
409 // This conservatively assumes that the cost of a subvector insert into an
410 // undef value is 0. We could add that cost if the cost model accurately
411 // reflects the real cost of that operation.
412 InstructionCost OldCost =
413 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: LoadTy, Alignment, AddressSpace: AS, CostKind);
414
415 // New pattern: load PtrOp
416 InstructionCost NewCost =
417 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment, AddressSpace: AS, CostKind);
418
419 // We can aggressively convert to the vector form because the backend can
420 // invert this transform if it does not result in a performance win.
421 if (OldCost < NewCost || !NewCost.isValid())
422 return false;
423
424 Builder.SetInsertPoint(Load);
425 Value *CastedPtr =
426 Builder.CreatePointerBitCastOrAddrSpaceCast(V: SrcPtr, DestTy: Builder.getPtrTy(AddrSpace: AS));
427 Value *VecLd = Builder.CreateAlignedLoad(Ty, Ptr: CastedPtr, Align: Alignment);
428 replaceValue(Old&: I, New&: *VecLd);
429 ++NumVecLoad;
430 return true;
431}
432
433/// Determine which, if any, of the inputs should be replaced by a shuffle
434/// followed by extract from a different index.
435ExtractElementInst *VectorCombine::getShuffleExtract(
436 ExtractElementInst *Ext0, ExtractElementInst *Ext1,
437 unsigned PreferredExtractIndex = InvalidIndex) const {
438 auto *Index0C = dyn_cast<ConstantInt>(Val: Ext0->getIndexOperand());
439 auto *Index1C = dyn_cast<ConstantInt>(Val: Ext1->getIndexOperand());
440 assert(Index0C && Index1C && "Expected constant extract indexes");
441
442 unsigned Index0 = Index0C->getZExtValue();
443 unsigned Index1 = Index1C->getZExtValue();
444
445 // If the extract indexes are identical, no shuffle is needed.
446 if (Index0 == Index1)
447 return nullptr;
448
449 Type *VecTy = Ext0->getVectorOperand()->getType();
450 assert(VecTy == Ext1->getVectorOperand()->getType() && "Need matching types");
451 InstructionCost Cost0 =
452 TTI.getVectorInstrCost(I: *Ext0, Val: VecTy, CostKind, Index: Index0);
453 InstructionCost Cost1 =
454 TTI.getVectorInstrCost(I: *Ext1, Val: VecTy, CostKind, Index: Index1);
455
456 // If both costs are invalid no shuffle is needed
457 if (!Cost0.isValid() && !Cost1.isValid())
458 return nullptr;
459
460 // We are extracting from 2 different indexes, so one operand must be shuffled
461 // before performing a vector operation and/or extract. The more expensive
462 // extract will be replaced by a shuffle.
463 if (Cost0 > Cost1)
464 return Ext0;
465 if (Cost1 > Cost0)
466 return Ext1;
467
468 // If the costs are equal and there is a preferred extract index, shuffle the
469 // opposite operand.
470 if (PreferredExtractIndex == Index0)
471 return Ext1;
472 if (PreferredExtractIndex == Index1)
473 return Ext0;
474
475 // Otherwise, replace the extract with the higher index.
476 return Index0 > Index1 ? Ext0 : Ext1;
477}
478
479/// Compare the relative costs of 2 extracts followed by scalar operation vs.
480/// vector operation(s) followed by extract. Return true if the existing
481/// instructions are cheaper than a vector alternative. Otherwise, return false
482/// and if one of the extracts should be transformed to a shufflevector, set
483/// \p ConvertToShuffle to that extract instruction.
484bool VectorCombine::isExtractExtractCheap(ExtractElementInst *Ext0,
485 ExtractElementInst *Ext1,
486 const Instruction &I,
487 ExtractElementInst *&ConvertToShuffle,
488 unsigned PreferredExtractIndex) {
489 auto *Ext0IndexC = dyn_cast<ConstantInt>(Val: Ext0->getIndexOperand());
490 auto *Ext1IndexC = dyn_cast<ConstantInt>(Val: Ext1->getIndexOperand());
491 assert(Ext0IndexC && Ext1IndexC && "Expected constant extract indexes");
492
493 unsigned Opcode = I.getOpcode();
494 Value *Ext0Src = Ext0->getVectorOperand();
495 Value *Ext1Src = Ext1->getVectorOperand();
496 Type *ScalarTy = Ext0->getType();
497 auto *VecTy = cast<VectorType>(Val: Ext0Src->getType());
498 InstructionCost ScalarOpCost, VectorOpCost;
499
500 // Get cost estimates for scalar and vector versions of the operation.
501 bool IsBinOp = Instruction::isBinaryOp(Opcode);
502 if (IsBinOp) {
503 ScalarOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: ScalarTy, CostKind);
504 VectorOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
505 } else {
506 assert((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
507 "Expected a compare");
508 CmpInst::Predicate Pred = cast<CmpInst>(Val: I).getPredicate();
509 ScalarOpCost = TTI.getCmpSelInstrCost(
510 Opcode, ValTy: ScalarTy, CondTy: CmpInst::makeCmpResultType(opnd_type: ScalarTy), VecPred: Pred, CostKind);
511 VectorOpCost = TTI.getCmpSelInstrCost(
512 Opcode, ValTy: VecTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VecTy), VecPred: Pred, CostKind);
513 }
514
515 // Get cost estimates for the extract elements. These costs will factor into
516 // both sequences.
517 unsigned Ext0Index = Ext0IndexC->getZExtValue();
518 unsigned Ext1Index = Ext1IndexC->getZExtValue();
519
520 InstructionCost Extract0Cost =
521 TTI.getVectorInstrCost(I: *Ext0, Val: VecTy, CostKind, Index: Ext0Index);
522 InstructionCost Extract1Cost =
523 TTI.getVectorInstrCost(I: *Ext1, Val: VecTy, CostKind, Index: Ext1Index);
524
525 // A more expensive extract will always be replaced by a splat shuffle.
526 // For example, if Ext0 is more expensive:
527 // opcode (extelt V0, Ext0), (ext V1, Ext1) -->
528 // extelt (opcode (splat V0, Ext0), V1), Ext1
529 // TODO: Evaluate whether that always results in lowest cost. Alternatively,
530 // check the cost of creating a broadcast shuffle and shuffling both
531 // operands to element 0.
532 unsigned BestExtIndex = Extract0Cost > Extract1Cost ? Ext0Index : Ext1Index;
533 unsigned BestInsIndex = Extract0Cost > Extract1Cost ? Ext1Index : Ext0Index;
534 InstructionCost CheapExtractCost = std::min(a: Extract0Cost, b: Extract1Cost);
535
536 // Extra uses of the extracts mean that we include those costs in the
537 // vector total because those instructions will not be eliminated.
538 InstructionCost OldCost, NewCost;
539 if (Ext0Src == Ext1Src && Ext0Index == Ext1Index) {
540 // Handle a special case. If the 2 extracts are identical, adjust the
541 // formulas to account for that. The extra use charge allows for either the
542 // CSE'd pattern or an unoptimized form with identical values:
543 // opcode (extelt V, C), (extelt V, C) --> extelt (opcode V, V), C
544 bool HasUseTax = Ext0 == Ext1 ? !Ext0->hasNUses(N: 2)
545 : !Ext0->hasOneUse() || !Ext1->hasOneUse();
546 OldCost = CheapExtractCost + ScalarOpCost;
547 NewCost = VectorOpCost + CheapExtractCost + HasUseTax * CheapExtractCost;
548 } else {
549 // Handle the general case. Each extract is actually a different value:
550 // opcode (extelt V0, C0), (extelt V1, C1) --> extelt (opcode V0, V1), C
551 OldCost = Extract0Cost + Extract1Cost + ScalarOpCost;
552 NewCost = VectorOpCost + CheapExtractCost +
553 !Ext0->hasOneUse() * Extract0Cost +
554 !Ext1->hasOneUse() * Extract1Cost;
555 }
556
557 ConvertToShuffle = getShuffleExtract(Ext0, Ext1, PreferredExtractIndex);
558 if (ConvertToShuffle) {
559 if (IsBinOp && Opts.disable_binop_extract_shuffle)
560 return true;
561
562 // If we are extracting from 2 different indexes, then one operand must be
563 // shuffled before performing the vector operation. The shuffle mask is
564 // poison except for 1 lane that is being translated to the remaining
565 // extraction lane. Therefore, it is a splat shuffle. Ex:
566 // ShufMask = { poison, poison, 0, poison }
567 // TODO: The cost model has an option for a "broadcast" shuffle
568 // (splat-from-element-0), but no option for a more general splat.
569 if (auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: VecTy)) {
570 SmallVector<int> ShuffleMask(FixedVecTy->getNumElements(),
571 PoisonMaskElem);
572 ShuffleMask[BestInsIndex] = BestExtIndex;
573 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
574 DstTy: VecTy, SrcTy: VecTy, CostKind, Mask: ShuffleMask, Index: 0,
575 SubTp: nullptr, Args: {ConvertToShuffle});
576 } else {
577 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
578 DstTy: VecTy, SrcTy: VecTy, CostKind, Mask: {}, Index: 0, SubTp: nullptr,
579 Args: {ConvertToShuffle});
580 }
581 }
582
583 LLVM_DEBUG(dbgs() << "Found a binop of extractions: " << I << "\n OldCost: "
584 << OldCost << " vs NewCost: " << NewCost << "\n");
585
586 // Aggressively form a vector op if the cost is equal because the transform
587 // may enable further optimization.
588 // Codegen can reverse this transform (scalarize) if it was not profitable.
589 return OldCost < NewCost;
590}
591
592/// Create a shuffle that translates (shifts) 1 element from the input vector
593/// to a new element location.
594static Value *createShiftShuffle(Value *Vec, unsigned OldIndex,
595 unsigned NewIndex, IRBuilderBase &Builder) {
596 // The shuffle mask is poison except for 1 lane that is being translated
597 // to the new element index. Example for OldIndex == 2 and NewIndex == 0:
598 // ShufMask = { 2, poison, poison, poison }
599 auto *VecTy = cast<FixedVectorType>(Val: Vec->getType());
600 SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
601 ShufMask[NewIndex] = OldIndex;
602 return Builder.CreateShuffleVector(V: Vec, Mask: ShufMask, Name: "shift");
603}
604
605/// Given an extract element instruction with constant index operand, shuffle
606/// the source vector (shift the scalar element) to a NewIndex for extraction.
607/// Return null if the input can be constant folded, so that we are not creating
608/// unnecessary instructions.
609static Value *translateExtract(ExtractElementInst *ExtElt, unsigned NewIndex,
610 IRBuilderBase &Builder) {
611 // Shufflevectors can only be created for fixed-width vectors.
612 Value *X = ExtElt->getVectorOperand();
613 if (!isa<FixedVectorType>(Val: X->getType()))
614 return nullptr;
615
616 // If the extract can be constant-folded, this code is unsimplified. Defer
617 // to other passes to handle that.
618 Value *C = ExtElt->getIndexOperand();
619 assert(isa<ConstantInt>(C) && "Expected a constant index operand");
620 if (isa<Constant>(Val: X))
621 return nullptr;
622
623 Value *Shuf = createShiftShuffle(Vec: X, OldIndex: cast<ConstantInt>(Val: C)->getZExtValue(),
624 NewIndex, Builder);
625 return Shuf;
626}
627
628/// Try to reduce extract element costs by converting scalar compares to vector
629/// compares followed by extract.
630/// cmp (ext0 V0, ExtIndex), (ext1 V1, ExtIndex)
631Value *VectorCombine::foldExtExtCmp(Value *V0, Value *V1, Value *ExtIndex,
632 Instruction &I) {
633 assert(isa<CmpInst>(&I) && "Expected a compare");
634
635 // cmp Pred (extelt V0, ExtIndex), (extelt V1, ExtIndex)
636 // --> extelt (cmp Pred V0, V1), ExtIndex
637 ++NumVecCmp;
638 CmpInst::Predicate Pred = cast<CmpInst>(Val: &I)->getPredicate();
639 Value *VecCmp = Builder.CreateCmp(Pred, LHS: V0, RHS: V1);
640 return Builder.CreateExtractElement(Vec: VecCmp, Idx: ExtIndex, Name: "foldExtExtCmp");
641}
642
643/// Try to reduce extract element costs by converting scalar binops to vector
644/// binops followed by extract.
645/// bo (ext0 V0, ExtIndex), (ext1 V1, ExtIndex)
646Value *VectorCombine::foldExtExtBinop(Value *V0, Value *V1, Value *ExtIndex,
647 Instruction &I) {
648 assert(isa<BinaryOperator>(&I) && "Expected a binary operator");
649
650 // bo (extelt V0, ExtIndex), (extelt V1, ExtIndex)
651 // --> extelt (bo V0, V1), ExtIndex
652 ++NumVecBO;
653 Value *VecBO = Builder.CreateBinOp(Opc: cast<BinaryOperator>(Val: &I)->getOpcode(), LHS: V0,
654 RHS: V1, Name: "foldExtExtBinop");
655
656 // All IR flags are safe to back-propagate because any potential poison
657 // created in unused vector elements is discarded by the extract.
658 if (auto *VecBOInst = dyn_cast<Instruction>(Val: VecBO))
659 VecBOInst->copyIRFlags(V: &I);
660
661 return Builder.CreateExtractElement(Vec: VecBO, Idx: ExtIndex, Name: "foldExtExtBinop");
662}
663
664/// Match an instruction with extracted vector operands.
665bool VectorCombine::foldExtractExtract(Instruction &I) {
666 // It is not safe to transform things like div, urem, etc. because we may
667 // create undefined behavior when executing those on unknown vector elements.
668 if (!isSafeToSpeculativelyExecute(I: &I))
669 return false;
670
671 Instruction *I0, *I1;
672 CmpPredicate Pred = CmpInst::BAD_ICMP_PREDICATE;
673 if (!match(V: &I, P: m_Cmp(Pred, L: m_Instruction(I&: I0), R: m_Instruction(I&: I1))) &&
674 !match(V: &I, P: m_BinOp(L: m_Instruction(I&: I0), R: m_Instruction(I&: I1))))
675 return false;
676
677 Value *V0, *V1;
678 uint64_t C0, C1;
679 if (!match(V: I0, P: m_ExtractElt(Val: m_Value(V&: V0), Idx: m_ConstantInt(V&: C0))) ||
680 !match(V: I1, P: m_ExtractElt(Val: m_Value(V&: V1), Idx: m_ConstantInt(V&: C1))) ||
681 V0->getType() != V1->getType())
682 return false;
683
684 // For fixed-width vectors, reject out-of-bounds extract indexes
685 if (auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: V0->getType())) {
686 unsigned NumElts = FixedVecTy->getNumElements();
687 if (C0 >= NumElts || C1 >= NumElts)
688 return false;
689 }
690
691 // If the scalar value 'I' is going to be re-inserted into a vector, then try
692 // to create an extract to that same element. The extract/insert can be
693 // reduced to a "select shuffle".
694 // TODO: If we add a larger pattern match that starts from an insert, this
695 // probably becomes unnecessary.
696 auto *Ext0 = cast<ExtractElementInst>(Val: I0);
697 auto *Ext1 = cast<ExtractElementInst>(Val: I1);
698 uint64_t InsertIndex = InvalidIndex;
699 if (I.hasOneUse())
700 match(V: I.user_back(),
701 P: m_InsertElt(Val: m_Value(), Elt: m_Value(), Idx: m_ConstantInt(V&: InsertIndex)));
702
703 ExtractElementInst *ExtractToChange;
704 if (isExtractExtractCheap(Ext0, Ext1, I, ConvertToShuffle&: ExtractToChange, PreferredExtractIndex: InsertIndex))
705 return false;
706
707 Value *ExtOp0 = Ext0->getVectorOperand();
708 Value *ExtOp1 = Ext1->getVectorOperand();
709
710 if (ExtractToChange) {
711 unsigned CheapExtractIdx = ExtractToChange == Ext0 ? C1 : C0;
712 Value *NewExtOp =
713 translateExtract(ExtElt: ExtractToChange, NewIndex: CheapExtractIdx, Builder);
714 if (!NewExtOp)
715 return false;
716 if (ExtractToChange == Ext0)
717 ExtOp0 = NewExtOp;
718 else
719 ExtOp1 = NewExtOp;
720 }
721
722 Value *ExtIndex = ExtractToChange == Ext0 ? Ext1->getIndexOperand()
723 : Ext0->getIndexOperand();
724 Value *NewExt = Pred != CmpInst::BAD_ICMP_PREDICATE
725 ? foldExtExtCmp(V0: ExtOp0, V1: ExtOp1, ExtIndex, I)
726 : foldExtExtBinop(V0: ExtOp0, V1: ExtOp1, ExtIndex, I);
727 replaceValue(Old&: I, New&: *NewExt);
728 return true;
729}
730
731/// Try to replace an extract + scalar fneg + insert with a vector fneg +
732/// shuffle.
733bool VectorCombine::foldInsExtFNeg(Instruction &I) {
734 // Match an insert (op (extract)) pattern.
735 Value *DstVec;
736 uint64_t ExtIdx, InsIdx;
737 Instruction *FNeg;
738 if (!match(V: &I, P: m_InsertElt(Val: m_Value(V&: DstVec), Elt: m_OneUse(SubPattern: m_Instruction(I&: FNeg)),
739 Idx: m_ConstantInt(V&: InsIdx))))
740 return false;
741
742 // Note: This handles the canonical fneg instruction and "fsub -0.0, X".
743 Value *SrcVec;
744 Instruction *Extract;
745 if (!match(V: FNeg, P: m_FNeg(X: m_CombineAnd(
746 Ps: m_Instruction(I&: Extract),
747 Ps: m_ExtractElt(Val: m_Value(V&: SrcVec), Idx: m_ConstantInt(V&: ExtIdx))))))
748 return false;
749
750 auto *DstVecTy = cast<FixedVectorType>(Val: DstVec->getType());
751 auto *DstVecScalarTy = DstVecTy->getScalarType();
752 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcVec->getType());
753 if (!SrcVecTy || DstVecScalarTy != SrcVecTy->getScalarType())
754 return false;
755
756 // Ignore if insert/extract index is out of bounds or destination vector has
757 // one element
758 unsigned NumDstElts = DstVecTy->getNumElements();
759 unsigned NumSrcElts = SrcVecTy->getNumElements();
760 if (ExtIdx > NumSrcElts || InsIdx >= NumDstElts || NumDstElts == 1)
761 return false;
762
763 // We are inserting the negated element into the same lane that we extracted
764 // from. This is equivalent to a select-shuffle that chooses all but the
765 // negated element from the destination vector.
766 SmallVector<int> Mask(NumDstElts);
767 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
768 Mask[InsIdx] = (ExtIdx % NumDstElts) + NumDstElts;
769 InstructionCost OldCost =
770 TTI.getArithmeticInstrCost(Opcode: Instruction::FNeg, Ty: DstVecScalarTy, CostKind) +
771 TTI.getVectorInstrCost(I, Val: DstVecTy, CostKind, Index: InsIdx);
772
773 // If the extract has one use, it will be eliminated, so count it in the
774 // original cost. If it has more than one use, ignore the cost because it will
775 // be the same before/after.
776 if (Extract->hasOneUse())
777 OldCost += TTI.getVectorInstrCost(I: *Extract, Val: SrcVecTy, CostKind, Index: ExtIdx);
778
779 InstructionCost NewCost =
780 TTI.getArithmeticInstrCost(Opcode: Instruction::FNeg, Ty: SrcVecTy, CostKind) +
781 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: DstVecTy,
782 SrcTy: DstVecTy, CostKind, Mask);
783
784 bool NeedLenChg = SrcVecTy->getNumElements() != NumDstElts;
785 // If the lengths of the two vectors are not equal,
786 // we need to add a length-change vector. Add this cost.
787 SmallVector<int> SrcMask;
788 if (NeedLenChg) {
789 SrcMask.assign(NumElts: NumDstElts, Elt: PoisonMaskElem);
790 SrcMask[ExtIdx % NumDstElts] = ExtIdx;
791 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
792 DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask: SrcMask);
793 }
794
795 LLVM_DEBUG(dbgs() << "Found an insertion of (extract)fneg : " << I
796 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
797 << "\n");
798 if (NewCost > OldCost)
799 return false;
800
801 Value *NewShuf, *LenChgShuf = nullptr;
802 // insertelt DstVec, (fneg (extractelt SrcVec, Index)), Index
803 Value *VecFNeg = Builder.CreateFNegFMF(V: SrcVec, FMFSource: FNeg);
804 if (NeedLenChg) {
805 // shuffle DstVec, (shuffle (fneg SrcVec), poison, SrcMask), Mask
806 LenChgShuf = Builder.CreateShuffleVector(V: VecFNeg, Mask: SrcMask);
807 NewShuf = Builder.CreateShuffleVector(V1: DstVec, V2: LenChgShuf, Mask);
808 } else {
809 // shuffle DstVec, (fneg SrcVec), Mask
810 NewShuf = Builder.CreateShuffleVector(V1: DstVec, V2: VecFNeg, Mask);
811 }
812
813 replaceValue(Old&: I, New&: *NewShuf);
814 return true;
815}
816
817/// Try to fold insert(binop(x,y),binop(a,b),idx)
818/// --> binop(insert(x,a,idx),insert(y,b,idx))
819bool VectorCombine::foldInsExtBinop(Instruction &I) {
820 BinaryOperator *VecBinOp, *SclBinOp;
821 uint64_t Index;
822 if (!match(V: &I,
823 P: m_InsertElt(Val: m_OneUse(SubPattern: m_BinOp(I&: VecBinOp)),
824 Elt: m_OneUse(SubPattern: m_BinOp(I&: SclBinOp)), Idx: m_ConstantInt(V&: Index))))
825 return false;
826
827 // TODO: Add support for addlike etc.
828 Instruction::BinaryOps BinOpcode = VecBinOp->getOpcode();
829 if (BinOpcode != SclBinOp->getOpcode())
830 return false;
831
832 auto *ResultTy = dyn_cast<FixedVectorType>(Val: I.getType());
833 if (!ResultTy)
834 return false;
835
836 // TODO: Attempt to detect m_ExtractElt for scalar operands and convert to
837 // shuffle?
838
839 InstructionCost OldCost = TTI.getInstructionCost(U: &I, CostKind) +
840 TTI.getInstructionCost(U: VecBinOp, CostKind) +
841 TTI.getInstructionCost(U: SclBinOp, CostKind);
842 InstructionCost NewCost =
843 TTI.getArithmeticInstrCost(Opcode: BinOpcode, Ty: ResultTy, CostKind) +
844 TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: ResultTy, CostKind,
845 Index, Op0: VecBinOp->getOperand(i_nocapture: 0),
846 Op1: SclBinOp->getOperand(i_nocapture: 0)) +
847 TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: ResultTy, CostKind,
848 Index, Op0: VecBinOp->getOperand(i_nocapture: 1),
849 Op1: SclBinOp->getOperand(i_nocapture: 1));
850
851 LLVM_DEBUG(dbgs() << "Found an insertion of two binops: " << I
852 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
853 << "\n");
854 if (NewCost > OldCost)
855 return false;
856
857 Value *NewIns0 = Builder.CreateInsertElement(Vec: VecBinOp->getOperand(i_nocapture: 0),
858 NewElt: SclBinOp->getOperand(i_nocapture: 0), Idx: Index);
859 Value *NewIns1 = Builder.CreateInsertElement(Vec: VecBinOp->getOperand(i_nocapture: 1),
860 NewElt: SclBinOp->getOperand(i_nocapture: 1), Idx: Index);
861 Value *NewBO = Builder.CreateBinOp(Opc: BinOpcode, LHS: NewIns0, RHS: NewIns1);
862
863 // Intersect flags from the old binops.
864 if (auto *NewInst = dyn_cast<Instruction>(Val: NewBO)) {
865 NewInst->copyIRFlags(V: VecBinOp);
866 NewInst->andIRFlags(V: SclBinOp);
867 }
868
869 replaceValue(Old&: I, New&: *NewBO);
870 return true;
871}
872
873/// Match: bitop(castop(x), castop(y)) -> castop(bitop(x, y))
874/// Supports: bitcast, trunc, sext, zext
875bool VectorCombine::foldBitOpOfCastops(Instruction &I) {
876 // Check if this is a bitwise logic operation
877 auto *BinOp = dyn_cast<BinaryOperator>(Val: &I);
878 if (!BinOp || !BinOp->isBitwiseLogicOp())
879 return false;
880
881 // Get the cast instructions
882 auto *LHSCast = dyn_cast<CastInst>(Val: BinOp->getOperand(i_nocapture: 0));
883 auto *RHSCast = dyn_cast<CastInst>(Val: BinOp->getOperand(i_nocapture: 1));
884 if (!LHSCast || !RHSCast) {
885 LLVM_DEBUG(dbgs() << " One or both operands are not cast instructions\n");
886 return false;
887 }
888
889 // Both casts must be the same type
890 Instruction::CastOps CastOpcode = LHSCast->getOpcode();
891 if (CastOpcode != RHSCast->getOpcode())
892 return false;
893
894 // Only handle supported cast operations
895 switch (CastOpcode) {
896 case Instruction::BitCast:
897 case Instruction::Trunc:
898 case Instruction::SExt:
899 case Instruction::ZExt:
900 break;
901 default:
902 return false;
903 }
904
905 Value *LHSSrc = LHSCast->getOperand(i_nocapture: 0);
906 Value *RHSSrc = RHSCast->getOperand(i_nocapture: 0);
907
908 // Source types must match
909 if (LHSSrc->getType() != RHSSrc->getType())
910 return false;
911
912 auto *SrcTy = LHSSrc->getType();
913 auto *DstTy = I.getType();
914 // Bitcasts can handle scalar/vector mixes, such as i16 -> <16 x i1>.
915 // Other casts only handle vector types with integer elements.
916 if (CastOpcode != Instruction::BitCast &&
917 (!isa<FixedVectorType>(Val: SrcTy) || !isa<FixedVectorType>(Val: DstTy)))
918 return false;
919
920 // Only integer scalar/vector values are legal for bitwise logic operations.
921 if (!SrcTy->getScalarType()->isIntegerTy() ||
922 !DstTy->getScalarType()->isIntegerTy())
923 return false;
924
925 // Cost Check :
926 // OldCost = bitlogic + 2*casts
927 // NewCost = bitlogic + cast
928
929 // Calculate specific costs for each cast with instruction context
930 InstructionCost LHSCastCost = TTI.getCastInstrCost(
931 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind, I: LHSCast);
932 InstructionCost RHSCastCost = TTI.getCastInstrCost(
933 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind, I: RHSCast);
934
935 InstructionCost OldCost =
936 TTI.getArithmeticInstrCost(Opcode: BinOp->getOpcode(), Ty: DstTy, CostKind) +
937 LHSCastCost + RHSCastCost;
938
939 // For new cost, we can't provide an instruction (it doesn't exist yet)
940 InstructionCost GenericCastCost = TTI.getCastInstrCost(
941 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
942
943 InstructionCost NewCost =
944 TTI.getArithmeticInstrCost(Opcode: BinOp->getOpcode(), Ty: SrcTy, CostKind) +
945 GenericCastCost;
946
947 // Account for multi-use casts using specific costs
948 if (!LHSCast->hasOneUse())
949 NewCost += LHSCastCost;
950 if (!RHSCast->hasOneUse())
951 NewCost += RHSCastCost;
952
953 LLVM_DEBUG(dbgs() << "foldBitOpOfCastops: OldCost=" << OldCost
954 << " NewCost=" << NewCost << "\n");
955
956 if (NewCost > OldCost)
957 return false;
958
959 // Create the operation on the source type
960 Value *NewOp = Builder.CreateBinOp(Opc: BinOp->getOpcode(), LHS: LHSSrc, RHS: RHSSrc,
961 Name: BinOp->getName() + ".inner");
962 if (auto *NewBinOp = dyn_cast<BinaryOperator>(Val: NewOp))
963 NewBinOp->copyIRFlags(V: BinOp);
964
965 // Create the cast operation directly to ensure we get a new instruction
966 Instruction *NewCast = CastInst::Create(CastOpcode, S: NewOp, Ty: I.getType());
967
968 // Preserve cast instruction flags
969 NewCast->copyIRFlags(V: LHSCast);
970 NewCast->andIRFlags(V: RHSCast);
971
972 // Insert the new instruction
973 Value *Result = Builder.Insert(I: NewCast);
974
975 replaceValue(Old&: I, New&: *Result);
976 return true;
977}
978
979/// Match:
980// bitop(castop(x), C) ->
981// bitop(castop(x), castop(InvC)) ->
982// castop(bitop(x, InvC))
983// Supports: bitcast
984bool VectorCombine::foldBitOpOfCastConstant(Instruction &I) {
985 Instruction *LHS;
986 Constant *C;
987
988 // Check if this is a bitwise logic operation
989 if (!match(V: &I, P: m_c_BitwiseLogic(L: m_Instruction(I&: LHS), R: m_Constant(C))))
990 return false;
991
992 // Get the cast instructions
993 auto *LHSCast = dyn_cast<CastInst>(Val: LHS);
994 if (!LHSCast)
995 return false;
996
997 Instruction::CastOps CastOpcode = LHSCast->getOpcode();
998
999 // Only handle supported cast operations
1000 switch (CastOpcode) {
1001 case Instruction::BitCast:
1002 case Instruction::ZExt:
1003 case Instruction::SExt:
1004 case Instruction::Trunc:
1005 break;
1006 default:
1007 return false;
1008 }
1009
1010 Value *LHSSrc = LHSCast->getOperand(i_nocapture: 0);
1011
1012 auto *SrcTy = LHSSrc->getType();
1013 auto *DstTy = I.getType();
1014 // Bitcasts can handle scalar/vector mixes, such as i16 -> <16 x i1>.
1015 // Other casts only handle vector types with integer elements.
1016 if (CastOpcode != Instruction::BitCast &&
1017 (!isa<FixedVectorType>(Val: SrcTy) || !isa<FixedVectorType>(Val: DstTy)))
1018 return false;
1019
1020 // Only integer scalar/vector values are legal for bitwise logic operations.
1021 if (!SrcTy->getScalarType()->isIntegerTy() ||
1022 !DstTy->getScalarType()->isIntegerTy())
1023 return false;
1024
1025 // Find the constant InvC, such that castop(InvC) equals to C.
1026 PreservedCastFlags RHSFlags;
1027 Constant *InvC = getLosslessInvCast(C, InvCastTo: SrcTy, CastOp: CastOpcode, DL: *DL, Flags: &RHSFlags);
1028 if (!InvC)
1029 return false;
1030
1031 // Cost Check :
1032 // OldCost = bitlogic + cast
1033 // NewCost = bitlogic + cast
1034
1035 // Calculate specific costs for each cast with instruction context
1036 InstructionCost LHSCastCost = TTI.getCastInstrCost(
1037 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind, I: LHSCast);
1038
1039 InstructionCost OldCost =
1040 TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: DstTy, CostKind) + LHSCastCost;
1041
1042 // For new cost, we can't provide an instruction (it doesn't exist yet)
1043 InstructionCost GenericCastCost = TTI.getCastInstrCost(
1044 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
1045
1046 InstructionCost NewCost =
1047 TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: SrcTy, CostKind) +
1048 GenericCastCost;
1049
1050 // Account for multi-use casts using specific costs
1051 if (!LHSCast->hasOneUse())
1052 NewCost += LHSCastCost;
1053
1054 LLVM_DEBUG(dbgs() << "foldBitOpOfCastConstant: OldCost=" << OldCost
1055 << " NewCost=" << NewCost << "\n");
1056
1057 if (NewCost > OldCost)
1058 return false;
1059
1060 // Create the operation on the source type
1061 Value *NewOp = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)I.getOpcode(),
1062 LHS: LHSSrc, RHS: InvC, Name: I.getName() + ".inner");
1063 if (auto *NewBinOp = dyn_cast<BinaryOperator>(Val: NewOp))
1064 NewBinOp->copyIRFlags(V: &I);
1065
1066 // Create the cast operation directly to ensure we get a new instruction
1067 Instruction *NewCast = CastInst::Create(CastOpcode, S: NewOp, Ty: I.getType());
1068
1069 // Preserve cast instruction flags
1070 if (RHSFlags.NNeg)
1071 NewCast->setNonNeg();
1072 if (RHSFlags.NUW)
1073 NewCast->setHasNoUnsignedWrap();
1074 if (RHSFlags.NSW)
1075 NewCast->setHasNoSignedWrap();
1076
1077 NewCast->andIRFlags(V: LHSCast);
1078
1079 // Insert the new instruction
1080 Value *Result = Builder.Insert(I: NewCast);
1081
1082 replaceValue(Old&: I, New&: *Result);
1083 return true;
1084}
1085
1086/// If this is a bitcast of a shuffle, try to bitcast the source vector to the
1087/// destination type followed by shuffle. This can enable further transforms by
1088/// moving bitcasts or shuffles together.
1089bool VectorCombine::foldBitcastShuffle(Instruction &I) {
1090 Value *V0, *V1;
1091 ArrayRef<int> Mask;
1092 if (!match(V: &I, P: m_BitCast(Op: m_OneUse(
1093 SubPattern: m_Shuffle(v1: m_Value(V&: V0), v2: m_Value(V&: V1), mask: m_Mask(Mask))))))
1094 return false;
1095
1096 // 1) Do not fold bitcast shuffle for scalable type. First, shuffle cost for
1097 // scalable type is unknown; Second, we cannot reason if the narrowed shuffle
1098 // mask for scalable type is a splat or not.
1099 // 2) Disallow non-vector casts.
1100 // TODO: We could allow any shuffle.
1101 auto *DestTy = dyn_cast<FixedVectorType>(Val: I.getType());
1102 auto *SrcTy = dyn_cast<FixedVectorType>(Val: V0->getType());
1103 if (!DestTy || !SrcTy)
1104 return false;
1105
1106 unsigned DestEltSize = DestTy->getScalarSizeInBits();
1107 unsigned SrcEltSize = SrcTy->getScalarSizeInBits();
1108 if (SrcTy->getPrimitiveSizeInBits() % DestEltSize != 0)
1109 return false;
1110
1111 bool IsUnary = isa<UndefValue>(Val: V1);
1112
1113 // For binary shuffles, only fold bitcast(shuffle(X,Y))
1114 // if it won't increase the number of bitcasts.
1115 if (!IsUnary) {
1116 auto *BCTy0 = dyn_cast<FixedVectorType>(Val: peekThroughBitcasts(V: V0)->getType());
1117 auto *BCTy1 = dyn_cast<FixedVectorType>(Val: peekThroughBitcasts(V: V1)->getType());
1118 if (!(BCTy0 && BCTy0->getElementType() == DestTy->getElementType()) &&
1119 !(BCTy1 && BCTy1->getElementType() == DestTy->getElementType()))
1120 return false;
1121 }
1122
1123 SmallVector<int, 16> NewMask;
1124 if (DestEltSize <= SrcEltSize) {
1125 // The bitcast is from wide to narrow/equal elements. The shuffle mask can
1126 // always be expanded to the equivalent form choosing narrower elements.
1127 if (SrcEltSize % DestEltSize != 0)
1128 return false;
1129 unsigned ScaleFactor = SrcEltSize / DestEltSize;
1130 narrowShuffleMaskElts(Scale: ScaleFactor, Mask, ScaledMask&: NewMask);
1131 } else {
1132 // The bitcast is from narrow elements to wide elements. The shuffle mask
1133 // must choose consecutive elements to allow casting first.
1134 if (DestEltSize % SrcEltSize != 0)
1135 return false;
1136 unsigned ScaleFactor = DestEltSize / SrcEltSize;
1137 if (!widenShuffleMaskElts(Scale: ScaleFactor, Mask, ScaledMask&: NewMask))
1138 return false;
1139 }
1140
1141 // Bitcast the shuffle src - keep its original width but using the destination
1142 // scalar type.
1143 unsigned NumSrcElts = SrcTy->getPrimitiveSizeInBits() / DestEltSize;
1144 auto *NewShuffleTy =
1145 FixedVectorType::get(ElementType: DestTy->getScalarType(), NumElts: NumSrcElts);
1146 auto *OldShuffleTy =
1147 FixedVectorType::get(ElementType: SrcTy->getScalarType(), NumElts: Mask.size());
1148 unsigned NumOps = IsUnary ? 1 : 2;
1149
1150 // The new shuffle must not cost more than the old shuffle.
1151 TargetTransformInfo::ShuffleKind SK =
1152 IsUnary ? TargetTransformInfo::SK_PermuteSingleSrc
1153 : TargetTransformInfo::SK_PermuteTwoSrc;
1154
1155 InstructionCost NewCost =
1156 TTI.getShuffleCost(Kind: SK, DstTy: DestTy, SrcTy: NewShuffleTy, CostKind, Mask: NewMask) +
1157 (NumOps * TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: NewShuffleTy, Src: SrcTy,
1158 CCH: TargetTransformInfo::CastContextHint::None,
1159 CostKind));
1160 InstructionCost OldCost =
1161 TTI.getShuffleCost(Kind: SK, DstTy: OldShuffleTy, SrcTy, CostKind, Mask) +
1162 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: DestTy, Src: OldShuffleTy,
1163 CCH: TargetTransformInfo::CastContextHint::None,
1164 CostKind);
1165
1166 LLVM_DEBUG(dbgs() << "Found a bitcasted shuffle: " << I << "\n OldCost: "
1167 << OldCost << " vs NewCost: " << NewCost << "\n");
1168
1169 if (NewCost > OldCost || !NewCost.isValid())
1170 return false;
1171
1172 // bitcast (shuf V0, V1, MaskC) --> shuf (bitcast V0), (bitcast V1), MaskC'
1173 ++NumShufOfBitcast;
1174 Value *CastV0 = Builder.CreateBitCast(V: peekThroughBitcasts(V: V0), DestTy: NewShuffleTy);
1175 Value *CastV1 = Builder.CreateBitCast(V: peekThroughBitcasts(V: V1), DestTy: NewShuffleTy);
1176 Value *Shuf = Builder.CreateShuffleVector(V1: CastV0, V2: CastV1, Mask: NewMask);
1177 replaceValue(Old&: I, New&: *Shuf);
1178 return true;
1179}
1180
1181/// Match a vector op/compare/intrinsic with at least one
1182/// inserted scalar operand and convert to scalar op/cmp/intrinsic followed
1183/// by insertelement.
1184bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
1185 auto *UO = dyn_cast<UnaryOperator>(Val: &I);
1186 auto *BO = dyn_cast<BinaryOperator>(Val: &I);
1187 auto *CI = dyn_cast<CmpInst>(Val: &I);
1188 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
1189 if (!UO && !BO && !CI && !II)
1190 return false;
1191
1192 // TODO: Allow intrinsics with different argument types
1193 if (II) {
1194 if (!isTriviallyVectorizable(ID: II->getIntrinsicID()))
1195 return false;
1196 for (auto [Idx, Arg] : enumerate(First: II->args()))
1197 if (Arg->getType() != II->getType() &&
1198 !isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(), ScalarOpdIdx: Idx, TTI: &TTI))
1199 return false;
1200 }
1201
1202 // Do not convert the vector condition of a vector select into a scalar
1203 // condition. That may cause problems for codegen because of differences in
1204 // boolean formats and register-file transfers.
1205 // TODO: Can we account for that in the cost model?
1206 if (CI)
1207 for (User *U : I.users())
1208 if (match(V: U, P: m_Select(C: m_Specific(V: &I), L: m_Value(), R: m_Value())))
1209 return false;
1210
1211 // Match constant vectors or scalars being inserted into constant vectors:
1212 // vec_op [VecC0 | (inselt VecC0, V0, Index)], ...
1213 SmallVector<Value *> VecCs, ScalarOps;
1214 std::optional<uint64_t> Index;
1215
1216 auto Ops = II ? II->args() : I.operands();
1217 for (auto [OpNum, Op] : enumerate(First&: Ops)) {
1218 Constant *VecC;
1219 Value *V;
1220 uint64_t InsIdx = 0;
1221 if (match(V: Op.get(), P: m_InsertElt(Val: m_Constant(C&: VecC), Elt: m_Value(V),
1222 Idx: m_ConstantInt(V&: InsIdx)))) {
1223 // Bail if any inserts are out of bounds.
1224 VectorType *OpTy = cast<VectorType>(Val: Op->getType());
1225 if (OpTy->getElementCount().getKnownMinValue() <= InsIdx)
1226 return false;
1227 // All inserts must have the same index.
1228 // TODO: Deal with mismatched index constants and variable indexes?
1229 if (!Index)
1230 Index = InsIdx;
1231 else if (InsIdx != *Index)
1232 return false;
1233 VecCs.push_back(Elt: VecC);
1234 ScalarOps.push_back(Elt: V);
1235 } else if (II && isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(),
1236 ScalarOpdIdx: OpNum, TTI: &TTI)) {
1237 VecCs.push_back(Elt: Op.get());
1238 ScalarOps.push_back(Elt: Op.get());
1239 } else if (match(V: Op.get(), P: m_Constant(C&: VecC))) {
1240 VecCs.push_back(Elt: VecC);
1241 ScalarOps.push_back(Elt: nullptr);
1242 } else {
1243 return false;
1244 }
1245 }
1246
1247 // Bail if all operands are constant.
1248 if (!Index.has_value())
1249 return false;
1250
1251 VectorType *VecTy = cast<VectorType>(Val: I.getType());
1252 Type *ScalarTy = VecTy->getScalarType();
1253 assert(VecTy->isVectorTy() &&
1254 (ScalarTy->isIntegerTy() || ScalarTy->isFloatingPointTy() ||
1255 ScalarTy->isPointerTy()) &&
1256 "Unexpected types for insert element into binop or cmp");
1257
1258 unsigned Opcode = I.getOpcode();
1259 InstructionCost ScalarOpCost, VectorOpCost;
1260 if (CI) {
1261 CmpInst::Predicate Pred = CI->getPredicate();
1262 ScalarOpCost = TTI.getCmpSelInstrCost(
1263 Opcode, ValTy: ScalarTy, CondTy: CmpInst::makeCmpResultType(opnd_type: ScalarTy), VecPred: Pred, CostKind);
1264 VectorOpCost = TTI.getCmpSelInstrCost(
1265 Opcode, ValTy: VecTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VecTy), VecPred: Pred, CostKind);
1266 } else if (UO || BO) {
1267 ScalarOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: ScalarTy, CostKind);
1268 VectorOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1269 } else {
1270 IntrinsicCostAttributes ScalarICA(
1271 II->getIntrinsicID(), ScalarTy,
1272 SmallVector<Type *>(II->arg_size(), ScalarTy));
1273 ScalarOpCost = TTI.getIntrinsicInstrCost(ICA: ScalarICA, CostKind);
1274 IntrinsicCostAttributes VectorICA(
1275 II->getIntrinsicID(), VecTy,
1276 SmallVector<Type *>(II->arg_size(), VecTy));
1277 VectorOpCost = TTI.getIntrinsicInstrCost(ICA: VectorICA, CostKind);
1278 }
1279
1280 // Fold the vector constants in the original vectors into a new base vector to
1281 // get more accurate cost modelling.
1282 Value *NewVecC = nullptr;
1283 if (CI)
1284 NewVecC = simplifyCmpInst(Predicate: CI->getPredicate(), LHS: VecCs[0], RHS: VecCs[1], Q: SQ);
1285 else if (UO)
1286 NewVecC =
1287 simplifyUnOp(Opcode: UO->getOpcode(), Op: VecCs[0], FMF: UO->getFastMathFlags(), Q: SQ);
1288 else if (BO)
1289 NewVecC = simplifyBinOp(Opcode: BO->getOpcode(), LHS: VecCs[0], RHS: VecCs[1], Q: SQ);
1290 else if (II)
1291 NewVecC = simplifyCall(Call: II, Callee: II->getCalledOperand(), Args: VecCs, Q: SQ);
1292
1293 if (!NewVecC)
1294 return false;
1295
1296 // Get cost estimate for the insert element. This cost will factor into
1297 // both sequences.
1298 InstructionCost OldCost = VectorOpCost;
1299 InstructionCost NewCost =
1300 ScalarOpCost + TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: VecTy,
1301 CostKind, Index: *Index, Op0: NewVecC);
1302
1303 for (auto [Idx, Op, VecC, Scalar] : enumerate(First&: Ops, Rest&: VecCs, Rest&: ScalarOps)) {
1304 if (!Scalar || (II && isVectorIntrinsicWithScalarOpAtArg(
1305 ID: II->getIntrinsicID(), ScalarOpdIdx: Idx, TTI: &TTI)))
1306 continue;
1307 InstructionCost InsertCost = TTI.getVectorInstrCost(
1308 Opcode: Instruction::InsertElement, Val: VecTy, CostKind, Index: *Index, Op0: VecC, Op1: Scalar);
1309 OldCost += InsertCost;
1310 NewCost += !Op->hasOneUse() * InsertCost;
1311 }
1312
1313 // We want to scalarize unless the vector variant actually has lower cost.
1314 if (OldCost < NewCost || !NewCost.isValid())
1315 return false;
1316
1317 // vec_op (inselt VecC0, V0, Index), (inselt VecC1, V1, Index) -->
1318 // inselt NewVecC, (scalar_op V0, V1), Index
1319 if (CI)
1320 ++NumScalarCmp;
1321 else if (UO || BO)
1322 ++NumScalarOps;
1323 else
1324 ++NumScalarIntrinsic;
1325
1326 // For constant cases, extract the scalar element, this should constant fold.
1327 for (auto [OpIdx, Scalar, VecC] : enumerate(First&: ScalarOps, Rest&: VecCs))
1328 if (!Scalar)
1329 ScalarOps[OpIdx] = ConstantExpr::getExtractElement(
1330 Vec: cast<Constant>(Val: VecC), Idx: Builder.getInt64(C: *Index));
1331
1332 Value *Scalar;
1333 // We need to pass the flags during the creation of instrucitons. Constant
1334 // folding might remove the instructions, so post setting the flags might
1335 // pollute the later instructions.
1336 if (CI) {
1337 if (FPMathOperator *FPMO = dyn_cast<FPMathOperator>(Val: &I)) {
1338 Scalar = Builder.CreateFCmpFMF(P: CI->getPredicate(), LHS: ScalarOps[0],
1339 RHS: ScalarOps[1], FMFSource: FPMO->getFastMathFlags(),
1340 Name: CI->getName() + ".scalar");
1341 } else {
1342 Scalar = Builder.CreateICmp(P: CI->getPredicate(), LHS: ScalarOps[0],
1343 RHS: ScalarOps[1], Name: CI->getName() + ".scalar");
1344 }
1345 } else if (UO) {
1346 Scalar = Builder.CreateUnOpFMF(Opc: UO->getOpcode(), V: ScalarOps[0], FMFSource: UO,
1347 Name: UO->getName() + ".scalar");
1348 } else if (BO) {
1349 if (OverflowingBinaryOperator *OBO =
1350 dyn_cast<OverflowingBinaryOperator>(Val: &I)) {
1351 Scalar = Builder.CreateNoWrapBinOp(
1352 Opc: BO->getOpcode(), LHS: ScalarOps[0], RHS: ScalarOps[1], IsNUW: OBO->hasNoUnsignedWrap(),
1353 IsNSW: OBO->hasNoSignedWrap(), Name: BO->getName() + ".scalar");
1354 } else if (PossiblyDisjointInst *PDI = dyn_cast<PossiblyDisjointInst>(Val: &I)) {
1355 Scalar = Builder.CreateOr(LHS: ScalarOps[0], RHS: ScalarOps[1],
1356 Name: BO->getName() + ".scalar", IsDisjoint: PDI->isDisjoint());
1357 } else if (PossiblyExactOperator *PEO =
1358 dyn_cast<PossiblyExactOperator>(Val: &I)) {
1359 Scalar =
1360 Builder.CreateExactBinOp(Opc: BO->getOpcode(), LHS: ScalarOps[0], RHS: ScalarOps[1],
1361 IsExact: PEO->isExact(), Name: BO->getName() + ".scalar");
1362 } else if (FPMathOperator *FPMO = dyn_cast<FPMathOperator>(Val: &I)) {
1363 Scalar = Builder.CreateBinOpFMF(Opc: BO->getOpcode(), LHS: ScalarOps[0],
1364 RHS: ScalarOps[1], FMFSource: FPMO->getFastMathFlags(),
1365 Name: BO->getName() + ".scalar");
1366 } else {
1367 Scalar = Builder.CreateBinOp(Opc: BO->getOpcode(), LHS: ScalarOps[0], RHS: ScalarOps[1],
1368 Name: BO->getName() + ".scalar");
1369 }
1370 } else {
1371 FastMathFlags FMF;
1372 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &I))
1373 FMF = FPMO->getFastMathFlags();
1374 Scalar = Builder.CreateIntrinsic(RetTy: ScalarTy, ID: II->getIntrinsicID(), Args: ScalarOps,
1375 FMFSource: FMF, Name: II->getName() + ".scalar");
1376 }
1377
1378 Value *Insert = Builder.CreateInsertElement(Vec: NewVecC, NewElt: Scalar, Idx: *Index);
1379 replaceValue(Old&: I, New&: *Insert);
1380 return true;
1381}
1382
1383/// Try to combine a scalar binop + 2 scalar compares of extracted elements of
1384/// a vector into vector operations followed by extract. Note: The SLP pass
1385/// may miss this pattern because of implementation problems.
1386bool VectorCombine::foldExtractedCmps(Instruction &I) {
1387 auto *BI = dyn_cast<BinaryOperator>(Val: &I);
1388
1389 // We are looking for a scalar binop of booleans.
1390 // binop i1 (cmp Pred I0, C0), (cmp Pred I1, C1)
1391 if (!BI || !I.getType()->isIntegerTy(BitWidth: 1))
1392 return false;
1393
1394 // The compare predicates should match, and each compare should have a
1395 // constant operand.
1396 Value *B0 = I.getOperand(i: 0), *B1 = I.getOperand(i: 1);
1397 Instruction *I0, *I1;
1398 Constant *C0, *C1;
1399 CmpPredicate P0, P1;
1400 if (!match(V: B0, P: m_Cmp(Pred&: P0, L: m_Instruction(I&: I0), R: m_Constant(C&: C0))) ||
1401 !match(V: B1, P: m_Cmp(Pred&: P1, L: m_Instruction(I&: I1), R: m_Constant(C&: C1))))
1402 return false;
1403
1404 auto MatchingPred = CmpPredicate::getMatching(A: P0, B: P1);
1405 if (!MatchingPred)
1406 return false;
1407
1408 // The compare operands must be extracts of the same vector with constant
1409 // extract indexes.
1410 Value *X;
1411 uint64_t Index0, Index1;
1412 if (!match(V: I0, P: m_ExtractElt(Val: m_Value(V&: X), Idx: m_ConstantInt(V&: Index0))) ||
1413 !match(V: I1, P: m_ExtractElt(Val: m_Specific(V: X), Idx: m_ConstantInt(V&: Index1))))
1414 return false;
1415
1416 auto *Ext0 = cast<ExtractElementInst>(Val: I0);
1417 auto *Ext1 = cast<ExtractElementInst>(Val: I1);
1418 ExtractElementInst *ConvertToShuf = getShuffleExtract(Ext0, Ext1, PreferredExtractIndex: CostKind);
1419 if (!ConvertToShuf)
1420 return false;
1421 assert((ConvertToShuf == Ext0 || ConvertToShuf == Ext1) &&
1422 "Unknown ExtractElementInst");
1423
1424 // The original scalar pattern is:
1425 // binop i1 (cmp Pred (ext X, Index0), C0), (cmp Pred (ext X, Index1), C1)
1426 CmpInst::Predicate Pred = *MatchingPred;
1427 unsigned CmpOpcode =
1428 CmpInst::isFPPredicate(P: Pred) ? Instruction::FCmp : Instruction::ICmp;
1429 auto *VecTy = dyn_cast<FixedVectorType>(Val: X->getType());
1430 if (!VecTy)
1431 return false;
1432
1433 if (Index0 >= VecTy->getNumElements() || Index1 >= VecTy->getNumElements())
1434 return false;
1435
1436 InstructionCost Ext0Cost =
1437 TTI.getVectorInstrCost(I: *Ext0, Val: VecTy, CostKind, Index: Index0);
1438 InstructionCost Ext1Cost =
1439 TTI.getVectorInstrCost(I: *Ext1, Val: VecTy, CostKind, Index: Index1);
1440 InstructionCost CmpCost = TTI.getCmpSelInstrCost(
1441 Opcode: CmpOpcode, ValTy: I0->getType(), CondTy: CmpInst::makeCmpResultType(opnd_type: I0->getType()), VecPred: Pred,
1442 CostKind);
1443
1444 InstructionCost OldCost =
1445 Ext0Cost + Ext1Cost + CmpCost * 2 +
1446 TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: I.getType(), CostKind);
1447
1448 // The proposed vector pattern is:
1449 // vcmp = cmp Pred X, VecC
1450 // ext (binop vNi1 vcmp, (shuffle vcmp, Index1)), Index0
1451 int CheapIndex = ConvertToShuf == Ext0 ? Index1 : Index0;
1452 int ExpensiveIndex = ConvertToShuf == Ext0 ? Index0 : Index1;
1453 auto *CmpTy = cast<FixedVectorType>(Val: CmpInst::makeCmpResultType(opnd_type: VecTy));
1454 InstructionCost NewCost = TTI.getCmpSelInstrCost(
1455 Opcode: CmpOpcode, ValTy: VecTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VecTy), VecPred: Pred, CostKind);
1456 SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
1457 ShufMask[CheapIndex] = ExpensiveIndex;
1458 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc, DstTy: CmpTy,
1459 SrcTy: CmpTy, CostKind, Mask: ShufMask);
1460 NewCost += TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: CmpTy, CostKind);
1461 NewCost += TTI.getVectorInstrCost(I: *Ext0, Val: CmpTy, CostKind, Index: CheapIndex);
1462 NewCost += Ext0->hasOneUse() ? 0 : Ext0Cost;
1463 NewCost += Ext1->hasOneUse() ? 0 : Ext1Cost;
1464
1465 // Aggressively form vector ops if the cost is equal because the transform
1466 // may enable further optimization.
1467 // Codegen can reverse this transform (scalarize) if it was not profitable.
1468 if (OldCost < NewCost || !NewCost.isValid())
1469 return false;
1470
1471 // Create a vector constant from the 2 scalar constants.
1472 SmallVector<Constant *, 32> CmpC(VecTy->getNumElements(),
1473 PoisonValue::get(T: VecTy->getElementType()));
1474 CmpC[Index0] = C0;
1475 CmpC[Index1] = C1;
1476 Value *VCmp = Builder.CreateCmp(Pred, LHS: X, RHS: ConstantVector::get(V: CmpC));
1477 Value *Shuf = createShiftShuffle(Vec: VCmp, OldIndex: ExpensiveIndex, NewIndex: CheapIndex, Builder);
1478 Value *LHS = ConvertToShuf == Ext0 ? Shuf : VCmp;
1479 Value *RHS = ConvertToShuf == Ext0 ? VCmp : Shuf;
1480 Value *VecLogic = Builder.CreateBinOp(Opc: BI->getOpcode(), LHS, RHS);
1481 Value *NewExt = Builder.CreateExtractElement(Vec: VecLogic, Idx: CheapIndex);
1482 replaceValue(Old&: I, New&: *NewExt);
1483 ++NumVecCmpBO;
1484 return true;
1485}
1486
1487/// Try to fold scalar selects that select between extracted elements and zero
1488/// into extracting from a vector select. This is rooted at the bitcast.
1489///
1490/// This pattern arises when a vector is bitcast to a smaller element type,
1491/// elements are extracted, and then conditionally selected with zero:
1492///
1493/// %bc = bitcast <4 x i32> %src to <16 x i8>
1494/// %e0 = extractelement <16 x i8> %bc, i32 0
1495/// %s0 = select i1 %cond, i8 %e0, i8 0
1496/// %e1 = extractelement <16 x i8> %bc, i32 1
1497/// %s1 = select i1 %cond, i8 %e1, i8 0
1498/// ...
1499///
1500/// Transforms to:
1501/// %sel = select i1 %cond, <4 x i32> %src, <4 x i32> zeroinitializer
1502/// %bc = bitcast <4 x i32> %sel to <16 x i8>
1503/// %e0 = extractelement <16 x i8> %bc, i32 0
1504/// %e1 = extractelement <16 x i8> %bc, i32 1
1505/// ...
1506///
1507/// This is profitable because vector select on wider types produces fewer
1508/// select/cndmask instructions than scalar selects on each element.
1509bool VectorCombine::foldSelectsFromBitcast(Instruction &I) {
1510 auto *BC = dyn_cast<BitCastInst>(Val: &I);
1511 if (!BC)
1512 return false;
1513
1514 FixedVectorType *SrcVecTy = dyn_cast<FixedVectorType>(Val: BC->getSrcTy());
1515 FixedVectorType *DstVecTy = dyn_cast<FixedVectorType>(Val: BC->getDestTy());
1516 if (!SrcVecTy || !DstVecTy)
1517 return false;
1518
1519 // Source must be 32-bit or 64-bit elements, destination must be smaller
1520 // integer elements. Zero in all these types is all-bits-zero.
1521 Type *SrcEltTy = SrcVecTy->getElementType();
1522 Type *DstEltTy = DstVecTy->getElementType();
1523 unsigned SrcEltBits = SrcEltTy->getPrimitiveSizeInBits();
1524 unsigned DstEltBits = DstEltTy->getPrimitiveSizeInBits();
1525
1526 if (SrcEltBits != 32 && SrcEltBits != 64)
1527 return false;
1528
1529 if (!DstEltTy->isIntegerTy() || DstEltBits >= SrcEltBits)
1530 return false;
1531
1532 // Check profitability using TTI before collecting users.
1533 Type *CondTy = CmpInst::makeCmpResultType(opnd_type: DstEltTy);
1534 Type *VecCondTy = CmpInst::makeCmpResultType(opnd_type: SrcVecTy);
1535
1536 InstructionCost ScalarSelCost =
1537 TTI.getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: DstEltTy, CondTy,
1538 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1539 InstructionCost VecSelCost =
1540 TTI.getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: SrcVecTy, CondTy: VecCondTy,
1541 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1542
1543 // We need at least this many selects for vectorization to be profitable.
1544 // VecSelCost < ScalarSelCost * NumSelects => NumSelects > VecSelCost /
1545 // ScalarSelCost
1546 if (!ScalarSelCost.isValid() || ScalarSelCost == 0)
1547 return false;
1548
1549 unsigned MinSelects = (VecSelCost.getValue() / ScalarSelCost.getValue()) + 1;
1550
1551 // Quick check: if bitcast doesn't have enough users, bail early.
1552 if (!BC->hasNUsesOrMore(N: MinSelects))
1553 return false;
1554
1555 // Collect all select users that match the pattern, grouped by condition.
1556 // Pattern: select i1 %cond, (extractelement %bc, idx), 0
1557 DenseMap<Value *, SmallVector<SelectInst *, 8>> CondToSelects;
1558
1559 for (User *U : BC->users()) {
1560 auto *Ext = dyn_cast<ExtractElementInst>(Val: U);
1561 if (!Ext)
1562 continue;
1563
1564 for (User *ExtUser : Ext->users()) {
1565 Value *Cond;
1566 // Match: select i1 %cond, %ext, 0
1567 if (match(V: ExtUser, P: m_Select(C: m_Value(V&: Cond), L: m_Specific(V: Ext), R: m_Zero())) &&
1568 Cond->getType()->isIntegerTy(BitWidth: 1))
1569 CondToSelects[Cond].push_back(Elt: cast<SelectInst>(Val: ExtUser));
1570 }
1571 }
1572
1573 if (CondToSelects.empty())
1574 return false;
1575
1576 bool MadeChange = false;
1577 Value *SrcVec = BC->getOperand(i_nocapture: 0);
1578
1579 // Process each group of selects with the same condition.
1580 for (auto [Cond, Selects] : CondToSelects) {
1581 // Only profitable if vector select cost < total scalar select cost.
1582 if (Selects.size() < MinSelects) {
1583 LLVM_DEBUG(dbgs() << "VectorCombine: foldSelectsFromBitcast not "
1584 << "profitable (VecCost=" << VecSelCost
1585 << ", ScalarCost=" << ScalarSelCost
1586 << ", NumSelects=" << Selects.size() << ")\n");
1587 continue;
1588 }
1589
1590 // Create the vector select and bitcast once for this condition.
1591 auto InsertPt = std::next(x: BC->getIterator());
1592
1593 if (auto *CondInst = dyn_cast<Instruction>(Val: Cond))
1594 if (DT.dominates(Def: BC, User: CondInst))
1595 InsertPt = std::next(x: CondInst->getIterator());
1596
1597 Builder.SetInsertPoint(InsertPt);
1598 Value *VecSel =
1599 Builder.CreateSelect(C: Cond, True: SrcVec, False: Constant::getNullValue(Ty: SrcVecTy));
1600 Value *NewBC = Builder.CreateBitCast(V: VecSel, DestTy: DstVecTy);
1601
1602 // Replace each scalar select with an extract from the new bitcast.
1603 for (SelectInst *Sel : Selects) {
1604 auto *Ext = cast<ExtractElementInst>(Val: Sel->getTrueValue());
1605 Value *Idx = Ext->getIndexOperand();
1606
1607 Builder.SetInsertPoint(Sel);
1608 Value *NewExt = Builder.CreateExtractElement(Vec: NewBC, Idx);
1609 replaceValue(Old&: *Sel, New&: *NewExt);
1610 MadeChange = true;
1611 }
1612
1613 LLVM_DEBUG(dbgs() << "VectorCombine: folded " << Selects.size()
1614 << " selects into vector select\n");
1615 }
1616
1617 return MadeChange;
1618}
1619
1620static void analyzeCostOfVecReduction(const IntrinsicInst &II,
1621 TTI::TargetCostKind CostKind,
1622 const TargetTransformInfo &TTI,
1623 InstructionCost &CostBeforeReduction,
1624 InstructionCost &CostAfterReduction) {
1625 Instruction *Op0, *Op1;
1626 auto *RedOp = dyn_cast<Instruction>(Val: II.getOperand(i_nocapture: 0));
1627 auto *VecRedTy = cast<VectorType>(Val: II.getOperand(i_nocapture: 0)->getType());
1628 unsigned ReductionOpc =
1629 getArithmeticReductionInstruction(RdxID: II.getIntrinsicID());
1630 if (RedOp && match(V: RedOp, P: m_ZExtOrSExt(Op: m_Value()))) {
1631 bool IsUnsigned = isa<ZExtInst>(Val: RedOp);
1632 auto *ExtType = cast<VectorType>(Val: RedOp->getOperand(i: 0)->getType());
1633
1634 CostBeforeReduction =
1635 TTI.getCastInstrCost(Opcode: RedOp->getOpcode(), Dst: VecRedTy, Src: ExtType,
1636 CCH: TTI::CastContextHint::None, CostKind, I: RedOp);
1637 CostAfterReduction =
1638 TTI.getExtendedReductionCost(Opcode: ReductionOpc, IsUnsigned, ResTy: II.getType(),
1639 Ty: ExtType, FMF: FastMathFlags(), CostKind);
1640 return;
1641 }
1642 if (RedOp && II.getIntrinsicID() == Intrinsic::vector_reduce_add &&
1643 match(V: RedOp,
1644 P: m_ZExtOrSExt(Op: m_Mul(L: m_Instruction(I&: Op0), R: m_Instruction(I&: Op1)))) &&
1645 match(V: Op0, P: m_ZExtOrSExt(Op: m_Value())) &&
1646 Op0->getOpcode() == Op1->getOpcode() &&
1647 Op0->getOperand(i: 0)->getType() == Op1->getOperand(i: 0)->getType() &&
1648 (Op0->getOpcode() == RedOp->getOpcode() || Op0 == Op1)) {
1649 // Matched reduce.add(ext(mul(ext(A), ext(B)))
1650 bool IsUnsigned = isa<ZExtInst>(Val: Op0);
1651 auto *ExtType = cast<VectorType>(Val: Op0->getOperand(i: 0)->getType());
1652 VectorType *MulType = VectorType::get(ElementType: Op0->getType(), Other: VecRedTy);
1653
1654 InstructionCost ExtCost =
1655 TTI.getCastInstrCost(Opcode: Op0->getOpcode(), Dst: MulType, Src: ExtType,
1656 CCH: TTI::CastContextHint::None, CostKind, I: Op0);
1657 InstructionCost MulCost =
1658 TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: MulType, CostKind);
1659 InstructionCost Ext2Cost =
1660 TTI.getCastInstrCost(Opcode: RedOp->getOpcode(), Dst: VecRedTy, Src: MulType,
1661 CCH: TTI::CastContextHint::None, CostKind, I: RedOp);
1662
1663 CostBeforeReduction = ExtCost * 2 + MulCost + Ext2Cost;
1664 CostAfterReduction = TTI.getMulAccReductionCost(
1665 IsUnsigned, RedOpcode: ReductionOpc, ResTy: II.getType(), Ty: ExtType, CostKind);
1666 return;
1667 }
1668 CostAfterReduction = TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: VecRedTy,
1669 FMF: std::nullopt, CostKind);
1670}
1671
1672bool VectorCombine::foldBinopOfReductions(Instruction &I) {
1673 Instruction::BinaryOps BinOpOpc = cast<BinaryOperator>(Val: &I)->getOpcode();
1674 Intrinsic::ID ReductionIID = getReductionForBinop(Opc: BinOpOpc);
1675 if (BinOpOpc == Instruction::Sub)
1676 ReductionIID = Intrinsic::vector_reduce_add;
1677 if (ReductionIID == Intrinsic::not_intrinsic)
1678 return false;
1679 // FP reductions have a start-value operand that this fold doesn't handle.
1680 if (ReductionIID == Intrinsic::vector_reduce_fadd ||
1681 ReductionIID == Intrinsic::vector_reduce_fmul)
1682 return false;
1683
1684 auto checkIntrinsicAndGetItsArgument = [](Value *V,
1685 Intrinsic::ID IID) -> Value * {
1686 auto *II = dyn_cast<IntrinsicInst>(Val: V);
1687 if (!II)
1688 return nullptr;
1689 if (II->getIntrinsicID() == IID && II->hasOneUse())
1690 return II->getArgOperand(i: 0);
1691 return nullptr;
1692 };
1693
1694 Value *V0 = checkIntrinsicAndGetItsArgument(I.getOperand(i: 0), ReductionIID);
1695 if (!V0)
1696 return false;
1697 Value *V1 = checkIntrinsicAndGetItsArgument(I.getOperand(i: 1), ReductionIID);
1698 if (!V1)
1699 return false;
1700
1701 auto *VTy = cast<VectorType>(Val: V0->getType());
1702 if (V1->getType() != VTy)
1703 return false;
1704 const auto &II0 = *cast<IntrinsicInst>(Val: I.getOperand(i: 0));
1705 const auto &II1 = *cast<IntrinsicInst>(Val: I.getOperand(i: 1));
1706 unsigned ReductionOpc =
1707 getArithmeticReductionInstruction(RdxID: II0.getIntrinsicID());
1708
1709 InstructionCost OldCost = 0;
1710 InstructionCost NewCost = 0;
1711 InstructionCost CostOfRedOperand0 = 0;
1712 InstructionCost CostOfRed0 = 0;
1713 InstructionCost CostOfRedOperand1 = 0;
1714 InstructionCost CostOfRed1 = 0;
1715 analyzeCostOfVecReduction(II: II0, CostKind, TTI, CostBeforeReduction&: CostOfRedOperand0, CostAfterReduction&: CostOfRed0);
1716 analyzeCostOfVecReduction(II: II1, CostKind, TTI, CostBeforeReduction&: CostOfRedOperand1, CostAfterReduction&: CostOfRed1);
1717 OldCost = CostOfRed0 + CostOfRed1 + TTI.getInstructionCost(U: &I, CostKind);
1718 NewCost =
1719 CostOfRedOperand0 + CostOfRedOperand1 +
1720 TTI.getArithmeticInstrCost(Opcode: BinOpOpc, Ty: VTy, CostKind) +
1721 TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: VTy, FMF: std::nullopt, CostKind);
1722 if (NewCost >= OldCost || !NewCost.isValid())
1723 return false;
1724
1725 LLVM_DEBUG(dbgs() << "Found two mergeable reductions: " << I
1726 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
1727 << "\n");
1728 Value *VectorBO;
1729 if (BinOpOpc == Instruction::Or)
1730 VectorBO = Builder.CreateOr(LHS: V0, RHS: V1, Name: "",
1731 IsDisjoint: cast<PossiblyDisjointInst>(Val&: I).isDisjoint());
1732 else
1733 VectorBO = Builder.CreateBinOp(Opc: BinOpOpc, LHS: V0, RHS: V1);
1734
1735 Value *Rdx = Builder.CreateIntrinsic(ID: ReductionIID, OverloadTypes: {VTy}, Args: {VectorBO});
1736 replaceValue(Old&: I, New&: *Rdx);
1737 return true;
1738}
1739
1740// Check if memory is modified, freed, or synchronized between two instrs in
1741// the same BB.
1742static bool isMemModifiedBetween(const VectorizeOptions &Opts,
1743 BasicBlock::iterator Begin,
1744 BasicBlock::iterator End,
1745 const MemoryLocation &Loc, AAResults &AA) {
1746 unsigned NumScanned = 0;
1747 if (std::any_of(first: Begin, last: End, pred: [&](const Instruction &Instr) {
1748 return isModSet(MRI: AA.getModRefInfo(I: &Instr, OptLoc: Loc)) ||
1749 ++NumScanned > Opts.vector_combine_max_scan_instrs;
1750 }))
1751 return true;
1752
1753 // willNotFreeBetween expects instructions rather than iterators. An empty
1754 // range cannot free or synchronize, so avoid dereferencing its end.
1755 return Begin != End && !willNotFreeBetween(Assume: &*Begin, CtxI: &*End);
1756}
1757
1758namespace {
1759/// Helper class to indicate whether a vector index can be safely scalarized and
1760/// if a freeze needs to be inserted.
1761class ScalarizationResult {
1762 enum class StatusTy { Unsafe, Safe, SafeWithFreeze };
1763
1764 StatusTy Status;
1765 Value *ToFreeze;
1766
1767 ScalarizationResult(StatusTy Status, Value *ToFreeze = nullptr)
1768 : Status(Status), ToFreeze(ToFreeze) {}
1769
1770public:
1771 ScalarizationResult(const ScalarizationResult &Other) = default;
1772 ~ScalarizationResult() {
1773 assert(!ToFreeze && "freeze() not called with ToFreeze being set");
1774 }
1775
1776 static ScalarizationResult unsafe() { return {StatusTy::Unsafe}; }
1777 static ScalarizationResult safe() { return {StatusTy::Safe}; }
1778 static ScalarizationResult safeWithFreeze(Value *ToFreeze) {
1779 return {StatusTy::SafeWithFreeze, ToFreeze};
1780 }
1781
1782 /// Returns true if the index can be scalarize without requiring a freeze.
1783 bool isSafe() const { return Status == StatusTy::Safe; }
1784 /// Returns true if the index cannot be scalarized.
1785 bool isUnsafe() const { return Status == StatusTy::Unsafe; }
1786 /// Returns true if the index can be scalarize, but requires inserting a
1787 /// freeze.
1788 bool isSafeWithFreeze() const { return Status == StatusTy::SafeWithFreeze; }
1789
1790 /// Reset the state of Unsafe and clear ToFreze if set.
1791 void discard() {
1792 ToFreeze = nullptr;
1793 Status = StatusTy::Unsafe;
1794 }
1795
1796 /// Freeze the ToFreeze and update the use in \p User to use it.
1797 void freeze(IRBuilderBase &Builder, Instruction &UserI) {
1798 assert(isSafeWithFreeze() &&
1799 "should only be used when freezing is required");
1800 assert(is_contained(ToFreeze->users(), &UserI) &&
1801 "UserI must be a user of ToFreeze");
1802 IRBuilder<>::InsertPointGuard Guard(Builder);
1803 Builder.SetInsertPoint(cast<Instruction>(Val: &UserI));
1804 Value *Frozen =
1805 Builder.CreateFreeze(V: ToFreeze, Name: ToFreeze->getName() + ".frozen");
1806 for (Use &U : make_early_inc_range(Range: (UserI.operands())))
1807 if (U.get() == ToFreeze)
1808 U.set(Frozen);
1809
1810 ToFreeze = nullptr;
1811 }
1812};
1813} // namespace
1814
1815/// Check if it is legal to scalarize a memory access to \p VecTy at index \p
1816/// Idx. \p Idx must access a valid vector element.
1817static ScalarizationResult canScalarizeAccess(VectorType *VecTy, Value *Idx,
1818 const SimplifyQuery &SQ) {
1819 // We do checks for both fixed vector types and scalable vector types.
1820 // This is the number of elements of fixed vector types,
1821 // or the minimum number of elements of scalable vector types.
1822 uint64_t NumElements = VecTy->getElementCount().getKnownMinValue();
1823 unsigned IntWidth = Idx->getType()->getScalarSizeInBits();
1824
1825 if (auto *C = dyn_cast<ConstantInt>(Val: Idx)) {
1826 if (C->getValue().ult(RHS: NumElements))
1827 return ScalarizationResult::safe();
1828 return ScalarizationResult::unsafe();
1829 }
1830
1831 // Always unsafe if the index type can't handle all inbound values.
1832 if (!llvm::isUIntN(N: IntWidth, x: NumElements))
1833 return ScalarizationResult::unsafe();
1834
1835 APInt Zero(IntWidth, 0);
1836 APInt MaxElts(IntWidth, NumElements);
1837 ConstantRange ValidIndices(Zero, MaxElts);
1838 ConstantRange IdxRange(IntWidth, true);
1839
1840 if (isGuaranteedNotToBePoison(V: Idx, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)) {
1841 if (ValidIndices.contains(
1842 CR: computeConstantRange(V: Idx, /*ForSigned=*/false, SQ)))
1843 return ScalarizationResult::safe();
1844 return ScalarizationResult::unsafe();
1845 }
1846
1847 // If the index may be poison, check if we can insert a freeze before the
1848 // range of the index is restricted.
1849 Value *IdxBase;
1850 ConstantInt *CI;
1851 if (match(V: Idx, P: m_And(L: m_Value(V&: IdxBase), R: m_ConstantInt(CI)))) {
1852 IdxRange = IdxRange.binaryAnd(Other: CI->getValue());
1853 } else if (match(V: Idx, P: m_URem(L: m_Value(V&: IdxBase), R: m_ConstantInt(CI)))) {
1854 IdxRange = IdxRange.urem(Other: CI->getValue());
1855 }
1856
1857 if (ValidIndices.contains(CR: IdxRange))
1858 return ScalarizationResult::safeWithFreeze(ToFreeze: IdxBase);
1859 return ScalarizationResult::unsafe();
1860}
1861
1862/// Return the GEP index type if the unsigned vector index \p Idx can be
1863/// represented by an inbounds GEP. A null result means that the maximum byte
1864/// offset cannot be represented by the pointer's signed GEP index type.
1865///
1866/// unsigned lane range
1867/// |
1868/// v
1869/// MaxByteOffset = MaxLane * element store size
1870/// |
1871/// +-- unavailable or outside signed GEP range --> reject
1872/// |
1873/// v
1874/// valid range --> use the pointer's GEP index type
1875static IntegerType *getScalarizedGEPIndexInfo(VectorType *VecTy, Value *Idx,
1876 Type *PtrTy,
1877 const DataLayout &DL) {
1878 auto *GEPIndexTy = cast<IntegerType>(Val: DL.getIndexType(PtrTy));
1879 unsigned GEPBits = GEPIndexTy->getBitWidth();
1880 uint64_t NumElements = VecTy->getElementCount().getKnownMinValue();
1881
1882 uint64_t MaxLane = NumElements - 1;
1883 if (auto *C = dyn_cast<ConstantInt>(Val: Idx)) {
1884 if (C->getValue().uge(RHS: NumElements))
1885 return nullptr;
1886 MaxLane = C->getZExtValue();
1887 }
1888
1889 Type *ElemTy = VecTy->getElementType();
1890 if (!DL.typeSizeEqualsStoreSize(Ty: ElemTy))
1891 return nullptr;
1892
1893 TypeSize ElemStride = DL.getTypeStoreSize(Ty: ElemTy);
1894 if (ElemStride.isScalable())
1895 return nullptr;
1896
1897 // Compare both values in a common width:
1898 //
1899 // MaxLane (uint64_t) * ElemStride (uint64_t) signed_max(GEPBits)
1900 // | |
1901 // v v
1902 // ByteOffset (up to 128 bits) sext to WideBits
1903 // \ /
1904 // +------------ ugt ------------+
1905 // |
1906 // greater -> reject
1907 //
1908 // WideBits = max(GEPBits, 128) prevents the multiplication from wrapping
1909 // and preserves the GEP limit during the comparison.
1910 unsigned WideBits = std::max(a: GEPBits, b: 128u);
1911 APInt MaxLaneValue(WideBits, MaxLane);
1912 APInt ByteOffset = MaxLaneValue;
1913 ByteOffset *= APInt(WideBits, ElemStride.getFixedValue());
1914 APInt MaxGEPOffset = APInt::getSignedMaxValue(numBits: GEPBits).sext(width: WideBits);
1915 // Reject offsets outside the GEP's positive signed range. Compare as
1916 // unsigned because the full 128-bit product may set its sign bit.
1917 if (ByteOffset.ugt(RHS: MaxGEPOffset))
1918 return nullptr;
1919
1920 return GEPIndexTy;
1921}
1922
1923/// Materialize an index for a scalarized GEP after profitability is known.
1924/// Vector element indices are unsigned, but GEP sign-extends narrow integer
1925/// indices. Widen a narrow index explicitly so its unsigned value is retained.
1926static Value *materializeScalarizedGEPIndex(Value *Idx, IntegerType *GEPIndexTy,
1927 IRBuilderBase &Builder) {
1928 unsigned SrcBits = Idx->getType()->getIntegerBitWidth();
1929 unsigned DstBits = GEPIndexTy->getBitWidth();
1930 if (SrcBits >= DstBits)
1931 return Idx;
1932
1933 return Builder.CreateZExt(V: Idx, DestTy: GEPIndexTy, Name: Idx->getName() + ".gepidx");
1934}
1935
1936/// The memory operation on a vector of \p ScalarType had alignment of
1937/// \p VectorAlignment. Compute the maximal, but conservatively correct,
1938/// alignment that will be valid for the memory operation on a single scalar
1939/// element of the same type with index \p Idx.
1940static Align computeAlignmentAfterScalarization(Align VectorAlignment,
1941 Type *ScalarType, Value *Idx,
1942 const DataLayout &DL) {
1943 if (auto *C = dyn_cast<ConstantInt>(Val: Idx))
1944 return commonAlignment(A: VectorAlignment,
1945 Offset: C->getZExtValue() * DL.getTypeStoreSize(Ty: ScalarType));
1946 return commonAlignment(A: VectorAlignment, Offset: DL.getTypeStoreSize(Ty: ScalarType));
1947}
1948
1949/// Fold a vector store fed by a single-use insertelement chain into scalar
1950/// stores.
1951///
1952/// Before:
1953///
1954/// %p --> vector load --> insert %x, lane 1 --> insert %y, lane 3
1955/// |
1956/// v
1957/// vector store to %p
1958///
1959/// Vector lanes: [ 0 ] [ 1 ] [ 2 ] [ 3 ]
1960/// Stored value: [ old | x | old | y ] (one vector store)
1961///
1962/// After:
1963///
1964/// +--> GEP(%p, lane 1) --> store %x
1965/// %p -------------+
1966/// +--> GEP(%p, lane 3) --> store %y
1967///
1968/// Vector lanes: [ 0 ] [ 1 ] [ 2 ] [ 3 ]
1969/// Scalar stores: x y
1970/// store@1 store@3
1971///
1972/// Step 1. Gate:
1973/// target supports vector-element GEP addressing
1974///
1975/// Step 2. Trace:
1976/// vector store <-- insertelement <-- ... <-- insertelement <-- load
1977///
1978/// Steps 3-5. Validate:
1979/// reject unprofitable full overwrites; require simple accesses, a
1980/// common address/block, no memory write in between, and scalarizable
1981/// indices.
1982bool VectorCombine::foldInsertElementsToStores(Instruction &I) {
1983 // Step 1: The target must support addressing a vector element with a GEP.
1984 if (!TTI.allowVectorElementIndexingUsingGEP())
1985 return false;
1986
1987 auto *SI = cast<StoreInst>(Val: &I);
1988 if (!SI->isSimple() || !isa<VectorType>(Val: SI->getValueOperand()->getType()))
1989 return false;
1990
1991 // Step 2: Collect a single-use insertelement chain, starting at the vector
1992 // store and walking back to the candidate load.
1993 Value *Source = SI->getValueOperand();
1994 SmallVector<std::pair<Value *, Value *>, 4> InsertElements;
1995 Value *Base = Source;
1996 while (auto *Insert = dyn_cast<InsertElementInst>(Val: Base)) {
1997 if (!Insert->hasOneUse())
1998 break;
1999 Value *InsertVal = Insert->getOperand(i_nocapture: 1);
2000 Value *Idx = Insert->getOperand(i_nocapture: 2);
2001 InsertElements.push_back(Elt: {InsertVal, Idx});
2002 Base = Insert->getOperand(i_nocapture: 0);
2003 }
2004
2005 if (InsertElements.empty())
2006 return false;
2007
2008 // The backwards walk collected the inserts in reverse program order. Restore
2009 // it now so later scalar stores preserve writes to duplicate/equal indices.
2010 std::reverse(first: InsertElements.begin(), last: InsertElements.end());
2011 auto *Load = dyn_cast<LoadInst>(Val: Base);
2012 if (!Load)
2013 return false;
2014 auto *VecTy = cast<VectorType>(Val: SI->getValueOperand()->getType());
2015
2016 // Step 3: Avoid replacing a complete overwrite with scalar stores when every
2017 // lane receives the same value; keeping the vector operation is preferable.
2018 if (auto *FVT = dyn_cast<FixedVectorType>(Val: VecTy)) {
2019 if (InsertElements.size() == FVT->getNumElements()) {
2020 Value *FirstVal = InsertElements.front().first;
2021 if (all_of(Range&: InsertElements,
2022 P: [FirstVal](const auto &Elt) { return Elt.first == FirstVal; }))
2023 return false;
2024 }
2025 }
2026 Value *SrcAddr = Load->getPointerOperand()->stripPointerCasts();
2027 // Step 4: Establish the load/store update is legal: both accesses are simple,
2028 // have the same base address and block, have scalar elements whose type size
2029 // equals their store size, and no intervening operation modifies the updated
2030 // memory.
2031 if (!Load->isSimple() || Load->getParent() != SI->getParent() ||
2032 !DL->typeSizeEqualsStoreSize(Ty: Load->getType()->getScalarType()) ||
2033 SrcAddr != SI->getPointerOperand()->stripPointerCasts())
2034 return false;
2035
2036 if (isMemModifiedBetween(Opts, Begin: Load->getIterator(), End: SI->getIterator(),
2037 Loc: MemoryLocation::get(SI), AA))
2038 return false;
2039
2040 // Step 5: Validate every index before changing IR. A safe-with-freeze result
2041 // is recorded by ScalarizationResult, so discard it until profitability is
2042 // known; otherwise a rejected candidate could leave a freeze behind.
2043 for (auto [InsertVal, Idx] : InsertElements) {
2044 auto ScalarizableIdx =
2045 canScalarizeAccess(VecTy, Idx, SQ: SQ.getWithInstruction(I: &I));
2046 if (ScalarizableIdx.isUnsafe())
2047 return false;
2048
2049 auto GEPIndex =
2050 getScalarizedGEPIndexInfo(VecTy, Idx, PtrTy: SI->getPointerOperandType(), DL: *DL);
2051 if (!GEPIndex) {
2052 ScalarizableIdx.discard();
2053 return false;
2054 }
2055
2056 // We are only checking legality here. Do not mutate IR before the
2057 // profitability check, but also do not leave a pending ToFreeze behind.
2058 ScalarizableIdx.discard();
2059 }
2060
2061 InstructionCost OldCost = TTI.getMemoryOpCost(
2062 Opcode: Instruction::Store, Src: SI->getValueOperand()->getType(), Alignment: SI->getAlign(),
2063 AddressSpace: SI->getPointerAddressSpace(), CostKind);
2064
2065 if (Load->hasOneUse())
2066 OldCost += TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: Load->getType(),
2067 Alignment: Load->getAlign(),
2068 AddressSpace: Load->getPointerAddressSpace(), CostKind);
2069
2070 for (auto [InsertVal, Idx] : InsertElements) {
2071 int Index = -1;
2072 if (auto *CIdx = dyn_cast<ConstantInt>(Val: Idx))
2073 Index = CIdx->getZExtValue();
2074
2075 OldCost += TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: VecTy,
2076 CostKind, Index);
2077 }
2078
2079 InstructionCost NewCost = 0;
2080 // This transform replaces insertelement operations on a single vector with
2081 // GEPs and scalar stores, so assume constant-index GEP offsets stay within
2082 // addressing-mode ranges that getGEPCost considers TCC_Free. Cost only GEPs
2083 // with dynamic indices.
2084 for (auto [InsertVal, Idx] : InsertElements) {
2085 if (isa<ConstantInt>(Val: Idx))
2086 continue;
2087 const Value *GEPIndices[] = {ConstantInt::get(Ty: Idx->getType(), V: 0), Idx};
2088 NewCost += TTI.getGEPCost(PointeeType: VecTy, Ptr: SI->getPointerOperand(), Operands: GEPIndices,
2089 CostKind, AccessType: InsertVal->getType());
2090 }
2091
2092 for (auto [InsertVal, Idx] : InsertElements) {
2093 Align ScalarOpAlignment = computeAlignmentAfterScalarization(
2094 VectorAlignment: std::max(a: SI->getAlign(), b: Load->getAlign()), ScalarType: InsertVal->getType(), Idx,
2095 DL: *DL);
2096
2097 NewCost += TTI.getMemoryOpCost(Opcode: Instruction::Store, Src: InsertVal->getType(),
2098 Alignment: ScalarOpAlignment,
2099 AddressSpace: SI->getPointerAddressSpace(), CostKind);
2100 }
2101
2102 LLVM_DEBUG(dbgs() << "Found an insert-elements vector store scalarization "
2103 "candidate: "
2104 << I << "\n"
2105 << " NumInserts: " << InsertElements.size() << "\n"
2106 << " OldCost: " << OldCost << " vs NewCost: " << NewCost
2107 << "\n");
2108
2109 if (OldCost <= NewCost)
2110 return false;
2111
2112 for (auto [InsertVal, Idx] : InsertElements) {
2113 auto ScalarizableIdx =
2114 canScalarizeAccess(VecTy, Idx, SQ: SQ.getWithInstruction(I: &I));
2115 assert(!ScalarizableIdx.isUnsafe() && "already checked above");
2116
2117 if (ScalarizableIdx.isSafeWithFreeze())
2118 ScalarizableIdx.freeze(Builder, UserI&: *cast<Instruction>(Val: Idx));
2119 }
2120
2121 Worklist.push(I: Load);
2122 StoreInst *LastStore = nullptr;
2123 for (auto [InsertVal, Idx] : InsertElements) {
2124 auto ScalarizableIdx =
2125 canScalarizeAccess(VecTy, Idx, SQ: SQ.getWithInstruction(I: &I));
2126 if (ScalarizableIdx.isUnsafe())
2127 return false;
2128
2129 IntegerType *GEPIndexTy =
2130 getScalarizedGEPIndexInfo(VecTy, Idx, PtrTy: SI->getPointerOperandType(), DL: *DL);
2131
2132 Value *GEPIdx = materializeScalarizedGEPIndex(Idx, GEPIndexTy, Builder);
2133 Value *GEP = Builder.CreateInBoundsGEP(
2134 Ty: SI->getValueOperand()->getType(), Ptr: SI->getPointerOperand(),
2135 IdxList: {ConstantInt::get(Ty: GEPIdx->getType(), V: 0), GEPIdx});
2136
2137 LastStore = Builder.CreateStore(Val: InsertVal, Ptr: GEP);
2138 LastStore->copyMetadata(SrcInst: *SI);
2139
2140 // The new GEP may change the pointer operand, so !invariant.group cannot
2141 // be transferred to the scalar store.
2142 LastStore->setMetadata(KindID: LLVMContext::MD_invariant_group, Node: nullptr);
2143 Align ScalarOpAlignment = computeAlignmentAfterScalarization(
2144 VectorAlignment: std::max(a: SI->getAlign(), b: Load->getAlign()), ScalarType: InsertVal->getType(), Idx,
2145 DL: *DL);
2146 LastStore->setAlignment(ScalarOpAlignment);
2147 }
2148
2149 replaceValue(Old&: I, New&: *LastStore);
2150 eraseInstruction(I);
2151 return true;
2152}
2153
2154/// Try to scalarize vector loads feeding extractelement or bitcast
2155/// instructions.
2156bool VectorCombine::scalarizeLoad(Instruction &I) {
2157 Value *Ptr;
2158 if (!match(V: &I, P: m_Load(Op: m_Value(V&: Ptr))))
2159 return false;
2160
2161 auto *LI = cast<LoadInst>(Val: &I);
2162 auto *VecTy = cast<VectorType>(Val: LI->getType());
2163
2164 // The isSimple() check could be isUnordered(), but for now we cowardly
2165 // refuse to handle even unordered atomics.
2166 if (!LI->isSimple() || !DL->typeSizeEqualsStoreSize(Ty: VecTy->getScalarType()))
2167 return false;
2168
2169 bool AllExtracts = true;
2170 bool AllBitcasts = true;
2171 Instruction *LastCheckedInst = LI;
2172 unsigned NumInstChecked = 0;
2173
2174 // Check what type of users we have (must either all be extracts or
2175 // bitcasts) and ensure no memory modifications between the load and
2176 // its users.
2177 for (User *U : LI->users()) {
2178 auto *UI = dyn_cast<Instruction>(Val: U);
2179 if (!UI || UI->getParent() != LI->getParent())
2180 return false;
2181
2182 // If any user is waiting to be erased, then bail out as this will
2183 // distort the cost calculation and possibly lead to infinite loops.
2184 if (UI->use_empty())
2185 return false;
2186
2187 if (!isa<ExtractElementInst>(Val: UI))
2188 AllExtracts = false;
2189 if (!isa<BitCastInst>(Val: UI))
2190 AllBitcasts = false;
2191
2192 // Check if any instruction between the load and the user may modify memory.
2193 if (LastCheckedInst->comesBefore(Other: UI)) {
2194 for (Instruction &I :
2195 make_range(x: std::next(x: LI->getIterator()), y: UI->getIterator())) {
2196 // Bail out if we reached the check limit or the instruction may write
2197 // to memory.
2198 if (NumInstChecked == Opts.vector_combine_max_scan_instrs ||
2199 I.mayWriteToMemory())
2200 return false;
2201 NumInstChecked++;
2202 }
2203 LastCheckedInst = UI;
2204 }
2205 }
2206
2207 if (AllExtracts)
2208 return scalarizeLoadExtract(LI, VecTy, Ptr);
2209 if (AllBitcasts)
2210 return scalarizeLoadBitcast(LI, VecTy, Ptr);
2211 return false;
2212}
2213
2214/// Try to scalarize vector loads feeding extractelement instructions.
2215bool VectorCombine::scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy,
2216 Value *Ptr) {
2217 if (!TTI.allowVectorElementIndexingUsingGEP())
2218 return false;
2219
2220 DenseMap<ExtractElementInst *, ScalarizationResult> NeedFreeze;
2221 DenseMap<ExtractElementInst *, IntegerType *> GEPIndexInfos;
2222 llvm::scope_exit FailureGuard([&]() {
2223 // If the transform is aborted, discard the ScalarizationResults.
2224 for (auto &Pair : NeedFreeze)
2225 Pair.second.discard();
2226 });
2227
2228 InstructionCost OriginalCost =
2229 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: VecTy, Alignment: LI->getAlign(),
2230 AddressSpace: LI->getPointerAddressSpace(), CostKind);
2231 InstructionCost ScalarizedCost = 0;
2232
2233 for (User *U : LI->users()) {
2234 auto *UI = cast<ExtractElementInst>(Val: U);
2235
2236 auto ScalarIdx = canScalarizeAccess(VecTy, Idx: UI->getIndexOperand(),
2237 SQ: SQ.getWithInstruction(I: LI));
2238 if (ScalarIdx.isUnsafe())
2239 return false;
2240
2241 IntegerType *GEPIndex = getScalarizedGEPIndexInfo(
2242 VecTy, Idx: UI->getIndexOperand(), PtrTy: LI->getPointerOperandType(), DL: *DL);
2243 if (!GEPIndex) {
2244 ScalarIdx.discard();
2245 return false;
2246 }
2247
2248 GEPIndexInfos.try_emplace(Key: UI, Args&: GEPIndex);
2249
2250 if (ScalarIdx.isSafeWithFreeze()) {
2251 NeedFreeze.try_emplace(Key: UI, Args&: ScalarIdx);
2252 ScalarIdx.discard();
2253 }
2254
2255 auto *Index = dyn_cast<ConstantInt>(Val: UI->getIndexOperand());
2256 OriginalCost +=
2257 TTI.getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: VecTy, CostKind,
2258 Index: Index ? Index->getZExtValue() : -1);
2259 ScalarizedCost +=
2260 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: VecTy->getElementType(),
2261 Alignment: Align(1), AddressSpace: LI->getPointerAddressSpace(), CostKind);
2262 ScalarizedCost += TTI.getAddressComputationCost(PtrTy: LI->getPointerOperandType(),
2263 SE: nullptr, Ptr: nullptr, CostKind);
2264 if (!Index && UI->getIndexOperand()->getType()->getIntegerBitWidth() <
2265 GEPIndex->getBitWidth())
2266 ScalarizedCost += TTI.getCastInstrCost(
2267 Opcode: Instruction::ZExt, Dst: GEPIndex, Src: UI->getIndexOperand()->getType(),
2268 CCH: TTI::CastContextHint::None, CostKind);
2269 }
2270
2271 LLVM_DEBUG(dbgs() << "Found all extractions of a vector load: " << *LI
2272 << "\n LoadExtractCost: " << OriginalCost
2273 << " vs ScalarizedCost: " << ScalarizedCost << "\n");
2274
2275 if (ScalarizedCost > OriginalCost)
2276 return false;
2277 if (ScalarizedCost == OriginalCost && !LI->hasOneUse())
2278 return false;
2279
2280 // Ensure we add the load back to the worklist BEFORE its users so they can
2281 // erased in the correct order.
2282 Worklist.push(I: LI);
2283
2284 Type *ElemType = VecTy->getElementType();
2285
2286 // Replace extracts with narrow scalar loads.
2287 for (User *U : LI->users()) {
2288 auto *EI = cast<ExtractElementInst>(Val: U);
2289 Value *Idx = EI->getIndexOperand();
2290
2291 // Insert 'freeze' for poison indexes.
2292 if (auto It = NeedFreeze.find(Val: EI); It != NeedFreeze.end())
2293 It->second.freeze(Builder, UserI&: *cast<Instruction>(Val: Idx));
2294
2295 Builder.SetInsertPoint(EI);
2296 auto It = GEPIndexInfos.find(Val: EI);
2297 assert(It != GEPIndexInfos.end() &&
2298 "Missing scalarized GEP index information");
2299 Value *GEPIdx = materializeScalarizedGEPIndex(Idx, GEPIndexTy: It->second, Builder);
2300 Value *GEP = Builder.CreateInBoundsGEP(
2301 Ty: VecTy, Ptr, IdxList: {ConstantInt::get(Ty: GEPIdx->getType(), V: 0), GEPIdx});
2302 auto *NewLoad = cast<LoadInst>(
2303 Val: Builder.CreateLoad(Ty: ElemType, Ptr: GEP, Name: EI->getName() + ".scalar"));
2304
2305 Align ScalarOpAlignment =
2306 computeAlignmentAfterScalarization(VectorAlignment: LI->getAlign(), ScalarType: ElemType, Idx, DL: *DL);
2307 NewLoad->setAlignment(ScalarOpAlignment);
2308
2309 if (auto *ConstIdx = dyn_cast<ConstantInt>(Val: Idx)) {
2310 size_t Offset = ConstIdx->getZExtValue() * DL->getTypeStoreSize(Ty: ElemType);
2311 AAMDNodes OldAAMD = LI->getAAMetadata();
2312 NewLoad->setAAMetadata(OldAAMD.adjustForAccess(Offset, AccessTy: ElemType, DL: *DL));
2313 }
2314
2315 replaceValue(Old&: *EI, New&: *NewLoad, Erase: false);
2316 }
2317
2318 FailureGuard.release();
2319 return true;
2320}
2321
2322/// Try to scalarize vector loads feeding bitcast instructions.
2323bool VectorCombine::scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy,
2324 Value *Ptr) {
2325 InstructionCost OriginalCost =
2326 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: VecTy, Alignment: LI->getAlign(),
2327 AddressSpace: LI->getPointerAddressSpace(), CostKind);
2328
2329 if (!isa<FixedVectorType>(Val: VecTy))
2330 return false;
2331
2332 Type *TargetScalarType = nullptr;
2333 unsigned VecBitWidth = DL->getTypeSizeInBits(Ty: VecTy);
2334
2335 for (User *U : LI->users()) {
2336 auto *BC = cast<BitCastInst>(Val: U);
2337
2338 Type *DestTy = BC->getDestTy();
2339 if (!DestTy->isIntegerTy() && !DestTy->isFloatingPointTy())
2340 return false;
2341
2342 unsigned DestBitWidth = DL->getTypeSizeInBits(Ty: DestTy);
2343 if (DestBitWidth != VecBitWidth)
2344 return false;
2345
2346 // All bitcasts must target the same scalar type.
2347 if (!TargetScalarType)
2348 TargetScalarType = DestTy;
2349 else if (TargetScalarType != DestTy)
2350 return false;
2351
2352 OriginalCost +=
2353 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: TargetScalarType, Src: VecTy,
2354 CCH: TTI.getCastContextHint(I: BC), CostKind, I: BC);
2355 }
2356
2357 if (!TargetScalarType)
2358 return false;
2359
2360 assert(!LI->user_empty() && "Unexpected load without bitcast users");
2361 InstructionCost ScalarizedCost =
2362 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: TargetScalarType, Alignment: LI->getAlign(),
2363 AddressSpace: LI->getPointerAddressSpace(), CostKind);
2364
2365 LLVM_DEBUG(dbgs() << "Found vector load feeding only bitcasts: " << *LI
2366 << "\n OriginalCost: " << OriginalCost
2367 << " vs ScalarizedCost: " << ScalarizedCost << "\n");
2368
2369 if (ScalarizedCost >= OriginalCost)
2370 return false;
2371
2372 // Ensure we add the load back to the worklist BEFORE its users so they can
2373 // erased in the correct order.
2374 Worklist.push(I: LI);
2375
2376 Builder.SetInsertPoint(LI);
2377 auto *ScalarLoad =
2378 Builder.CreateLoad(Ty: TargetScalarType, Ptr, Name: LI->getName() + ".scalar");
2379 ScalarLoad->setAlignment(LI->getAlign());
2380 ScalarLoad->copyMetadata(SrcInst: *LI);
2381
2382 // Replace all bitcast users with the scalar load.
2383 for (User *U : LI->users()) {
2384 auto *BC = cast<BitCastInst>(Val: U);
2385 replaceValue(Old&: *BC, New&: *ScalarLoad, Erase: false);
2386 }
2387
2388 return true;
2389}
2390
2391bool VectorCombine::scalarizeExtExtract(Instruction &I) {
2392 if (!TTI.allowVectorElementIndexingUsingGEP())
2393 return false;
2394 auto *Ext = dyn_cast<ZExtInst>(Val: &I);
2395 if (!Ext)
2396 return false;
2397
2398 // Try to convert a vector zext feeding only extracts to a set of scalar
2399 // (Src << ExtIdx *Size) & (Size -1)
2400 // if profitable .
2401 auto *SrcTy = dyn_cast<FixedVectorType>(Val: Ext->getOperand(i_nocapture: 0)->getType());
2402 if (!SrcTy)
2403 return false;
2404 auto *DstTy = cast<FixedVectorType>(Val: Ext->getType());
2405
2406 Type *ScalarDstTy = DstTy->getElementType();
2407 if (DL->getTypeSizeInBits(Ty: SrcTy) != DL->getTypeSizeInBits(Ty: ScalarDstTy))
2408 return false;
2409
2410 InstructionCost VectorCost =
2411 TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: DstTy, Src: SrcTy,
2412 CCH: TTI::CastContextHint::None, CostKind, I: Ext);
2413 unsigned ExtCnt = 0;
2414 bool ExtLane0 = false;
2415 for (User *U : Ext->users()) {
2416 uint64_t Idx;
2417 if (!match(V: U, P: m_ExtractElt(Val: m_Value(), Idx: m_ConstantInt(V&: Idx))))
2418 return false;
2419 // An out-of-bounds extractelement produces poison; bail out rather
2420 // than computing a shift amount that overflows the packed type.
2421 if (Idx >= SrcTy->getNumElements())
2422 return false;
2423 if (cast<Instruction>(Val: U)->use_empty())
2424 continue;
2425 ExtCnt += 1;
2426 ExtLane0 |= !Idx;
2427 VectorCost += TTI.getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: DstTy,
2428 CostKind, Index: Idx, Op0: U);
2429 }
2430
2431 InstructionCost ScalarCost =
2432 ExtCnt * TTI.getArithmeticInstrCost(
2433 Opcode: Instruction::And, Ty: ScalarDstTy, CostKind,
2434 Opd1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
2435 Opd2Info: {.Kind: TTI::OK_NonUniformConstantValue, .Properties: TTI::OP_None}) +
2436 (ExtCnt - ExtLane0) *
2437 TTI.getArithmeticInstrCost(
2438 Opcode: Instruction::LShr, Ty: ScalarDstTy, CostKind,
2439 Opd1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
2440 Opd2Info: {.Kind: TTI::OK_NonUniformConstantValue, .Properties: TTI::OP_None});
2441 if (ScalarCost > VectorCost)
2442 return false;
2443
2444 Value *ScalarV = Ext->getOperand(i_nocapture: 0);
2445 if (!isGuaranteedNotToBePoison(V: ScalarV, AC: SQ.AC, CtxI: dyn_cast<Instruction>(Val: ScalarV),
2446 DT: SQ.DT)) {
2447 // Check wether all lanes are extracted, all extracts trigger UB
2448 // on poison, and the last extract (and hence all previous ones)
2449 // are guaranteed to execute if Ext executes. If so, we do not
2450 // need to insert a freeze.
2451 SmallDenseSet<ConstantInt *, 8> ExtractedLanes;
2452 bool AllExtractsTriggerUB = true;
2453 ExtractElementInst *LastExtract = nullptr;
2454 BasicBlock *ExtBB = Ext->getParent();
2455 for (User *U : Ext->users()) {
2456 auto *Extract = cast<ExtractElementInst>(Val: U);
2457 if (Extract->getParent() != ExtBB || !programUndefinedIfPoison(Inst: Extract)) {
2458 AllExtractsTriggerUB = false;
2459 break;
2460 }
2461 ExtractedLanes.insert(V: cast<ConstantInt>(Val: Extract->getIndexOperand()));
2462 if (!LastExtract || LastExtract->comesBefore(Other: Extract))
2463 LastExtract = Extract;
2464 }
2465 if (ExtractedLanes.size() != DstTy->getNumElements() ||
2466 !AllExtractsTriggerUB ||
2467 !isGuaranteedToTransferExecutionToSuccessor(Begin: Ext->getIterator(),
2468 End: LastExtract->getIterator()))
2469 ScalarV = Builder.CreateFreeze(V: ScalarV);
2470 }
2471 ScalarV = Builder.CreateBitCast(
2472 V: ScalarV,
2473 DestTy: IntegerType::get(C&: SrcTy->getContext(), NumBits: DL->getTypeSizeInBits(Ty: SrcTy)));
2474 uint64_t SrcEltSizeInBits = DL->getTypeSizeInBits(Ty: SrcTy->getElementType());
2475 uint64_t TotalBits = DL->getTypeSizeInBits(Ty: SrcTy);
2476 APInt EltBitMask = APInt::getLowBitsSet(numBits: TotalBits, loBitsSet: SrcEltSizeInBits);
2477 Type *PackedTy = IntegerType::get(C&: SrcTy->getContext(), NumBits: TotalBits);
2478 Value *Mask = ConstantInt::get(Ty: PackedTy, V: EltBitMask);
2479 for (User *U : Ext->users()) {
2480 auto *Extract = cast<ExtractElementInst>(Val: U);
2481 uint64_t Idx =
2482 cast<ConstantInt>(Val: Extract->getIndexOperand())->getZExtValue();
2483 uint64_t ShiftAmt =
2484 DL->isBigEndian()
2485 ? (TotalBits - SrcEltSizeInBits - Idx * SrcEltSizeInBits)
2486 : (Idx * SrcEltSizeInBits);
2487 Value *LShr = Builder.CreateLShr(LHS: ScalarV, RHS: ShiftAmt);
2488 Value *And = Builder.CreateAnd(LHS: LShr, RHS: Mask);
2489 U->replaceAllUsesWith(V: And);
2490 }
2491 return true;
2492}
2493
2494/// Try to fold "(or (zext (bitcast X)), (shl (zext (bitcast Y)), C))"
2495/// to "(bitcast (concat X, Y))"
2496/// where X/Y are bitcasted from i1 mask vectors.
2497bool VectorCombine::foldConcatOfBoolMasks(Instruction &I) {
2498 Type *Ty = I.getType();
2499 if (!Ty->isIntegerTy())
2500 return false;
2501
2502 // TODO: Add big endian test coverage
2503 if (DL->isBigEndian())
2504 return false;
2505
2506 // Restrict to disjoint cases so the mask vectors aren't overlapping.
2507 Instruction *X, *Y;
2508 if (!match(V: &I, P: m_DisjointOr(L: m_Instruction(I&: X), R: m_Instruction(I&: Y))))
2509 return false;
2510
2511 // Allow both sources to contain shl, to handle more generic pattern:
2512 // "(or (shl (zext (bitcast X)), C1), (shl (zext (bitcast Y)), C2))"
2513 Value *SrcX;
2514 uint64_t ShAmtX = 0;
2515 if (!match(V: X, P: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcX)))))) &&
2516 !match(V: X, P: m_OneUse(
2517 SubPattern: m_Shl(L: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcX))))),
2518 R: m_ConstantInt(V&: ShAmtX)))))
2519 return false;
2520
2521 Value *SrcY;
2522 uint64_t ShAmtY = 0;
2523 if (!match(V: Y, P: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcY)))))) &&
2524 !match(V: Y, P: m_OneUse(
2525 SubPattern: m_Shl(L: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcY))))),
2526 R: m_ConstantInt(V&: ShAmtY)))))
2527 return false;
2528
2529 // Canonicalize larger shift to the RHS.
2530 if (ShAmtX > ShAmtY) {
2531 std::swap(a&: X, b&: Y);
2532 std::swap(a&: SrcX, b&: SrcY);
2533 std::swap(a&: ShAmtX, b&: ShAmtY);
2534 }
2535
2536 // Ensure both sources are matching vXi1 bool mask types, and that the shift
2537 // difference is the mask width so they can be easily concatenated together.
2538 uint64_t ShAmtDiff = ShAmtY - ShAmtX;
2539 unsigned NumSHL = (ShAmtX > 0) + (ShAmtY > 0);
2540 unsigned BitWidth = Ty->getPrimitiveSizeInBits();
2541 auto *MaskTy = dyn_cast<FixedVectorType>(Val: SrcX->getType());
2542 if (!MaskTy || SrcX->getType() != SrcY->getType() ||
2543 !MaskTy->getElementType()->isIntegerTy(BitWidth: 1) ||
2544 MaskTy->getNumElements() != ShAmtDiff ||
2545 MaskTy->getNumElements() > (BitWidth / 2))
2546 return false;
2547
2548 auto *ConcatTy = FixedVectorType::getDoubleElementsVectorType(VTy: MaskTy);
2549 auto *ConcatIntTy =
2550 Type::getIntNTy(C&: Ty->getContext(), N: ConcatTy->getNumElements());
2551 auto *MaskIntTy = Type::getIntNTy(C&: Ty->getContext(), N: ShAmtDiff);
2552
2553 SmallVector<int, 32> ConcatMask(ConcatTy->getNumElements());
2554 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
2555
2556 // TODO: Is it worth supporting multi use cases?
2557 InstructionCost OldCost = 0;
2558 OldCost += TTI.getArithmeticInstrCost(Opcode: Instruction::Or, Ty, CostKind);
2559 OldCost +=
2560 NumSHL * TTI.getArithmeticInstrCost(Opcode: Instruction::Shl, Ty, CostKind);
2561 OldCost += 2 * TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: Ty, Src: MaskIntTy,
2562 CCH: TTI::CastContextHint::None, CostKind);
2563 OldCost += 2 * TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: MaskIntTy, Src: MaskTy,
2564 CCH: TTI::CastContextHint::None, CostKind);
2565
2566 InstructionCost NewCost = 0;
2567 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ConcatTy,
2568 SrcTy: MaskTy, CostKind, Mask: ConcatMask);
2569 NewCost += TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: ConcatIntTy, Src: ConcatTy,
2570 CCH: TTI::CastContextHint::None, CostKind);
2571 if (Ty != ConcatIntTy)
2572 NewCost += TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: Ty, Src: ConcatIntTy,
2573 CCH: TTI::CastContextHint::None, CostKind);
2574 if (ShAmtX > 0)
2575 NewCost += TTI.getArithmeticInstrCost(Opcode: Instruction::Shl, Ty, CostKind);
2576
2577 LLVM_DEBUG(dbgs() << "Found a concatenation of bitcasted bool masks: " << I
2578 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2579 << "\n");
2580
2581 if (NewCost > OldCost)
2582 return false;
2583
2584 // Build bool mask concatenation, bitcast back to scalar integer, and perform
2585 // any residual zero-extension or shifting.
2586 Value *Concat = Builder.CreateShuffleVector(V1: SrcX, V2: SrcY, Mask: ConcatMask);
2587
2588 Value *Result = Builder.CreateBitCast(V: Concat, DestTy: ConcatIntTy);
2589
2590 if (Ty != ConcatIntTy) {
2591 Result = Builder.CreateZExt(V: Result, DestTy: Ty);
2592 }
2593
2594 if (ShAmtX > 0) {
2595 Result = Builder.CreateShl(LHS: Result, RHS: ShAmtX);
2596 }
2597
2598 replaceValue(Old&: I, New&: *Result);
2599 return true;
2600}
2601
2602/// Try to convert "shuffle (binop (shuffle, shuffle)), undef"
2603/// --> "binop (shuffle), (shuffle)".
2604bool VectorCombine::foldPermuteOfBinops(Instruction &I) {
2605 BinaryOperator *BinOp;
2606 ArrayRef<int> OuterMask;
2607 if (!match(V: &I, P: m_Shuffle(v1: m_BinOp(I&: BinOp), v2: m_Undef(), mask: m_Mask(OuterMask))))
2608 return false;
2609
2610 // Don't introduce poison into div/rem.
2611 if (BinOp->isIntDivRem() && llvm::is_contained(Range&: OuterMask, Element: PoisonMaskElem))
2612 return false;
2613
2614 Value *Op00, *Op01, *Op10, *Op11;
2615 ArrayRef<int> Mask0, Mask1;
2616 bool Match0 = match(V: BinOp->getOperand(i_nocapture: 0),
2617 P: m_Shuffle(v1: m_Value(V&: Op00), v2: m_Value(V&: Op01), mask: m_Mask(Mask0)));
2618 bool Match1 = match(V: BinOp->getOperand(i_nocapture: 1),
2619 P: m_Shuffle(v1: m_Value(V&: Op10), v2: m_Value(V&: Op11), mask: m_Mask(Mask1)));
2620 if (!Match0 && !Match1)
2621 return false;
2622
2623 Op00 = Match0 ? Op00 : BinOp->getOperand(i_nocapture: 0);
2624 Op01 = Match0 ? Op01 : BinOp->getOperand(i_nocapture: 0);
2625 Op10 = Match1 ? Op10 : BinOp->getOperand(i_nocapture: 1);
2626 Op11 = Match1 ? Op11 : BinOp->getOperand(i_nocapture: 1);
2627
2628 Instruction::BinaryOps Opcode = BinOp->getOpcode();
2629 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
2630 auto *BinOpTy = dyn_cast<FixedVectorType>(Val: BinOp->getType());
2631 auto *Op0Ty = dyn_cast<FixedVectorType>(Val: Op00->getType());
2632 auto *Op1Ty = dyn_cast<FixedVectorType>(Val: Op10->getType());
2633 if (!ShuffleDstTy || !BinOpTy || !Op0Ty || !Op1Ty)
2634 return false;
2635
2636 unsigned NumSrcElts = BinOpTy->getNumElements();
2637
2638 // Don't accept shuffles that reference the second operand in
2639 // div/rem or if its an undef arg.
2640 if ((BinOp->isIntDivRem() || !isa<PoisonValue>(Val: I.getOperand(i: 1))) &&
2641 any_of(Range&: OuterMask, P: [NumSrcElts](int M) { return M >= (int)NumSrcElts; }))
2642 return false;
2643
2644 // Merge outer / inner (or identity if no match) shuffles.
2645 SmallVector<int> NewMask0, NewMask1;
2646 for (int M : OuterMask) {
2647 if (M < 0 || M >= (int)NumSrcElts) {
2648 NewMask0.push_back(Elt: PoisonMaskElem);
2649 NewMask1.push_back(Elt: PoisonMaskElem);
2650 } else {
2651 NewMask0.push_back(Elt: Match0 ? Mask0[M] : M);
2652 NewMask1.push_back(Elt: Match1 ? Mask1[M] : M);
2653 }
2654 }
2655
2656 unsigned NumOpElts = Op0Ty->getNumElements();
2657 bool IsIdentity0 = ShuffleDstTy == Op0Ty &&
2658 all_of(Range&: NewMask0, P: [NumOpElts](int M) { return M < (int)NumOpElts; }) &&
2659 ShuffleVectorInst::isIdentityMask(Mask: NewMask0, NumSrcElts: NumOpElts);
2660 bool IsIdentity1 = ShuffleDstTy == Op1Ty &&
2661 all_of(Range&: NewMask1, P: [NumOpElts](int M) { return M < (int)NumOpElts; }) &&
2662 ShuffleVectorInst::isIdentityMask(Mask: NewMask1, NumSrcElts: NumOpElts);
2663
2664 InstructionCost NewCost = 0;
2665 // Try to merge shuffles across the binop if the new shuffles are not costly.
2666 InstructionCost BinOpCost =
2667 TTI.getArithmeticInstrCost(Opcode, Ty: BinOpTy, CostKind);
2668 InstructionCost OldCost =
2669 BinOpCost + TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
2670 DstTy: ShuffleDstTy, SrcTy: BinOpTy, CostKind, Mask: OuterMask,
2671 Index: 0, SubTp: nullptr, Args: {BinOp}, CtxI: &I);
2672 if (!BinOp->hasOneUse())
2673 NewCost += BinOpCost;
2674
2675 if (Match0) {
2676 InstructionCost Shuf0Cost = TTI.getShuffleCost(
2677 Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: BinOpTy, SrcTy: Op0Ty, CostKind, Mask: Mask0,
2678 Index: 0, SubTp: nullptr, Args: {Op00, Op01}, CtxI: cast<Instruction>(Val: BinOp->getOperand(i_nocapture: 0)));
2679 OldCost += Shuf0Cost;
2680 if (!BinOp->hasOneUse() || !BinOp->getOperand(i_nocapture: 0)->hasOneUse())
2681 NewCost += Shuf0Cost;
2682 }
2683 if (Match1) {
2684 InstructionCost Shuf1Cost = TTI.getShuffleCost(
2685 Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: BinOpTy, SrcTy: Op1Ty, CostKind, Mask: Mask1,
2686 Index: 0, SubTp: nullptr, Args: {Op10, Op11}, CtxI: cast<Instruction>(Val: BinOp->getOperand(i_nocapture: 1)));
2687 OldCost += Shuf1Cost;
2688 if (!BinOp->hasOneUse() || !BinOp->getOperand(i_nocapture: 1)->hasOneUse())
2689 NewCost += Shuf1Cost;
2690 }
2691
2692 NewCost += TTI.getArithmeticInstrCost(Opcode, Ty: ShuffleDstTy, CostKind);
2693
2694 if (!IsIdentity0)
2695 NewCost +=
2696 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ShuffleDstTy,
2697 SrcTy: Op0Ty, CostKind, Mask: NewMask0, Index: 0, SubTp: nullptr, Args: {Op00, Op01});
2698 if (!IsIdentity1)
2699 NewCost +=
2700 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ShuffleDstTy,
2701 SrcTy: Op1Ty, CostKind, Mask: NewMask1, Index: 0, SubTp: nullptr, Args: {Op10, Op11});
2702
2703 LLVM_DEBUG(dbgs() << "Found a shuffle feeding a shuffled binop: " << I
2704 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2705 << "\n");
2706
2707 // If costs are equal, still fold as we reduce instruction count.
2708 if (NewCost > OldCost)
2709 return false;
2710
2711 Value *LHS =
2712 IsIdentity0 ? Op00 : Builder.CreateShuffleVector(V1: Op00, V2: Op01, Mask: NewMask0);
2713 Value *RHS =
2714 IsIdentity1 ? Op10 : Builder.CreateShuffleVector(V1: Op10, V2: Op11, Mask: NewMask1);
2715 Value *NewBO = Builder.CreateBinOp(Opc: Opcode, LHS, RHS);
2716
2717 // Intersect flags from the old binops.
2718 if (auto *NewInst = dyn_cast<Instruction>(Val: NewBO))
2719 NewInst->copyIRFlags(V: BinOp);
2720
2721 replaceValue(Old&: I, New&: *NewBO);
2722 return true;
2723}
2724
2725/// Try to convert "shuffle (binop), (binop)" into "binop (shuffle), (shuffle)".
2726/// Try to convert "shuffle (cmpop), (cmpop)" into "cmpop (shuffle), (shuffle)".
2727bool VectorCombine::foldShuffleOfBinops(Instruction &I) {
2728 ArrayRef<int> OldMask;
2729 Instruction *LHS, *RHS;
2730 if (!match(V: &I, P: m_Shuffle(v1: m_Instruction(I&: LHS), v2: m_Instruction(I&: RHS),
2731 mask: m_Mask(OldMask))))
2732 return false;
2733
2734 // TODO: Add support for addlike etc.
2735 if (LHS->getOpcode() != RHS->getOpcode())
2736 return false;
2737
2738 Value *X, *Y, *Z, *W;
2739 bool IsCommutative = false;
2740 CmpPredicate PredLHS = CmpInst::BAD_ICMP_PREDICATE;
2741 CmpPredicate PredRHS = CmpInst::BAD_ICMP_PREDICATE;
2742 if (match(V: LHS, P: m_BinOp(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
2743 match(V: RHS, P: m_BinOp(L: m_Value(V&: Z), R: m_Value(V&: W)))) {
2744 auto *BO = cast<BinaryOperator>(Val: LHS);
2745 // Don't introduce poison into div/rem.
2746 if (llvm::is_contained(Range&: OldMask, Element: PoisonMaskElem) && BO->isIntDivRem())
2747 return false;
2748 IsCommutative = BinaryOperator::isCommutative(Opcode: BO->getOpcode());
2749 } else if (match(V: LHS, P: m_Cmp(Pred&: PredLHS, L: m_Value(V&: X), R: m_Value(V&: Y))) &&
2750 match(V: RHS, P: m_Cmp(Pred&: PredRHS, L: m_Value(V&: Z), R: m_Value(V&: W))) &&
2751 (CmpInst::Predicate)PredLHS == (CmpInst::Predicate)PredRHS) {
2752 IsCommutative = cast<CmpInst>(Val: LHS)->isCommutative();
2753 } else
2754 return false;
2755
2756 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
2757 auto *BinResTy = dyn_cast<FixedVectorType>(Val: LHS->getType());
2758 auto *BinOpTy = dyn_cast<FixedVectorType>(Val: X->getType());
2759 if (!ShuffleDstTy || !BinResTy || !BinOpTy || X->getType() != Z->getType())
2760 return false;
2761
2762 bool SameBinOp = LHS == RHS;
2763 unsigned NumSrcElts = BinOpTy->getNumElements();
2764
2765 // If we have something like "add X, Y" and "add Z, X", swap ops to match.
2766 if (IsCommutative && X != Z && Y != W && (X == W || Y == Z))
2767 std::swap(a&: X, b&: Y);
2768
2769 auto ConvertToUnary = [NumSrcElts](int &M) {
2770 if (M >= (int)NumSrcElts)
2771 M -= NumSrcElts;
2772 };
2773
2774 SmallVector<int> NewMask0(OldMask);
2775 TargetTransformInfo::ShuffleKind SK0 = TargetTransformInfo::SK_PermuteTwoSrc;
2776 TTI::OperandValueInfo Op0Info = TTI.commonOperandInfo(X, Y: Z);
2777 if (X == Z) {
2778 llvm::for_each(Range&: NewMask0, F: ConvertToUnary);
2779 SK0 = TargetTransformInfo::SK_PermuteSingleSrc;
2780 Z = PoisonValue::get(T: BinOpTy);
2781 }
2782
2783 SmallVector<int> NewMask1(OldMask);
2784 TargetTransformInfo::ShuffleKind SK1 = TargetTransformInfo::SK_PermuteTwoSrc;
2785 TTI::OperandValueInfo Op1Info = TTI.commonOperandInfo(X: Y, Y: W);
2786 if (Y == W) {
2787 llvm::for_each(Range&: NewMask1, F: ConvertToUnary);
2788 SK1 = TargetTransformInfo::SK_PermuteSingleSrc;
2789 W = PoisonValue::get(T: BinOpTy);
2790 }
2791
2792 // Try to replace a binop with a shuffle if the shuffle is not costly.
2793 // When SameBinOp, only count the binop cost once.
2794 InstructionCost LHSCost = TTI.getInstructionCost(U: LHS, CostKind);
2795 InstructionCost RHSCost = TTI.getInstructionCost(U: RHS, CostKind);
2796
2797 InstructionCost OldCost = LHSCost;
2798 if (!SameBinOp) {
2799 OldCost += RHSCost;
2800 }
2801 OldCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc,
2802 DstTy: ShuffleDstTy, SrcTy: BinResTy, CostKind, Mask: OldMask, Index: 0,
2803 SubTp: nullptr, Args: {LHS, RHS}, CtxI: &I);
2804
2805 // Handle shuffle(binop(shuffle(x),y),binop(z,shuffle(w))) style patterns
2806 // where one use shuffles have gotten split across the binop/cmp. These
2807 // often allow a major reduction in total cost that wouldn't happen as
2808 // individual folds.
2809 auto MergeInner = [&](Value *&Op, int Offset, MutableArrayRef<int> Mask,
2810 TTI::TargetCostKind CostKind) -> bool {
2811 Value *InnerOp;
2812 ArrayRef<int> InnerMask;
2813 if (match(V: Op, P: m_OneUse(SubPattern: m_Shuffle(v1: m_Value(V&: InnerOp), v2: m_Undef(),
2814 mask: m_Mask(InnerMask)))) &&
2815 InnerOp->getType() == Op->getType() &&
2816 all_of(Range&: InnerMask,
2817 P: [NumSrcElts](int M) { return M < (int)NumSrcElts; })) {
2818 for (int &M : Mask)
2819 if (Offset <= M && M < (int)(Offset + NumSrcElts)) {
2820 M = InnerMask[M - Offset];
2821 M = 0 <= M ? M + Offset : M;
2822 }
2823 OldCost += TTI.getInstructionCost(U: cast<Instruction>(Val: Op), CostKind);
2824 Op = InnerOp;
2825 return true;
2826 }
2827 return false;
2828 };
2829 bool ReducedInstCount = false;
2830 ReducedInstCount |= MergeInner(X, 0, NewMask0, CostKind);
2831 ReducedInstCount |= MergeInner(Y, 0, NewMask1, CostKind);
2832 ReducedInstCount |= MergeInner(Z, NumSrcElts, NewMask0, CostKind);
2833 ReducedInstCount |= MergeInner(W, NumSrcElts, NewMask1, CostKind);
2834 bool SingleSrcBinOp = (X == Y) && (Z == W) && (NewMask0 == NewMask1);
2835 // SingleSrcBinOp only reduces instruction count if we also eliminate the
2836 // original binop(s). If binops have multiple uses, they won't be eliminated.
2837 ReducedInstCount |= SingleSrcBinOp && LHS->hasOneUser() && RHS->hasOneUser();
2838
2839 // For concat shuffles of i1 vectors where both binops are one-use, the
2840 // transform keeps the same instruction count but canonicalises to a single
2841 // wider binop, enabling downstream folds (e.g. NOT(XOR(concat(a,b),
2842 // concat(c,d))) -> XNOR(concat(a,b),concat(c,d)) on AVX-512 mask regs).
2843 // Restrict to BinaryOperator (not CmpInst) since narrow comparisons may
2844 // be cheaper than wide ones on some targets (e.g. AVX-512 vpcmpeq).
2845 ReducedInstCount |= cast<ShuffleVectorInst>(Val: &I)->isConcat() &&
2846 I.getType()->getScalarType()->isIntegerTy(BitWidth: 1) &&
2847 isa<BinaryOperator>(Val: LHS) && LHS->hasOneUser() &&
2848 RHS->hasOneUser();
2849
2850 auto *ShuffleCmpTy =
2851 FixedVectorType::get(ElementType: BinOpTy->getElementType(), FVTy: ShuffleDstTy);
2852 InstructionCost NewCost = TTI.getShuffleCost(
2853 Kind: SK0, DstTy: ShuffleCmpTy, SrcTy: BinOpTy, CostKind, Mask: NewMask0, Index: 0, SubTp: nullptr, Args: {X, Z});
2854 if (!SingleSrcBinOp)
2855 NewCost += TTI.getShuffleCost(Kind: SK1, DstTy: ShuffleCmpTy, SrcTy: BinOpTy, CostKind,
2856 Mask: NewMask1, Index: 0, SubTp: nullptr, Args: {Y, W});
2857
2858 if (PredLHS == CmpInst::BAD_ICMP_PREDICATE) {
2859 NewCost += TTI.getArithmeticInstrCost(Opcode: LHS->getOpcode(), Ty: ShuffleDstTy,
2860 CostKind, Opd1Info: Op0Info, Opd2Info: Op1Info);
2861 } else {
2862 NewCost +=
2863 TTI.getCmpSelInstrCost(Opcode: LHS->getOpcode(), ValTy: ShuffleCmpTy, CondTy: ShuffleDstTy,
2864 VecPred: PredLHS, CostKind, Op1Info: Op0Info, Op2Info: Op1Info);
2865 }
2866 // If LHS/RHS have other uses, we need to account for the cost of keeping
2867 // the original instructions. When SameBinOp, only add the cost once.
2868 if (!LHS->hasOneUser())
2869 NewCost += LHSCost;
2870 if (!SameBinOp && !RHS->hasOneUser())
2871 NewCost += RHSCost;
2872
2873 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two binops: " << I
2874 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2875 << "\n");
2876
2877 // If either shuffle will constant fold away, then fold for the same cost as
2878 // we will reduce the instruction count.
2879 ReducedInstCount |= (isa<Constant>(Val: X) && isa<Constant>(Val: Z)) ||
2880 (isa<Constant>(Val: Y) && isa<Constant>(Val: W));
2881 if (ReducedInstCount ? (NewCost > OldCost) : (NewCost >= OldCost))
2882 return false;
2883
2884 Value *Shuf0 = Builder.CreateShuffleVector(V1: X, V2: Z, Mask: NewMask0);
2885 Value *Shuf1 =
2886 SingleSrcBinOp ? Shuf0 : Builder.CreateShuffleVector(V1: Y, V2: W, Mask: NewMask1);
2887 Value *NewBO = PredLHS == CmpInst::BAD_ICMP_PREDICATE
2888 ? Builder.CreateBinOp(
2889 Opc: cast<BinaryOperator>(Val: LHS)->getOpcode(), LHS: Shuf0, RHS: Shuf1)
2890 : Builder.CreateCmp(Pred: PredLHS, LHS: Shuf0, RHS: Shuf1);
2891
2892 // Intersect flags from the old binops.
2893 if (auto *NewInst = dyn_cast<Instruction>(Val: NewBO)) {
2894 NewInst->copyIRFlags(V: LHS);
2895 NewInst->andIRFlags(V: RHS);
2896 }
2897
2898 replaceValue(Old&: I, New&: *NewBO);
2899 return true;
2900}
2901
2902/// Try to convert,
2903/// (shuffle(select(c1,t1,f1)), (select(c2,t2,f2)), m) into
2904/// (select (shuffle c1,c2,m), (shuffle t1,t2,m), (shuffle f1,f2,m))
2905bool VectorCombine::foldShuffleOfSelects(Instruction &I) {
2906 ArrayRef<int> Mask;
2907 Value *C1, *T1, *F1, *C2, *T2, *F2;
2908 if (!match(V: &I, P: m_Shuffle(v1: m_Select(C: m_Value(V&: C1), L: m_Value(V&: T1), R: m_Value(V&: F1)),
2909 v2: m_Select(C: m_Value(V&: C2), L: m_Value(V&: T2), R: m_Value(V&: F2)),
2910 mask: m_Mask(Mask))))
2911 return false;
2912
2913 auto *Sel1 = cast<Instruction>(Val: I.getOperand(i: 0));
2914 auto *Sel2 = cast<Instruction>(Val: I.getOperand(i: 1));
2915
2916 auto *C1VecTy = dyn_cast<FixedVectorType>(Val: C1->getType());
2917 auto *C2VecTy = dyn_cast<FixedVectorType>(Val: C2->getType());
2918 if (!C1VecTy || !C2VecTy || C1VecTy != C2VecTy)
2919 return false;
2920
2921 auto *SI0FOp = dyn_cast<FPMathOperator>(Val: I.getOperand(i: 0));
2922 auto *SI1FOp = dyn_cast<FPMathOperator>(Val: I.getOperand(i: 1));
2923 // SelectInsts must have the same FMF.
2924 if (((SI0FOp == nullptr) != (SI1FOp == nullptr)) ||
2925 ((SI0FOp != nullptr) &&
2926 (SI0FOp->getFastMathFlags() != SI1FOp->getFastMathFlags())))
2927 return false;
2928
2929 auto *SrcVecTy = cast<FixedVectorType>(Val: T1->getType());
2930 auto *DstVecTy = cast<FixedVectorType>(Val: I.getType());
2931 auto SK = TargetTransformInfo::SK_PermuteTwoSrc;
2932 auto SelOp = Instruction::Select;
2933
2934 InstructionCost CostSel1 = TTI.getCmpSelInstrCost(
2935 Opcode: SelOp, ValTy: SrcVecTy, CondTy: C1VecTy, VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
2936 InstructionCost CostSel2 = TTI.getCmpSelInstrCost(
2937 Opcode: SelOp, ValTy: SrcVecTy, CondTy: C2VecTy, VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
2938
2939 InstructionCost OldCost =
2940 CostSel1 + CostSel2 +
2941 TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask, Index: 0, SubTp: nullptr,
2942 Args: {I.getOperand(i: 0), I.getOperand(i: 1)}, CtxI: &I);
2943
2944 InstructionCost NewCost = TTI.getShuffleCost(
2945 Kind: SK, DstTy: FixedVectorType::get(ElementType: C1VecTy->getScalarType(), NumElts: Mask.size()), SrcTy: C1VecTy,
2946 CostKind, Mask, Index: 0, SubTp: nullptr, Args: {C1, C2});
2947 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask, Index: 0,
2948 SubTp: nullptr, Args: {T1, T2});
2949 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask, Index: 0,
2950 SubTp: nullptr, Args: {F1, F2});
2951 auto *C1C2ShuffledVecTy = FixedVectorType::get(
2952 ElementType: Type::getInt1Ty(C&: I.getContext()), NumElts: DstVecTy->getNumElements());
2953 NewCost += TTI.getCmpSelInstrCost(Opcode: SelOp, ValTy: DstVecTy, CondTy: C1C2ShuffledVecTy,
2954 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
2955
2956 if (!Sel1->hasOneUse())
2957 NewCost += CostSel1;
2958 if (!Sel2->hasOneUse())
2959 NewCost += CostSel2;
2960
2961 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two selects: " << I
2962 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2963 << "\n");
2964 if (NewCost > OldCost)
2965 return false;
2966
2967 Value *ShuffleCmp = Builder.CreateShuffleVector(V1: C1, V2: C2, Mask);
2968 Value *ShuffleTrue = Builder.CreateShuffleVector(V1: T1, V2: T2, Mask);
2969 Value *ShuffleFalse = Builder.CreateShuffleVector(V1: F1, V2: F2, Mask);
2970 Value *NewSel;
2971 // We presuppose that the SelectInsts have the same FMF.
2972 if (SI0FOp)
2973 NewSel = Builder.CreateSelectFMF(C: ShuffleCmp, True: ShuffleTrue, False: ShuffleFalse,
2974 FMFSource: SI0FOp->getFastMathFlags());
2975 else
2976 NewSel = Builder.CreateSelect(C: ShuffleCmp, True: ShuffleTrue, False: ShuffleFalse);
2977
2978 replaceValue(Old&: I, New&: *NewSel);
2979 return true;
2980}
2981
2982/// Try to convert "shuffle (castop), (castop)" with a shared castop operand
2983/// into "castop (shuffle)".
2984bool VectorCombine::foldShuffleOfCastops(Instruction &I) {
2985 Value *V0, *V1;
2986 ArrayRef<int> OldMask;
2987 if (!match(V: &I, P: m_Shuffle(v1: m_Value(V&: V0), v2: m_Value(V&: V1), mask: m_Mask(OldMask))))
2988 return false;
2989
2990 // Check whether this is a binary shuffle.
2991 bool IsBinaryShuffle = !isa<UndefValue>(Val: V1);
2992
2993 auto *C0 = dyn_cast<CastInst>(Val: V0);
2994 auto *C1 = dyn_cast<CastInst>(Val: V1);
2995 if (!C0 || (IsBinaryShuffle && !C1))
2996 return false;
2997
2998 Instruction::CastOps Opcode = C0->getOpcode();
2999
3000 // If this is allowed, foldShuffleOfCastops can get stuck in a loop
3001 // with foldBitcastOfShuffle. Reject in favor of foldBitcastOfShuffle.
3002 if (!IsBinaryShuffle && Opcode == Instruction::BitCast)
3003 return false;
3004
3005 if (IsBinaryShuffle) {
3006 if (C0->getSrcTy() != C1->getSrcTy())
3007 return false;
3008 // Handle shuffle(zext_nneg(x), sext(y)) -> sext(shuffle(x,y)) folds.
3009 if (Opcode != C1->getOpcode()) {
3010 if (match(V: C0, P: m_SExtLike(Op: m_Value())) && match(V: C1, P: m_SExtLike(Op: m_Value())))
3011 Opcode = Instruction::SExt;
3012 else
3013 return false;
3014 }
3015 }
3016
3017 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3018 auto *CastDstTy = dyn_cast<FixedVectorType>(Val: C0->getDestTy());
3019 auto *CastSrcTy = dyn_cast<FixedVectorType>(Val: C0->getSrcTy());
3020 if (!ShuffleDstTy || !CastDstTy || !CastSrcTy)
3021 return false;
3022
3023 unsigned NumSrcElts = CastSrcTy->getNumElements();
3024 unsigned NumDstElts = CastDstTy->getNumElements();
3025 assert((NumDstElts == NumSrcElts || Opcode == Instruction::BitCast) &&
3026 "Only bitcasts expected to alter src/dst element counts");
3027
3028 // Check for bitcasting of unscalable vector types.
3029 // e.g. <32 x i40> -> <40 x i32>
3030 if (NumDstElts != NumSrcElts && (NumSrcElts % NumDstElts) != 0 &&
3031 (NumDstElts % NumSrcElts) != 0)
3032 return false;
3033
3034 SmallVector<int, 16> NewMask;
3035 if (NumSrcElts >= NumDstElts) {
3036 // The bitcast is from wide to narrow/equal elements. The shuffle mask can
3037 // always be expanded to the equivalent form choosing narrower elements.
3038 assert(NumSrcElts % NumDstElts == 0 && "Unexpected shuffle mask");
3039 unsigned ScaleFactor = NumSrcElts / NumDstElts;
3040 narrowShuffleMaskElts(Scale: ScaleFactor, Mask: OldMask, ScaledMask&: NewMask);
3041 } else {
3042 // The bitcast is from narrow elements to wide elements. The shuffle mask
3043 // must choose consecutive elements to allow casting first.
3044 assert(NumDstElts % NumSrcElts == 0 && "Unexpected shuffle mask");
3045 unsigned ScaleFactor = NumDstElts / NumSrcElts;
3046 if (!widenShuffleMaskElts(Scale: ScaleFactor, Mask: OldMask, ScaledMask&: NewMask))
3047 return false;
3048 }
3049
3050 auto *NewShuffleDstTy =
3051 FixedVectorType::get(ElementType: CastSrcTy->getScalarType(), NumElts: NewMask.size());
3052
3053 // Try to replace a castop with a shuffle if the shuffle is not costly.
3054 InstructionCost CostC0 =
3055 TTI.getCastInstrCost(Opcode: C0->getOpcode(), Dst: CastDstTy, Src: CastSrcTy,
3056 CCH: TTI::CastContextHint::None, CostKind, I: C0);
3057
3058 TargetTransformInfo::ShuffleKind ShuffleKind;
3059 if (IsBinaryShuffle)
3060 ShuffleKind = TargetTransformInfo::SK_PermuteTwoSrc;
3061 else
3062 ShuffleKind = TargetTransformInfo::SK_PermuteSingleSrc;
3063
3064 InstructionCost OldCost = CostC0;
3065 OldCost += TTI.getShuffleCost(Kind: ShuffleKind, DstTy: ShuffleDstTy, SrcTy: CastDstTy, CostKind,
3066 Mask: OldMask, Index: 0, SubTp: nullptr, Args: {}, CtxI: &I);
3067
3068 InstructionCost NewCost = TTI.getShuffleCost(Kind: ShuffleKind, DstTy: NewShuffleDstTy,
3069 SrcTy: CastSrcTy, CostKind, Mask: NewMask);
3070 NewCost += TTI.getCastInstrCost(Opcode, Dst: ShuffleDstTy, Src: NewShuffleDstTy,
3071 CCH: TTI::CastContextHint::None, CostKind);
3072 if (!C0->hasOneUse())
3073 NewCost += CostC0;
3074 if (IsBinaryShuffle) {
3075 InstructionCost CostC1 =
3076 TTI.getCastInstrCost(Opcode: C1->getOpcode(), Dst: CastDstTy, Src: CastSrcTy,
3077 CCH: TTI::CastContextHint::None, CostKind, I: C1);
3078 OldCost += CostC1;
3079 if (!C1->hasOneUse())
3080 NewCost += CostC1;
3081 }
3082
3083 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two casts: " << I
3084 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
3085 << "\n");
3086 if (NewCost > OldCost)
3087 return false;
3088
3089 Value *Shuf;
3090 if (IsBinaryShuffle)
3091 Shuf = Builder.CreateShuffleVector(V1: C0->getOperand(i_nocapture: 0), V2: C1->getOperand(i_nocapture: 0),
3092 Mask: NewMask);
3093 else
3094 Shuf = Builder.CreateShuffleVector(V: C0->getOperand(i_nocapture: 0), Mask: NewMask);
3095
3096 Value *Cast = Builder.CreateCast(Op: Opcode, V: Shuf, DestTy: ShuffleDstTy);
3097
3098 // Intersect flags from the old casts.
3099 if (auto *NewInst = dyn_cast<Instruction>(Val: Cast)) {
3100 NewInst->copyIRFlags(V: C0);
3101 if (IsBinaryShuffle)
3102 NewInst->andIRFlags(V: C1);
3103 }
3104
3105 replaceValue(Old&: I, New&: *Cast);
3106 return true;
3107}
3108
3109/// Try to convert any of:
3110/// "shuffle (shuffle x, y), (shuffle y, x)"
3111/// "shuffle (shuffle x, undef), (shuffle y, undef)"
3112/// "shuffle (shuffle x, undef), y"
3113/// "shuffle x, (shuffle y, undef)"
3114/// into "shuffle x, y".
3115bool VectorCombine::foldShuffleOfShuffles(Instruction &I) {
3116 ArrayRef<int> OuterMask;
3117 Value *OuterV0, *OuterV1;
3118 if (!match(V: &I,
3119 P: m_Shuffle(v1: m_Value(V&: OuterV0), v2: m_Value(V&: OuterV1), mask: m_Mask(OuterMask))))
3120 return false;
3121
3122 ArrayRef<int> InnerMask0, InnerMask1;
3123 Value *X0, *X1, *Y0, *Y1;
3124 bool Match0 =
3125 match(V: OuterV0, P: m_Shuffle(v1: m_Value(V&: X0), v2: m_Value(V&: Y0), mask: m_Mask(InnerMask0)));
3126 bool Match1 =
3127 match(V: OuterV1, P: m_Shuffle(v1: m_Value(V&: X1), v2: m_Value(V&: Y1), mask: m_Mask(InnerMask1)));
3128 if (!Match0 && !Match1)
3129 return false;
3130
3131 // If the outer shuffle is a permute, then create a fake inner all-poison
3132 // shuffle. This is easier than accounting for length-changing shuffles below.
3133 SmallVector<int, 16> PoisonMask1;
3134 if (!Match1 && isa<PoisonValue>(Val: OuterV1)) {
3135 X1 = X0;
3136 Y1 = Y0;
3137 PoisonMask1.append(NumInputs: InnerMask0.size(), Elt: PoisonMaskElem);
3138 InnerMask1 = PoisonMask1;
3139 Match1 = true; // fake match
3140 }
3141
3142 X0 = Match0 ? X0 : OuterV0;
3143 Y0 = Match0 ? Y0 : OuterV0;
3144 X1 = Match1 ? X1 : OuterV1;
3145 Y1 = Match1 ? Y1 : OuterV1;
3146 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3147 auto *ShuffleSrcTy = dyn_cast<FixedVectorType>(Val: X0->getType());
3148 auto *ShuffleImmTy = dyn_cast<FixedVectorType>(Val: OuterV0->getType());
3149 if (!ShuffleDstTy || !ShuffleSrcTy || !ShuffleImmTy ||
3150 X0->getType() != X1->getType())
3151 return false;
3152
3153 unsigned NumSrcElts = ShuffleSrcTy->getNumElements();
3154 unsigned NumImmElts = ShuffleImmTy->getNumElements();
3155
3156 // Attempt to merge shuffles, matching upto 2 source operands.
3157 // Replace index to a poison arg with PoisonMaskElem.
3158 // Bail if either inner masks reference an undef arg.
3159 SmallVector<int, 16> NewMask(OuterMask);
3160 Value *NewX = nullptr, *NewY = nullptr;
3161 for (int &M : NewMask) {
3162 Value *Src = nullptr;
3163 if (0 <= M && M < (int)NumImmElts) {
3164 Src = OuterV0;
3165 if (Match0) {
3166 M = InnerMask0[M];
3167 Src = M >= (int)NumSrcElts ? Y0 : X0;
3168 M = M >= (int)NumSrcElts ? (M - NumSrcElts) : M;
3169 }
3170 } else if (M >= (int)NumImmElts) {
3171 Src = OuterV1;
3172 M -= NumImmElts;
3173 if (Match1) {
3174 M = InnerMask1[M];
3175 Src = M >= (int)NumSrcElts ? Y1 : X1;
3176 M = M >= (int)NumSrcElts ? (M - NumSrcElts) : M;
3177 }
3178 }
3179 if (Src && M != PoisonMaskElem) {
3180 assert(0 <= M && M < (int)NumSrcElts && "Unexpected shuffle mask index");
3181 if (isa<UndefValue>(Val: Src)) {
3182 // We've referenced an undef element - if its poison, update the shuffle
3183 // mask, else bail.
3184 if (!isa<PoisonValue>(Val: Src))
3185 return false;
3186 M = PoisonMaskElem;
3187 continue;
3188 }
3189 if (!NewX || NewX == Src) {
3190 NewX = Src;
3191 continue;
3192 }
3193 if (!NewY || NewY == Src) {
3194 M += NumSrcElts;
3195 NewY = Src;
3196 continue;
3197 }
3198 return false;
3199 }
3200 }
3201
3202 if (!NewX) {
3203 replaceValue(Old&: I, New&: *PoisonValue::get(T: ShuffleDstTy));
3204 return true;
3205 }
3206
3207 if (!NewY)
3208 NewY = PoisonValue::get(T: ShuffleSrcTy);
3209
3210 // Have we folded to an Identity shuffle?
3211 if (ShuffleVectorInst::isIdentityMask(Mask: NewMask, NumSrcElts)) {
3212 replaceValue(Old&: I, New&: *NewX);
3213 return true;
3214 }
3215
3216 // Try to merge the shuffles if the new shuffle is not costly.
3217 InstructionCost InnerCost0 = 0;
3218 if (Match0)
3219 InnerCost0 = TTI.getInstructionCost(U: cast<User>(Val: OuterV0), CostKind);
3220
3221 InstructionCost InnerCost1 = 0;
3222 if (Match1)
3223 InnerCost1 = TTI.getInstructionCost(U: cast<User>(Val: OuterV1), CostKind);
3224
3225 InstructionCost OuterCost = TTI.getInstructionCost(U: &I, CostKind);
3226
3227 InstructionCost OldCost = InnerCost0 + InnerCost1 + OuterCost;
3228
3229 bool IsUnary = all_of(Range&: NewMask, P: [&](int M) { return M < (int)NumSrcElts; });
3230 TargetTransformInfo::ShuffleKind SK =
3231 IsUnary ? TargetTransformInfo::SK_PermuteSingleSrc
3232 : TargetTransformInfo::SK_PermuteTwoSrc;
3233 InstructionCost NewCost =
3234 TTI.getShuffleCost(Kind: SK, DstTy: ShuffleDstTy, SrcTy: ShuffleSrcTy, CostKind, Mask: NewMask, Index: 0,
3235 SubTp: nullptr, Args: {NewX, NewY});
3236 if (!OuterV0->hasOneUse())
3237 NewCost += InnerCost0;
3238 if (!OuterV1->hasOneUse())
3239 NewCost += InnerCost1;
3240
3241 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two shuffles: " << I
3242 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
3243 << "\n");
3244 if (NewCost > OldCost)
3245 return false;
3246
3247 Value *Shuf = Builder.CreateShuffleVector(V1: NewX, V2: NewY, Mask: NewMask);
3248 replaceValue(Old&: I, New&: *Shuf);
3249 return true;
3250}
3251
3252/// Try to convert a chain of length-preserving shuffles that are fed by
3253/// length-changing shuffles from the same source, e.g. a chain of length 3:
3254///
3255/// "shuffle (shuffle (shuffle x, (shuffle y, undef)),
3256/// (shuffle y, undef)),
3257// (shuffle y, undef)"
3258///
3259/// into a single shuffle fed by a length-changing shuffle:
3260///
3261/// "shuffle x, (shuffle y, undef)"
3262///
3263/// Such chains arise e.g. from folding extract/insert sequences.
3264bool VectorCombine::foldShufflesOfLengthChangingShuffles(Instruction &I) {
3265 FixedVectorType *TrunkType = dyn_cast<FixedVectorType>(Val: I.getType());
3266 if (!TrunkType)
3267 return false;
3268
3269 unsigned ChainLength = 0;
3270 SmallVector<int> Mask;
3271 SmallVector<int> YMask;
3272 InstructionCost OldCost = 0;
3273 InstructionCost NewCost = 0;
3274 Value *Trunk = &I;
3275 unsigned NumTrunkElts = TrunkType->getNumElements();
3276 Value *Y = nullptr;
3277
3278 for (;;) {
3279 // Match the current trunk against (commutations of) the pattern
3280 // "shuffle trunk', (shuffle y, undef)"
3281 ArrayRef<int> OuterMask;
3282 Value *OuterV0, *OuterV1;
3283 if (ChainLength != 0 && !Trunk->hasOneUse())
3284 break;
3285 if (!match(V: Trunk, P: m_Shuffle(v1: m_Value(V&: OuterV0), v2: m_Value(V&: OuterV1),
3286 mask: m_Mask(OuterMask))))
3287 break;
3288 if (OuterV0->getType() != TrunkType) {
3289 // This shuffle is not length-preserving, so it cannot be part of the
3290 // chain.
3291 break;
3292 }
3293
3294 ArrayRef<int> InnerMask0, InnerMask1;
3295 Value *A0, *A1, *B0, *B1;
3296 bool Match0 =
3297 match(V: OuterV0, P: m_Shuffle(v1: m_Value(V&: A0), v2: m_Value(V&: B0), mask: m_Mask(InnerMask0)));
3298 bool Match1 =
3299 match(V: OuterV1, P: m_Shuffle(v1: m_Value(V&: A1), v2: m_Value(V&: B1), mask: m_Mask(InnerMask1)));
3300 bool Match0Leaf = Match0 && A0->getType() != I.getType();
3301 bool Match1Leaf = Match1 && A1->getType() != I.getType();
3302 if (Match0Leaf == Match1Leaf) {
3303 // Only handle the case of exactly one leaf in each step. The "two leaves"
3304 // case is handled by foldShuffleOfShuffles.
3305 break;
3306 }
3307
3308 SmallVector<int> CommutedOuterMask;
3309 if (Match0Leaf) {
3310 std::swap(a&: OuterV0, b&: OuterV1);
3311 std::swap(a&: InnerMask0, b&: InnerMask1);
3312 std::swap(a&: A0, b&: A1);
3313 std::swap(a&: B0, b&: B1);
3314 llvm::append_range(C&: CommutedOuterMask, R&: OuterMask);
3315 for (int &M : CommutedOuterMask) {
3316 if (M == PoisonMaskElem)
3317 continue;
3318 if (M < (int)NumTrunkElts)
3319 M += NumTrunkElts;
3320 else
3321 M -= NumTrunkElts;
3322 }
3323 OuterMask = CommutedOuterMask;
3324 }
3325 if (!OuterV1->hasOneUse())
3326 break;
3327
3328 if (!isa<UndefValue>(Val: A1)) {
3329 if (!Y)
3330 Y = A1;
3331 else if (Y != A1)
3332 break;
3333 }
3334 if (!isa<UndefValue>(Val: B1)) {
3335 if (!Y)
3336 Y = B1;
3337 else if (Y != B1)
3338 break;
3339 }
3340
3341 auto *YType = cast<FixedVectorType>(Val: A1->getType());
3342 int NumLeafElts = YType->getNumElements();
3343 SmallVector<int> LocalYMask(InnerMask1);
3344 for (int &M : LocalYMask) {
3345 if (M >= NumLeafElts)
3346 M -= NumLeafElts;
3347 }
3348
3349 InstructionCost LocalOldCost =
3350 TTI.getInstructionCost(U: cast<User>(Val: Trunk), CostKind) +
3351 TTI.getInstructionCost(U: cast<User>(Val: OuterV1), CostKind);
3352
3353 // Handle the initial (start of chain) case.
3354 if (!ChainLength) {
3355 Mask.assign(AR: OuterMask);
3356 YMask.assign(RHS: LocalYMask);
3357 OldCost = NewCost = LocalOldCost;
3358 Trunk = OuterV0;
3359 ChainLength++;
3360 continue;
3361 }
3362
3363 // For the non-root case, first attempt to combine masks.
3364 SmallVector<int> NewYMask(YMask);
3365 bool Valid = true;
3366 for (auto [CombinedM, LeafM] : llvm::zip(t&: NewYMask, u&: LocalYMask)) {
3367 if (LeafM == -1 || CombinedM == LeafM)
3368 continue;
3369 if (CombinedM == -1) {
3370 CombinedM = LeafM;
3371 } else {
3372 Valid = false;
3373 break;
3374 }
3375 }
3376 if (!Valid)
3377 break;
3378
3379 SmallVector<int> NewMask;
3380 NewMask.reserve(N: NumTrunkElts);
3381 for (int M : Mask) {
3382 if (M < 0 || M >= static_cast<int>(NumTrunkElts))
3383 NewMask.push_back(Elt: M);
3384 else
3385 NewMask.push_back(Elt: OuterMask[M]);
3386 }
3387
3388 // Break the chain if adding this new step complicates the shuffles such
3389 // that it would increase the new cost by more than the old cost of this
3390 // step.
3391 InstructionCost LocalNewCost =
3392 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc, DstTy: TrunkType,
3393 SrcTy: YType, CostKind, Mask: NewYMask) +
3394 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: TrunkType,
3395 SrcTy: TrunkType, CostKind, Mask: NewMask);
3396
3397 if (LocalNewCost >= NewCost && LocalOldCost < LocalNewCost - NewCost)
3398 break;
3399
3400 LLVM_DEBUG({
3401 if (ChainLength == 1) {
3402 dbgs() << "Found chain of shuffles fed by length-changing shuffles: "
3403 << I << '\n';
3404 }
3405 dbgs() << " next chain link: " << *Trunk << '\n'
3406 << " old cost: " << (OldCost + LocalOldCost)
3407 << " new cost: " << LocalNewCost << '\n';
3408 });
3409
3410 Mask = NewMask;
3411 YMask = NewYMask;
3412 OldCost += LocalOldCost;
3413 NewCost = LocalNewCost;
3414 Trunk = OuterV0;
3415 ChainLength++;
3416 }
3417 if (ChainLength <= 1)
3418 return false;
3419
3420 // Bail out if all leaves were poison.
3421 if (!Y)
3422 return false;
3423
3424 if (llvm::all_of(Range&: Mask, P: [&](int M) {
3425 return M < 0 || M >= static_cast<int>(NumTrunkElts);
3426 })) {
3427 // Produce a canonical simplified form if all elements are sourced from Y.
3428 for (int &M : Mask) {
3429 if (M >= static_cast<int>(NumTrunkElts))
3430 M = YMask[M - NumTrunkElts];
3431 }
3432 Value *Root =
3433 Builder.CreateShuffleVector(V1: Y, V2: PoisonValue::get(T: Y->getType()), Mask);
3434 replaceValue(Old&: I, New&: *Root);
3435 return true;
3436 }
3437
3438 Value *Leaf =
3439 Builder.CreateShuffleVector(V1: Y, V2: PoisonValue::get(T: Y->getType()), Mask: YMask);
3440 Value *Root = Builder.CreateShuffleVector(V1: Trunk, V2: Leaf, Mask);
3441 replaceValue(Old&: I, New&: *Root);
3442 return true;
3443}
3444
3445/// Try to convert
3446/// "shuffle (intrinsic), (intrinsic)" into "intrinsic (shuffle), (shuffle)".
3447bool VectorCombine::foldShuffleOfIntrinsics(Instruction &I) {
3448 Value *V0, *V1;
3449 ArrayRef<int> OldMask;
3450 if (!match(V: &I, P: m_Shuffle(v1: m_Value(V&: V0), v2: m_Value(V&: V1), mask: m_Mask(OldMask))))
3451 return false;
3452
3453 auto *II0 = dyn_cast<IntrinsicInst>(Val: V0);
3454 auto *II1 = dyn_cast<IntrinsicInst>(Val: V1);
3455 if (!II0 || !II1)
3456 return false;
3457
3458 Intrinsic::ID IID = II0->getIntrinsicID();
3459 if (IID != II1->getIntrinsicID())
3460 return false;
3461 InstructionCost CostII0 =
3462 TTI.getIntrinsicInstrCost(ICA: IntrinsicCostAttributes(IID, *II0), CostKind);
3463 InstructionCost CostII1 =
3464 TTI.getIntrinsicInstrCost(ICA: IntrinsicCostAttributes(IID, *II1), CostKind);
3465
3466 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3467 auto *II0Ty = dyn_cast<FixedVectorType>(Val: II0->getType());
3468 if (!ShuffleDstTy || !II0Ty)
3469 return false;
3470
3471 if (!isTriviallyVectorizable(ID: IID))
3472 return false;
3473
3474 for (unsigned Idx = 0, E = II0->arg_size(); Idx != E; ++Idx) {
3475 Value *Arg0 = II0->getArgOperand(i: Idx);
3476 Value *Arg1 = II1->getArgOperand(i: Idx);
3477 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: Idx, TTI: &TTI)) {
3478 // Scalar operands must be identical.
3479 if (Arg0 != Arg1)
3480 return false;
3481 } else if (Arg0->getType() != Arg1->getType()) {
3482 // The corresponding vector operands are shuffled together, so they must
3483 // share the same type. For intrinsics overloaded on their operand type
3484 // (e.g. llvm.fptosi.sat), two calls can produce the same result type
3485 // from different operand types; shuffling those would be invalid.
3486 return false;
3487 }
3488 }
3489
3490 InstructionCost OldCost =
3491 CostII0 + CostII1 +
3492 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ShuffleDstTy,
3493 SrcTy: II0Ty, CostKind, Mask: OldMask, Index: 0, SubTp: nullptr, Args: {II0, II1}, CtxI: &I);
3494
3495 SmallVector<Type *> NewArgsTy;
3496 InstructionCost NewCost = 0;
3497 SmallDenseSet<std::pair<Value *, Value *>> SeenOperandPairs;
3498 for (unsigned Idx = 0, E = II0->arg_size(); Idx != E; ++Idx) {
3499 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: Idx, TTI: &TTI)) {
3500 NewArgsTy.push_back(Elt: II0->getArgOperand(i: Idx)->getType());
3501 } else {
3502 auto *VecTy = cast<FixedVectorType>(Val: II0->getArgOperand(i: Idx)->getType());
3503 auto *ArgTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
3504 NumElts: ShuffleDstTy->getNumElements());
3505 NewArgsTy.push_back(Elt: ArgTy);
3506 std::pair<Value *, Value *> OperandPair =
3507 std::make_pair(x: II0->getArgOperand(i: Idx), y: II1->getArgOperand(i: Idx));
3508 if (!SeenOperandPairs.insert(V: OperandPair).second) {
3509 // We've already computed the cost for this operand pair.
3510 continue;
3511 }
3512 NewCost += TTI.getShuffleCost(
3513 Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ArgTy, SrcTy: VecTy, CostKind,
3514 Mask: OldMask, Index: 0, SubTp: nullptr,
3515 Args: {II0->getArgOperand(i: Idx), II1->getArgOperand(i: Idx)});
3516 }
3517 }
3518 IntrinsicCostAttributes NewAttr(IID, ShuffleDstTy, NewArgsTy);
3519
3520 NewCost += TTI.getIntrinsicInstrCost(ICA: NewAttr, CostKind);
3521 if (!II0->hasOneUse())
3522 NewCost += CostII0;
3523 if (II1 != II0 && !II1->hasOneUse())
3524 NewCost += CostII1;
3525
3526 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two intrinsics: " << I
3527 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
3528 << "\n");
3529
3530 if (NewCost > OldCost)
3531 return false;
3532
3533 SmallVector<Value *> NewArgs;
3534 SmallDenseMap<std::pair<Value *, Value *>, Value *> ShuffleCache;
3535 for (unsigned Idx = 0, E = II0->arg_size(); Idx != E; ++Idx) {
3536 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: Idx, TTI: &TTI)) {
3537 NewArgs.push_back(Elt: II0->getArgOperand(i: Idx));
3538 } else {
3539 std::pair<Value *, Value *> OperandPair =
3540 std::make_pair(x: II0->getArgOperand(i: Idx), y: II1->getArgOperand(i: Idx));
3541 auto It = ShuffleCache.find(Val: OperandPair);
3542 if (It != ShuffleCache.end()) {
3543 // Reuse previously created shuffle for this operand pair.
3544 NewArgs.push_back(Elt: It->second);
3545 continue;
3546 }
3547 Value *Shuf = Builder.CreateShuffleVector(
3548 V1: II0->getArgOperand(i: Idx), V2: II1->getArgOperand(i: Idx), Mask: OldMask);
3549 ShuffleCache[OperandPair] = Shuf;
3550 NewArgs.push_back(Elt: Shuf);
3551 }
3552 }
3553 Value *NewIntrinsic = Builder.CreateIntrinsic(RetTy: ShuffleDstTy, ID: IID, Args: NewArgs);
3554
3555 // Intersect flags from the old intrinsics.
3556 if (auto *NewInst = dyn_cast<Instruction>(Val: NewIntrinsic)) {
3557 NewInst->copyIRFlags(V: II0);
3558 NewInst->andIRFlags(V: II1);
3559 }
3560
3561 replaceValue(Old&: I, New&: *NewIntrinsic);
3562 return true;
3563}
3564
3565/// Try to convert
3566/// "shuffle (intrinsic), (poison/undef)" into "intrinsic (shuffle)".
3567bool VectorCombine::foldPermuteOfIntrinsic(Instruction &I) {
3568 Value *V0;
3569 ArrayRef<int> Mask;
3570 if (!match(V: &I, P: m_Shuffle(v1: m_Value(V&: V0), v2: m_Undef(), mask: m_Mask(Mask))))
3571 return false;
3572
3573 auto *II0 = dyn_cast<IntrinsicInst>(Val: V0);
3574 if (!II0)
3575 return false;
3576
3577 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3578 auto *IntrinsicSrcTy = dyn_cast<FixedVectorType>(Val: II0->getType());
3579 if (!ShuffleDstTy || !IntrinsicSrcTy)
3580 return false;
3581
3582 // Validate it's a pure permute, mask should only reference the first vector
3583 unsigned NumSrcElts = IntrinsicSrcTy->getNumElements();
3584 if (any_of(Range&: Mask, P: [NumSrcElts](int M) { return M >= (int)NumSrcElts; }))
3585 return false;
3586
3587 Intrinsic::ID IID = II0->getIntrinsicID();
3588 if (!isTriviallyVectorizable(ID: IID))
3589 return false;
3590
3591 // Cost analysis
3592 InstructionCost IntrinsicCost =
3593 TTI.getIntrinsicInstrCost(ICA: IntrinsicCostAttributes(IID, *II0), CostKind);
3594 InstructionCost OldCost =
3595 IntrinsicCost +
3596 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc, DstTy: ShuffleDstTy,
3597 SrcTy: IntrinsicSrcTy, CostKind, Mask, Index: 0, SubTp: nullptr, Args: {V0}, CtxI: &I);
3598
3599 SmallVector<Type *> NewArgsTy;
3600 InstructionCost NewCost = 0;
3601 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I) {
3602 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3603 NewArgsTy.push_back(Elt: II0->getArgOperand(i: I)->getType());
3604 } else {
3605 auto *VecTy = cast<FixedVectorType>(Val: II0->getArgOperand(i: I)->getType());
3606 auto *ArgTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
3607 NumElts: ShuffleDstTy->getNumElements());
3608 NewArgsTy.push_back(Elt: ArgTy);
3609 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
3610 DstTy: ArgTy, SrcTy: VecTy, CostKind, Mask, Index: 0, SubTp: nullptr,
3611 Args: {II0->getArgOperand(i: I)});
3612 }
3613 }
3614 IntrinsicCostAttributes NewAttr(IID, ShuffleDstTy, NewArgsTy);
3615 NewCost += TTI.getIntrinsicInstrCost(ICA: NewAttr, CostKind);
3616
3617 // If the intrinsic has multiple uses, we need to account for the cost of
3618 // keeping the original intrinsic around.
3619 if (!II0->hasOneUse())
3620 NewCost += IntrinsicCost;
3621
3622 LLVM_DEBUG(dbgs() << "Found a permute of intrinsic: " << I << "\n OldCost: "
3623 << OldCost << " vs NewCost: " << NewCost << "\n");
3624
3625 if (NewCost > OldCost)
3626 return false;
3627
3628 // Transform
3629 SmallVector<Value *> NewArgs;
3630 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I) {
3631 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3632 NewArgs.push_back(Elt: II0->getArgOperand(i: I));
3633 } else {
3634 Value *Shuf = Builder.CreateShuffleVector(V: II0->getArgOperand(i: I), Mask);
3635 NewArgs.push_back(Elt: Shuf);
3636 }
3637 }
3638
3639 Value *NewIntrinsic = Builder.CreateIntrinsic(RetTy: ShuffleDstTy, ID: IID, Args: NewArgs);
3640
3641 if (auto *NewInst = dyn_cast<Instruction>(Val: NewIntrinsic))
3642 NewInst->copyIRFlags(V: II0);
3643
3644 replaceValue(Old&: I, New&: *NewIntrinsic);
3645 return true;
3646}
3647
3648using InstLane = std::pair<Value *, int>;
3649
3650static InstLane lookThroughShuffles(Value *V, int Lane) {
3651 while (auto *SV = dyn_cast<ShuffleVectorInst>(Val: V)) {
3652 unsigned NumElts =
3653 cast<FixedVectorType>(Val: SV->getOperand(i_nocapture: 0)->getType())->getNumElements();
3654 int M = SV->getMaskValue(Elt: Lane);
3655 if (M < 0)
3656 return {nullptr, PoisonMaskElem};
3657 if (static_cast<unsigned>(M) < NumElts) {
3658 V = SV->getOperand(i_nocapture: 0);
3659 Lane = M;
3660 } else {
3661 V = SV->getOperand(i_nocapture: 1);
3662 Lane = M - NumElts;
3663 }
3664 }
3665 return InstLane{V, Lane};
3666}
3667
3668static SmallVector<InstLane>
3669generateInstLaneVectorFromOperand(ArrayRef<InstLane> Item, int Op) {
3670 SmallVector<InstLane> NItem;
3671 for (InstLane IL : Item) {
3672 auto [U, Lane] = IL;
3673 InstLane OpLane =
3674 U ? lookThroughShuffles(V: cast<Instruction>(Val: U)->getOperand(i: Op), Lane)
3675 : InstLane{nullptr, PoisonMaskElem};
3676 NItem.emplace_back(Args&: OpLane);
3677 }
3678 return NItem;
3679}
3680
3681/// Detect concat of multiple values into a vector
3682static bool isFreeConcat(ArrayRef<InstLane> Item, TTI::TargetCostKind CostKind,
3683 const TargetTransformInfo &TTI) {
3684 auto *Ty = cast<FixedVectorType>(Val: Item.front().first->getType());
3685 unsigned NumElts = Ty->getNumElements();
3686 if (Item.size() == NumElts || NumElts == 1 || Item.size() % NumElts != 0)
3687 return false;
3688
3689 // Check that the concat is free, usually meaning that the type will be split
3690 // during legalization.
3691 SmallVector<int, 16> ConcatMask(NumElts * 2);
3692 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
3693 if (TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc,
3694 DstTy: FixedVectorType::get(ElementType: Ty->getScalarType(), NumElts: NumElts * 2),
3695 SrcTy: Ty, CostKind, Mask: ConcatMask) != 0)
3696 return false;
3697
3698 unsigned NumSlices = Item.size() / NumElts;
3699 // Currently we generate a tree of shuffles for the concats, which limits us
3700 // to a power2.
3701 if (!isPowerOf2_32(Value: NumSlices))
3702 return false;
3703 for (unsigned Slice = 0; Slice < NumSlices; ++Slice) {
3704 Value *SliceV = Item[Slice * NumElts].first;
3705 if (!SliceV || SliceV->getType() != Ty)
3706 return false;
3707 for (unsigned Elt = 0; Elt < NumElts; ++Elt) {
3708 auto [V, Lane] = Item[Slice * NumElts + Elt];
3709 if (Lane != static_cast<int>(Elt) || SliceV != V)
3710 return false;
3711 }
3712 }
3713 return true;
3714}
3715
3716static Value *
3717generateNewInstTree(ArrayRef<InstLane> Item, Use *From,
3718 const DenseSet<std::pair<Value *, Use *>> &IdentityLeafs,
3719 const DenseSet<std::pair<Value *, Use *>> &SplatLeafs,
3720 const DenseSet<std::pair<Value *, Use *>> &ConcatLeafs,
3721 IRBuilderBase &Builder, InstructionWorklist &WorkList,
3722 const TargetTransformInfo *TTI) {
3723 auto [FrontV, FrontLane] = Item.front();
3724
3725 if (IdentityLeafs.contains(V: std::make_pair(x&: FrontV, y&: From))) {
3726 return FrontV;
3727 }
3728 if (SplatLeafs.contains(V: std::make_pair(x&: FrontV, y&: From))) {
3729 SmallVector<int, 16> Mask(Item.size(), FrontLane);
3730 return Builder.CreateShuffleVector(V: FrontV, Mask);
3731 }
3732 if (ConcatLeafs.contains(V: std::make_pair(x&: FrontV, y&: From))) {
3733 unsigned NumElts =
3734 cast<FixedVectorType>(Val: FrontV->getType())->getNumElements();
3735 SmallVector<Value *> Values(Item.size() / NumElts, nullptr);
3736 for (unsigned S = 0; S < Values.size(); ++S)
3737 Values[S] = Item[S * NumElts].first;
3738
3739 while (Values.size() > 1) {
3740 NumElts *= 2;
3741 SmallVector<int, 16> Mask(NumElts, 0);
3742 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
3743 SmallVector<Value *> NewValues(Values.size() / 2, nullptr);
3744 for (unsigned S = 0; S < NewValues.size(); ++S)
3745 NewValues[S] =
3746 Builder.CreateShuffleVector(V1: Values[S * 2], V2: Values[S * 2 + 1], Mask);
3747 Values = NewValues;
3748 }
3749 return Values[0];
3750 }
3751
3752 auto *I = cast<Instruction>(Val: FrontV);
3753
3754 // Handle vector bitcasts that change element count. We cannot use
3755 // generateInstLaneVectorFromOperand for these because the lane indices
3756 // don't map 1:1 through the bitcast.
3757 if (auto *BitCast = dyn_cast<BitCastInst>(Val: I)) {
3758 auto *BCDstTy = dyn_cast<FixedVectorType>(Val: BitCast->getDestTy());
3759 auto *BCSrcTy = dyn_cast<FixedVectorType>(Val: BitCast->getSrcTy());
3760 if (BCDstTy && BCSrcTy &&
3761 BCDstTy->getElementCount() != BCSrcTy->getElementCount()) {
3762 unsigned DstElts = BCDstTy->getNumElements();
3763 unsigned SrcElts = BCSrcTy->getNumElements();
3764 SmallVector<InstLane> NewItem;
3765 if (DstElts > SrcElts) {
3766 // Widening: compress operand Item.
3767 unsigned R = DstElts / SrcElts;
3768 if (Item.size() % R != 0)
3769 return nullptr;
3770 for (unsigned Idx = 0, E = Item.size(); Idx < E; Idx += R) {
3771 auto [V, Lane] = Item[Idx];
3772 if (!V) {
3773 NewItem.push_back(Elt: {nullptr, PoisonMaskElem});
3774 continue;
3775 }
3776 NewItem.push_back(
3777 Elt: lookThroughShuffles(V: cast<Operator>(Val: V)->getOperand(i: 0), Lane: Lane / R));
3778 }
3779 } else {
3780 // Narrowing: expand operand Item.
3781 unsigned R = SrcElts / DstElts;
3782 for (auto [V, Lane] : Item) {
3783 if (!V) {
3784 NewItem.append(NumInputs: R, Elt: {nullptr, PoisonMaskElem});
3785 continue;
3786 }
3787 Value *Op = cast<Operator>(Val: V)->getOperand(i: 0);
3788 for (unsigned J = 0; J < R; ++J)
3789 NewItem.push_back(Elt: lookThroughShuffles(V: Op, Lane: Lane * R + J));
3790 }
3791 }
3792 Value *Op = generateNewInstTree(Item: NewItem, From: &BitCast->getOperandUse(i: 0),
3793 IdentityLeafs, SplatLeafs, ConcatLeafs,
3794 Builder, WorkList, TTI);
3795 return Builder.CreateBitCast(
3796 V: Op, DestTy: FixedVectorType::get(ElementType: BCDstTy->getScalarType(), NumElts: Item.size()));
3797 }
3798 }
3799 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3800 unsigned NumOps = I->getNumOperands() - (II ? 1 : 0);
3801 SmallVector<Value *> Ops(NumOps);
3802 for (unsigned Idx = 0; Idx < NumOps; Idx++) {
3803 if (II &&
3804 isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(), ScalarOpdIdx: Idx, TTI)) {
3805 Ops[Idx] = II->getOperand(i_nocapture: Idx);
3806 continue;
3807 }
3808 Ops[Idx] = generateNewInstTree(
3809 Item: generateInstLaneVectorFromOperand(Item, Op: Idx), From: &I->getOperandUse(i: Idx),
3810 IdentityLeafs, SplatLeafs, ConcatLeafs, Builder, WorkList, TTI);
3811 // Don't re-queue the operand of a bitcast we just regenerated. Doing so
3812 // lets foldBitcastShuffle sink the bitcast back into a shuffle(bitcast),
3813 // which foldShuffleToIdentity then re-matches as the same superfluous
3814 // identity - an infinite loop between the two folds.
3815 // TODO: we should fix algorithms in foldBitcastShuffle and
3816 // foldShuffleToIdentity instead of workaround the problem in worklist.
3817 if (isa<BitCastInst>(Val: I) && isa<Instruction>(Val: Ops[Idx]))
3818 WorkList.remove(I: cast<Instruction>(Val: Ops[Idx]));
3819 }
3820
3821 SmallVector<Value *, 8> ValueList;
3822 for (const auto &Lane : Item)
3823 if (Lane.first)
3824 ValueList.push_back(Elt: Lane.first);
3825
3826 Type *DstTy =
3827 FixedVectorType::get(ElementType: I->getType()->getScalarType(), NumElts: Item.size());
3828 if (auto *BI = dyn_cast<BinaryOperator>(Val: I)) {
3829 auto *Value = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)BI->getOpcode(),
3830 LHS: Ops[0], RHS: Ops[1]);
3831 propagateIRFlags(I: Value, VL: ValueList);
3832 return Value;
3833 }
3834 if (auto *CI = dyn_cast<CmpInst>(Val: I)) {
3835 auto *Value = Builder.CreateCmp(Pred: CI->getPredicate(), LHS: Ops[0], RHS: Ops[1]);
3836 propagateIRFlags(I: Value, VL: ValueList);
3837 return Value;
3838 }
3839 if (auto *SI = dyn_cast<SelectInst>(Val: I)) {
3840 auto *Value = Builder.CreateSelect(C: Ops[0], True: Ops[1], False: Ops[2], Name: "", MDFrom: SI);
3841 propagateIRFlags(I: Value, VL: ValueList);
3842 return Value;
3843 }
3844 if (auto *CI = dyn_cast<CastInst>(Val: I)) {
3845 auto *Value = Builder.CreateCast(Op: CI->getOpcode(), V: Ops[0], DestTy: DstTy);
3846 propagateIRFlags(I: Value, VL: ValueList);
3847 return Value;
3848 }
3849 if (II) {
3850 auto *Value = Builder.CreateIntrinsic(RetTy: DstTy, ID: II->getIntrinsicID(), Args: Ops);
3851 propagateIRFlags(I: Value, VL: ValueList);
3852 return Value;
3853 }
3854 assert(isa<UnaryInstruction>(I) && "Unexpected instruction type in Generate");
3855 auto *Value =
3856 Builder.CreateUnOp(Opc: (Instruction::UnaryOps)I->getOpcode(), V: Ops[0]);
3857 propagateIRFlags(I: Value, VL: ValueList);
3858 return Value;
3859}
3860
3861// Starting from a shuffle, look up through operands tracking the shuffled index
3862// of each lane. If we can simplify away the shuffles to identities then
3863// do so.
3864bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
3865 auto *Ty = dyn_cast<FixedVectorType>(Val: I.getType());
3866 if (!Ty || I.use_empty())
3867 return false;
3868
3869 SmallVector<InstLane> Start(Ty->getNumElements());
3870 for (unsigned M = 0, E = Ty->getNumElements(); M < E; ++M)
3871 Start[M] = lookThroughShuffles(V: &I, Lane: M);
3872
3873 SmallVector<std::pair<SmallVector<InstLane>, Use *>> Candidates;
3874 Candidates.push_back(Elt: std::make_pair(x&: Start, y: &*I.use_begin()));
3875 DenseSet<std::pair<Value *, Use *>> IdentityLeafs, SplatLeafs, ConcatLeafs;
3876 unsigned NumVisited = 0;
3877 bool TraversedElCountChangingBitcast = false;
3878
3879 while (!Candidates.empty()) {
3880 if (++NumVisited > Opts.vector_combine_max_scan_instrs)
3881 return false;
3882
3883 auto ItemFrom = Candidates.pop_back_val();
3884 auto Item = ItemFrom.first;
3885 auto From = ItemFrom.second;
3886 auto [FrontV, FrontLane] = Item.front();
3887
3888 // If we found an undef first lane then bail out to keep things simple.
3889 if (!FrontV)
3890 return false;
3891
3892 // Look for an identity value.
3893 if (FrontLane == 0 &&
3894 cast<FixedVectorType>(Val: FrontV->getType())->getNumElements() ==
3895 Item.size() &&
3896 all_of(Range: drop_begin(RangeOrContainer: enumerate(First&: Item)), P: [Item](const auto &E) {
3897 Value *FrontV = Item.front().first;
3898 return !E.value().first || (isEquivBitcast(E.value().first, FrontV) &&
3899 E.value().second == (int)E.index());
3900 })) {
3901 IdentityLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3902 continue;
3903 }
3904 // Look for constants, for the moment only supporting constant splats.
3905 if (auto *C = dyn_cast<Constant>(Val: FrontV);
3906 C && C->getSplatValue() &&
3907 all_of(Range: drop_begin(RangeOrContainer&: Item), P: [Item](InstLane &IL) {
3908 Value *FrontV = Item.front().first;
3909 Value *V = IL.first;
3910 return !V || (isa<Constant>(Val: V) &&
3911 cast<Constant>(Val: V)->getSplatValue() ==
3912 cast<Constant>(Val: FrontV)->getSplatValue());
3913 })) {
3914 SplatLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3915 continue;
3916 }
3917 // Look for a splat value.
3918 if (all_of(Range: drop_begin(RangeOrContainer&: Item), P: [Item](InstLane &IL) {
3919 auto [FrontV, FrontLane] = Item.front();
3920 auto [V, Lane] = IL;
3921 return !V || (V == FrontV && Lane == FrontLane);
3922 })) {
3923 SplatLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3924 continue;
3925 }
3926
3927 // We need each element to be the same type of value, and check that each
3928 // element has a single use.
3929 auto CheckLaneIsEquivalentToFirst = [Item](InstLane IL) {
3930 Value *FrontV = Item.front().first;
3931 if (!IL.first)
3932 return true;
3933 Value *V = IL.first;
3934 if (auto *I = dyn_cast<Instruction>(Val: V); I && !I->hasOneUser())
3935 return false;
3936 if (V->getValueID() != FrontV->getValueID())
3937 return false;
3938 if (auto *CI = dyn_cast<CmpInst>(Val: V))
3939 if (CI->getPredicate() != cast<CmpInst>(Val: FrontV)->getPredicate())
3940 return false;
3941 if (auto *CI = dyn_cast<CastInst>(Val: V))
3942 if (CI->getSrcTy()->getScalarType() !=
3943 cast<CastInst>(Val: FrontV)->getSrcTy()->getScalarType())
3944 return false;
3945 if (auto *SI = dyn_cast<SelectInst>(Val: V))
3946 if (!isa<VectorType>(Val: SI->getOperand(i_nocapture: 0)->getType()) ||
3947 SI->getOperand(i_nocapture: 0)->getType() !=
3948 cast<SelectInst>(Val: FrontV)->getOperand(i_nocapture: 0)->getType())
3949 return false;
3950 if (isa<CallInst>(Val: V) && !isa<IntrinsicInst>(Val: V))
3951 return false;
3952 auto *II = dyn_cast<IntrinsicInst>(Val: V);
3953 return !II || (isa<IntrinsicInst>(Val: FrontV) &&
3954 II->getIntrinsicID() ==
3955 cast<IntrinsicInst>(Val: FrontV)->getIntrinsicID() &&
3956 !II->hasOperandBundles());
3957 };
3958 if (all_of(Range: drop_begin(RangeOrContainer&: Item), P: CheckLaneIsEquivalentToFirst)) {
3959 // Check the operator is one that we support.
3960 if (isa<BinaryOperator, CmpInst>(Val: FrontV)) {
3961 // We exclude div/rem in case they hit UB from poison lanes.
3962 if (auto *BO = dyn_cast<BinaryOperator>(Val: FrontV);
3963 BO && BO->isIntDivRem())
3964 return false;
3965 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3966 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: 0));
3967 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 1),
3968 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: 1));
3969 continue;
3970 } else if (isa<UnaryOperator, TruncInst, ZExtInst, SExtInst, FPToSIInst,
3971 FPToUIInst, SIToFPInst, UIToFPInst>(Val: FrontV)) {
3972 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3973 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: 0));
3974 continue;
3975 } else if (auto *BitCast = dyn_cast<BitCastInst>(Val: FrontV)) {
3976 auto *BCDstTy = dyn_cast<FixedVectorType>(Val: BitCast->getDestTy());
3977 auto *BCSrcTy = dyn_cast<FixedVectorType>(Val: BitCast->getSrcTy());
3978 if (BCDstTy && BCSrcTy) {
3979 ElementCount DstEC = BCDstTy->getElementCount();
3980 ElementCount SrcEC = BCSrcTy->getElementCount();
3981 if (DstEC == SrcEC) {
3982 // Same element count - simple pass-through.
3983 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3984 Args: &BitCast->getOperandUse(i: 0));
3985 continue;
3986 }
3987 unsigned DstElts = DstEC.getFixedValue();
3988 unsigned SrcElts = SrcEC.getFixedValue();
3989 if (DstElts > SrcElts && DstElts % SrcElts == 0) {
3990 // Widening bitcast (e.g. <2 x i32> -> <4 x i16>). Compress
3991 // consecutive groups of R destination lanes into one source
3992 // lane.
3993 unsigned R = DstElts / SrcElts;
3994 SmallVector<InstLane> NItem;
3995 bool Valid = Item.size() % R == 0;
3996 for (unsigned Idx = 0, E = Item.size(); Valid && Idx < E;
3997 Idx += R) {
3998 auto [V0, L0] = Item[Idx];
3999 if (!V0) {
4000 if (any_of(Range: ArrayRef(Item).slice(N: Idx + 1, M: R - 1),
4001 P: [](InstLane IL) { return IL.first != nullptr; })) {
4002 Valid = false;
4003 break;
4004 }
4005 NItem.push_back(Elt: {nullptr, PoisonMaskElem});
4006 continue;
4007 }
4008 if (L0 % R != 0) {
4009 Valid = false;
4010 break;
4011 }
4012 for (unsigned J = 1; J < R; ++J) {
4013 auto [VJ, LJ] = Item[Idx + J];
4014 if (!VJ || VJ != V0 || LJ != L0 + (int)J) {
4015 Valid = false;
4016 break;
4017 }
4018 }
4019 if (!Valid)
4020 break;
4021 NItem.push_back(Elt: lookThroughShuffles(
4022 V: cast<Operator>(Val: V0)->getOperand(i: 0), Lane: L0 / R));
4023 }
4024 if (Valid) {
4025 TraversedElCountChangingBitcast = true;
4026 Candidates.emplace_back(Args&: NItem, Args: &BitCast->getOperandUse(i: 0));
4027 continue;
4028 }
4029 } else if (SrcElts > DstElts && SrcElts % DstElts == 0) {
4030 // Narrowing bitcast (e.g. <4 x i16> -> <2 x i32>). Expand
4031 // each destination lane into R source lanes.
4032 unsigned R = SrcElts / DstElts;
4033 SmallVector<InstLane> NItem;
4034 for (auto [V, Lane] : Item) {
4035 if (!V) {
4036 NItem.append(NumInputs: R, Elt: {nullptr, PoisonMaskElem});
4037 continue;
4038 }
4039 Value *Op = cast<Operator>(Val: V)->getOperand(i: 0);
4040 for (unsigned J = 0; J < R; ++J)
4041 NItem.push_back(Elt: lookThroughShuffles(V: Op, Lane: Lane * R + J));
4042 }
4043 TraversedElCountChangingBitcast = true;
4044 Candidates.emplace_back(Args&: NItem, Args: &BitCast->getOperandUse(i: 0));
4045 continue;
4046 }
4047 }
4048 } else if (auto *Sel = dyn_cast<SelectInst>(Val: FrontV)) {
4049 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
4050 Args: &Sel->getOperandUse(i: 0));
4051 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 1),
4052 Args: &Sel->getOperandUse(i: 1));
4053 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 2),
4054 Args: &Sel->getOperandUse(i: 2));
4055 continue;
4056 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: FrontV);
4057 II && isTriviallyVectorizable(ID: II->getIntrinsicID()) &&
4058 !II->hasOperandBundles()) {
4059 for (unsigned Op = 0, E = II->getNumOperands() - 1; Op < E; Op++) {
4060 if (isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(), ScalarOpdIdx: Op,
4061 TTI: &TTI)) {
4062 if (!all_of(Range: drop_begin(RangeOrContainer&: Item), P: [Item, Op](InstLane &IL) {
4063 Value *FrontV = Item.front().first;
4064 Value *V = IL.first;
4065 return !V || (cast<Instruction>(Val: V)->getOperand(i: Op) ==
4066 cast<Instruction>(Val: FrontV)->getOperand(i: Op));
4067 }))
4068 return false;
4069 continue;
4070 }
4071 Candidates.emplace_back(
4072 Args: generateInstLaneVectorFromOperand(Item, Op),
4073 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: Op));
4074 }
4075 continue;
4076 }
4077 }
4078
4079 if (isFreeConcat(Item, CostKind, TTI)) {
4080 ConcatLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
4081 continue;
4082 }
4083
4084 return false;
4085 }
4086
4087 if (NumVisited <= 1)
4088 return false;
4089
4090 // If the only non-leaf node traversed was a single bitcast that changes
4091 // element count, the fold would just commute the bitcast and shuffle.
4092 // foldBitcastShuffle does the reverse transform, causing an infinite loop.
4093 if (NumVisited == 2 && TraversedElCountChangingBitcast)
4094 return false;
4095
4096 LLVM_DEBUG(dbgs() << "Found a superfluous identity shuffle: " << I << "\n");
4097
4098 // If we got this far, we know the shuffles are superfluous and can be
4099 // removed. Scan through again and generate the new tree of instructions.
4100 Builder.SetInsertPoint(&I);
4101 Value *V =
4102 generateNewInstTree(Item: Start, From: &*I.use_begin(), IdentityLeafs, SplatLeafs,
4103 ConcatLeafs, Builder, WorkList&: Worklist, TTI: &TTI);
4104 replaceValue(Old&: I, New&: *V);
4105 return true;
4106}
4107
4108/// Given a commutative reduction, the order of the input lanes does not alter
4109/// the results. We can use this to remove certain shuffles feeding the
4110/// reduction, removing the need to shuffle at all.
4111bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
4112 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
4113 if (!II)
4114 return false;
4115 switch (II->getIntrinsicID()) {
4116 case Intrinsic::vector_reduce_add:
4117 case Intrinsic::vector_reduce_mul:
4118 case Intrinsic::vector_reduce_and:
4119 case Intrinsic::vector_reduce_or:
4120 case Intrinsic::vector_reduce_xor:
4121 case Intrinsic::vector_reduce_smin:
4122 case Intrinsic::vector_reduce_smax:
4123 case Intrinsic::vector_reduce_umin:
4124 case Intrinsic::vector_reduce_umax:
4125 break;
4126 default:
4127 return false;
4128 }
4129
4130 // Find all the inputs when looking through operations that do not alter the
4131 // lane order (binops, for example). Currently we look for a single shuffle,
4132 // and can ignore splat values.
4133 std::queue<Value *> Worklist;
4134 SmallPtrSet<Value *, 4> Visited;
4135 ShuffleVectorInst *Shuffle = nullptr;
4136 if (auto *Op = dyn_cast<Instruction>(Val: I.getOperand(i: 0)))
4137 Worklist.push(x: Op);
4138
4139 while (!Worklist.empty()) {
4140 Value *CV = Worklist.front();
4141 Worklist.pop();
4142 if (Visited.contains(Ptr: CV))
4143 continue;
4144
4145 // Splats don't change the order, so can be safely ignored.
4146 if (isSplatValue(V: CV))
4147 continue;
4148
4149 Visited.insert(Ptr: CV);
4150
4151 if (auto *CI = dyn_cast<Instruction>(Val: CV)) {
4152 if (CI->isBinaryOp()) {
4153 for (auto *Op : CI->operand_values())
4154 Worklist.push(x: Op);
4155 continue;
4156 } else if (auto *SV = dyn_cast<ShuffleVectorInst>(Val: CI)) {
4157 if (Shuffle && Shuffle != SV)
4158 return false;
4159 Shuffle = SV;
4160 continue;
4161 }
4162 }
4163
4164 // Anything else is currently an unknown node.
4165 return false;
4166 }
4167
4168 if (!Shuffle)
4169 return false;
4170
4171 // Check all uses of the binary ops and shuffles are also included in the
4172 // lane-invariant operations (Visited should be the list of lanewise
4173 // instructions, including the shuffle that we found).
4174 for (auto *V : Visited)
4175 for (auto *U : V->users())
4176 if (!Visited.contains(Ptr: U) && U != &I)
4177 return false;
4178
4179 FixedVectorType *VecType =
4180 dyn_cast<FixedVectorType>(Val: II->getOperand(i_nocapture: 0)->getType());
4181 if (!VecType)
4182 return false;
4183 FixedVectorType *ShuffleInputType =
4184 dyn_cast<FixedVectorType>(Val: Shuffle->getOperand(i_nocapture: 0)->getType());
4185 if (!ShuffleInputType)
4186 return false;
4187 unsigned NumInputElts = ShuffleInputType->getNumElements();
4188
4189 // Find the mask from sorting the lanes into order. This is most likely to
4190 // become a identity or concat mask. Undef elements are pushed to the end.
4191 SmallVector<int> ConcatMask;
4192 Shuffle->getShuffleMask(Result&: ConcatMask);
4193 sort(C&: ConcatMask, Comp: [](int X, int Y) { return (unsigned)X < (unsigned)Y; });
4194 bool UsesSecondVec =
4195 any_of(Range&: ConcatMask, P: [&](int M) { return M >= (int)NumInputElts; });
4196
4197 InstructionCost OldCost = TTI.getShuffleCost(
4198 Kind: UsesSecondVec ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc, DstTy: VecType,
4199 SrcTy: ShuffleInputType, CostKind, Mask: Shuffle->getShuffleMask());
4200 InstructionCost NewCost = TTI.getShuffleCost(
4201 Kind: UsesSecondVec ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc, DstTy: VecType,
4202 SrcTy: ShuffleInputType, CostKind, Mask: ConcatMask);
4203
4204 LLVM_DEBUG(dbgs() << "Found a reduction feeding from a shuffle: " << *Shuffle
4205 << "\n");
4206 LLVM_DEBUG(dbgs() << " OldCost: " << OldCost << " vs NewCost: " << NewCost
4207 << "\n");
4208 bool MadeChanges = false;
4209 if (NewCost < OldCost) {
4210 Builder.SetInsertPoint(Shuffle);
4211 Value *NewShuffle = Builder.CreateShuffleVector(
4212 V1: Shuffle->getOperand(i_nocapture: 0), V2: Shuffle->getOperand(i_nocapture: 1), Mask: ConcatMask);
4213 LLVM_DEBUG(dbgs() << "Created new shuffle: " << *NewShuffle << "\n");
4214 replaceValue(Old&: *Shuffle, New&: *NewShuffle);
4215 return true;
4216 }
4217
4218 // See if we can re-use foldSelectShuffle, getting it to reduce the size of
4219 // the shuffle into a nicer order, as it can ignore the order of the shuffles.
4220 MadeChanges |= foldSelectShuffle(I&: *Shuffle, FromReduction: true);
4221 return MadeChanges;
4222}
4223
4224/// Try to fold a chain of shuffles and ops feeding extractelement(..., 0)
4225/// into llvm.vector.reduce.*, by tracking which lanes contribute to the
4226/// extracted lane and reducing the widest vector whose lanes each contribute
4227/// once.
4228///
4229/// For example:
4230///
4231/// %lo = shufflevector <4 x i32> %a, poison, <2 x i32> <i32 0, i32 1>
4232/// %hi = shufflevector <4 x i32> %a, poison, <2 x i32> <i32 2, i32 3>
4233/// %s = add <2 x i32> %lo, %hi
4234/// %sh = shufflevector <2 x i32> %s, poison, <2 x i32> <i32 1, i32 poison>
4235/// %r = add <2 x i32> %s, %sh
4236/// %e = extractelement <2 x i32> %r, i64 0
4237///
4238/// transforms to:
4239///
4240/// %e = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
4241bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
4242 Value *VecOpEE;
4243 if (!match(V: &I, P: m_ExtractElt(Val: m_Value(V&: VecOpEE), Idx: m_Zero())))
4244 return false;
4245
4246 auto *FVT = dyn_cast<FixedVectorType>(Val: VecOpEE->getType());
4247 if (!FVT)
4248 return false;
4249
4250 if (FVT->getNumElements() < 2)
4251 return false;
4252
4253 std::optional<Instruction::BinaryOps> CommonBinOp;
4254 std::optional<Intrinsic::ID> CommonCallOp;
4255
4256 if (auto *BO = dyn_cast<BinaryOperator>(Val: VecOpEE)) {
4257 if (!getReductionForBinop(Opc: BO->getOpcode()))
4258 return false;
4259 CommonBinOp = BO->getOpcode();
4260 } else if (auto *MMI = dyn_cast<MinMaxIntrinsic>(Val: VecOpEE)) {
4261 CommonCallOp = MMI->getIntrinsicID();
4262 } else {
4263 return false;
4264 }
4265
4266 // For floating-point reductions, track FMF intersection across all binops.
4267 FastMathFlags CommonFMF;
4268 bool IsFloatReduction = false;
4269
4270 // A chain node is one we walk through, either a matching-opcode binop/min-max
4271 // or a single-source shuffle. Anything else is a leaf source.
4272 auto IsChainNode = [&](Value *V) {
4273 if (auto *BO = dyn_cast<BinaryOperator>(Val: V))
4274 return CommonBinOp && BO->getOpcode() == *CommonBinOp;
4275 if (auto *MMI = dyn_cast<MinMaxIntrinsic>(Val: V))
4276 return CommonCallOp && MMI->getIntrinsicID() == *CommonCallOp;
4277 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V))
4278 return isa<PoisonValue>(Val: SVI->getOperand(i_nocapture: 1));
4279 return false;
4280 };
4281
4282 // Collect the chain, building Nodes in postorder. Bail if the chain is empty
4283 // or exceeds MaxChainNodes.
4284 constexpr unsigned MaxChainNodes = 32;
4285 SmallSetVector<Value *, 16> Nodes;
4286 SmallSetVector<Value *, 4> Sources;
4287 unsigned NumVisited = 0;
4288 auto AddSource = [&](Value *V) {
4289 if (!isa<FixedVectorType>(Val: V->getType()))
4290 return false;
4291 Sources.insert(X: V);
4292 return true;
4293 };
4294 auto Walk = [&](Value *V, auto &&Walk) -> bool {
4295 if (Nodes.contains(key: V) || Sources.contains(key: V))
4296 return true;
4297 if (++NumVisited > MaxChainNodes)
4298 return false;
4299 if (!IsChainNode(V))
4300 return AddSource(V);
4301 // Chain shuffles always have poison as op1, so only op0 matters.
4302 auto *U = cast<Instruction>(Val: V);
4303 unsigned NumOps = isa<ShuffleVectorInst>(Val: U) ? 1 : 2;
4304 for (unsigned I = 0; I != NumOps; ++I)
4305 if (!Walk(U->getOperand(i: I), Walk))
4306 return false;
4307 if (isa<ShuffleVectorInst>(Val: U) || Nodes.contains(key: U->getOperand(i: 0)) ||
4308 Nodes.contains(key: U->getOperand(i: 1))) {
4309 Nodes.insert(X: V);
4310 return true;
4311 }
4312 // Both operands are leaves so treat this binop as a source rather than
4313 // walking into it.
4314 return AddSource(V);
4315 };
4316 if (!Walk(VecOpEE, Walk) || Nodes.empty())
4317 return false;
4318
4319 bool IsIdempotent =
4320 CommonCallOp || (CommonBinOp && Instruction::isIdempotent(Opcode: *CommonBinOp));
4321
4322 // For FP reductions, require reassoc on every binop and collect FMF.
4323 for (Value *V : Nodes) {
4324 auto *BinOp = dyn_cast<BinaryOperator>(Val: V);
4325 if (!BinOp || !BinOp->getType()->isFPOrFPVectorTy())
4326 continue;
4327 if (!BinOp->hasAllowReassoc())
4328 return false;
4329 if (!IsFloatReduction) {
4330 CommonFMF = BinOp->getFastMathFlags();
4331 IsFloatReduction = true;
4332 } else {
4333 CommonFMF &= BinOp->getFastMathFlags();
4334 }
4335 }
4336
4337 // Top-down demanded elements. For each chain value, track which lanes feed
4338 // the extracted lane 0 and which feed it more than once. Reverse postorder
4339 // visits every use before its value. A binop forwards its demand to both
4340 // operands and a shuffle follows its mask back to the source lane.
4341 struct Demand {
4342 APInt Lanes;
4343 APInt Duplicates;
4344 };
4345 DenseMap<Value *, Demand> Demands;
4346 auto DemandOf = [&](Value *V) -> Demand & {
4347 unsigned N = cast<FixedVectorType>(Val: V->getType())->getNumElements();
4348 Demand &D = Demands[V];
4349 if (D.Lanes.getBitWidth() != N)
4350 D.Lanes = D.Duplicates = APInt::getZero(numBits: N);
4351 return D;
4352 };
4353 DemandOf(VecOpEE).Lanes.setBit(0);
4354 for (Value *V : reverse(C&: Nodes)) {
4355 Demand DV = Demands.lookup(Val: V);
4356 if (DV.Lanes.isZero())
4357 continue;
4358 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V)) {
4359 ArrayRef<int> Mask = SVI->getShuffleMask();
4360 Demand &DS = DemandOf(SVI->getOperand(i_nocapture: 0));
4361 for (unsigned I = 0, E = Mask.size(); I != E; ++I) {
4362 // Skip lanes that are undemanded or map to poison.
4363 if (!DV.Lanes[I] || Mask[I] < 0 ||
4364 (unsigned)Mask[I] >= DS.Lanes.getBitWidth())
4365 continue;
4366 if (DS.Lanes[Mask[I]] || DV.Duplicates[I])
4367 DS.Duplicates.setBit(Mask[I]);
4368 DS.Lanes.setBit(Mask[I]);
4369 }
4370 } else {
4371 auto *U = cast<User>(Val: V);
4372 for (Value *Op : {U->getOperand(i: 0), U->getOperand(i: 1)}) {
4373 Demand &DOp = DemandOf(Op);
4374 // Lanes demanded through more than one path accumulate in Duplicates.
4375 DOp.Duplicates |= DV.Duplicates | (DOp.Lanes & DV.Lanes);
4376 DOp.Lanes |= DV.Lanes;
4377 }
4378 }
4379 }
4380
4381 // Reducing V replaces the entire chain, so every contribution to the result
4382 // must flow through V. Reject if anything above V reads outside the chain.
4383 auto CoversChain = [&](Value *V) {
4384 SmallVector<Value *, 8> Worklist(1, VecOpEE);
4385 SmallPtrSet<Value *, 8> Seen;
4386 Seen.insert(Ptr: VecOpEE);
4387 while (!Worklist.empty()) {
4388 auto *U = cast<Instruction>(Val: Worklist.pop_back_val());
4389 unsigned NumOps = isa<ShuffleVectorInst>(Val: U) ? 1 : 2;
4390 for (unsigned I = 0; I != NumOps; ++I) {
4391 Value *Op = U->getOperand(i: I);
4392 if (Op == V || !Seen.insert(Ptr: Op).second)
4393 continue;
4394 if (!Nodes.contains(key: Op))
4395 return false;
4396 Worklist.push_back(Elt: Op);
4397 }
4398 }
4399 return true;
4400 };
4401
4402 // Reduce a single cleanly demanded source if there is one, otherwise the
4403 // deepest intermediate that covers the chain.
4404 struct ReductionCut {
4405 Value *Src;
4406 APInt Elts;
4407 };
4408 std::optional<ReductionCut> Cut;
4409 for (Value *S : Sources) {
4410 auto It = Demands.find(Val: S);
4411 if (It == Demands.end() || It->second.Lanes.isZero())
4412 continue;
4413 if (!IsIdempotent && !It->second.Duplicates.isZero()) {
4414 Cut.reset();
4415 break;
4416 }
4417 if (!Cut) {
4418 Cut = ReductionCut{.Src: S, .Elts: It->second.Lanes};
4419 continue;
4420 }
4421 if (!isEquivBitcast(X: Cut->Src, Y: S)) {
4422 Cut.reset();
4423 break;
4424 }
4425 if (!IsIdempotent && !(Cut->Elts & It->second.Lanes).isZero()) {
4426 Cut.reset();
4427 break;
4428 }
4429 Cut->Elts |= It->second.Lanes;
4430 }
4431 if (!Cut) {
4432 for (Value *V : Nodes) {
4433 if (!isa<BinaryOperator>(Val: V) && !isa<MinMaxIntrinsic>(Val: V))
4434 continue;
4435 auto It = Demands.find(Val: V);
4436 if (It == Demands.end() || !It->second.Lanes.isAllOnes())
4437 continue;
4438 if (!IsIdempotent && !It->second.Duplicates.isZero())
4439 continue;
4440 if (!CoversChain(V))
4441 continue;
4442 Cut = ReductionCut{.Src: V, .Elts: It->second.Lanes};
4443 break;
4444 }
4445 }
4446 // Reducing one lane is just an extract and can refold forever.
4447 if (!Cut || Cut->Elts.popcount() < 2)
4448 return false;
4449
4450 Intrinsic::ID ReducedOp =
4451 (CommonCallOp ? getMinMaxReductionIntrinsicID(IID: *CommonCallOp)
4452 : getReductionForBinop(Opc: *CommonBinOp));
4453 if (!ReducedOp)
4454 return false;
4455
4456 InstructionCost OrigCost = 0;
4457 for (Value *V : Nodes)
4458 OrigCost += TTI.getInstructionCost(U: cast<Instruction>(Val: V), CostKind);
4459
4460 auto *SrcVT = cast<FixedVectorType>(Val: Cut->Src->getType());
4461 bool IsPartialReduction = !Cut->Elts.isAllOnes();
4462 FixedVectorType *ReduceVecTy =
4463 IsPartialReduction
4464 ? FixedVectorType::get(ElementType: FVT->getElementType(), NumElts: Cut->Elts.popcount())
4465 : SrcVT;
4466
4467 SmallVector<int> ExtractMask;
4468 InstructionCost NewCost = 0;
4469 if (IsPartialReduction) {
4470 for (unsigned I = 0, E = Cut->Elts.getBitWidth(); I != E; ++I)
4471 if (Cut->Elts[I])
4472 ExtractMask.push_back(Elt: I);
4473 unsigned SubIdx = 0, SubLen;
4474 auto SK = Cut->Elts.isShiftedMask(MaskIdx&: SubIdx, MaskLen&: SubLen)
4475 ? TargetTransformInfo::SK_ExtractSubvector
4476 : TargetTransformInfo::SK_PermuteSingleSrc;
4477 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: ReduceVecTy, SrcTy: SrcVT, CostKind, Mask: ExtractMask,
4478 Index: SubIdx, SubTp: ReduceVecTy);
4479 }
4480
4481 IntrinsicCostAttributes ICA(
4482 ReducedOp, ReduceVecTy->getElementType(),
4483 IsFloatReduction
4484 ? SmallVector<Type *, 2>{ReduceVecTy->getElementType(), ReduceVecTy}
4485 : SmallVector<Type *, 2>{ReduceVecTy},
4486 IsFloatReduction ? CommonFMF : FastMathFlags());
4487 NewCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
4488
4489 LLVM_DEBUG(dbgs() << "Found reduction shuffle chain: " << I << "\n OldCost : "
4490 << OrigCost << " vs NewCost: " << NewCost << "\n");
4491
4492 if (!OrigCost.isValid() || !NewCost.isValid())
4493 return false;
4494
4495 if (VecOpEE->hasOneUse() ? (NewCost > OrigCost) : (NewCost >= OrigCost))
4496 return false;
4497
4498 Value *ReduceInput = Cut->Src;
4499 if (IsPartialReduction)
4500 ReduceInput = Builder.CreateShuffleVector(V: Cut->Src, Mask: ExtractMask);
4501
4502 Value *ReducedResult;
4503 if (IsFloatReduction) {
4504 Value *Identity = ConstantExpr::getBinOpIdentity(
4505 Opcode: *CommonBinOp, Ty: ReduceVecTy->getElementType(), /*AllowRHSConstant=*/false,
4506 NSZ: CommonFMF.noSignedZeros());
4507 ReducedResult = Builder.CreateIntrinsic(ID: ReducedOp, OverloadTypes: {ReduceVecTy},
4508 Args: {Identity, ReduceInput}, FMFSource: CommonFMF);
4509 } else {
4510 ReducedResult =
4511 Builder.CreateIntrinsic(ID: ReducedOp, OverloadTypes: {ReduceVecTy}, Args: {ReduceInput});
4512 }
4513 replaceValue(Old&: I, New&: *ReducedResult);
4514
4515 return true;
4516}
4517
4518/// Determine if its more efficient to fold:
4519/// reduce(trunc(x)) -> trunc(reduce(x)).
4520/// reduce(sext(x)) -> sext(reduce(x)).
4521/// reduce(zext(x)) -> zext(reduce(x)).
4522bool VectorCombine::foldCastFromReductions(Instruction &I) {
4523 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
4524 if (!II)
4525 return false;
4526
4527 bool TruncOnly = false;
4528 Intrinsic::ID IID = II->getIntrinsicID();
4529 switch (IID) {
4530 case Intrinsic::vector_reduce_add:
4531 case Intrinsic::vector_reduce_mul:
4532 TruncOnly = true;
4533 break;
4534 case Intrinsic::vector_reduce_and:
4535 case Intrinsic::vector_reduce_or:
4536 case Intrinsic::vector_reduce_xor:
4537 break;
4538 default:
4539 return false;
4540 }
4541
4542 unsigned ReductionOpc = getArithmeticReductionInstruction(RdxID: IID);
4543 Value *ReductionSrc = I.getOperand(i: 0);
4544
4545 Value *Src;
4546 if (!match(V: ReductionSrc, P: m_OneUse(SubPattern: m_Trunc(Op: m_Value(V&: Src)))) &&
4547 (TruncOnly || !match(V: ReductionSrc, P: m_OneUse(SubPattern: m_ZExtOrSExt(Op: m_Value(V&: Src))))))
4548 return false;
4549
4550 auto CastOpc =
4551 (Instruction::CastOps)cast<Instruction>(Val: ReductionSrc)->getOpcode();
4552
4553 auto *SrcTy = cast<VectorType>(Val: Src->getType());
4554 auto *ReductionSrcTy = cast<VectorType>(Val: ReductionSrc->getType());
4555 Type *ResultTy = I.getType();
4556
4557 InstructionCost OldCost = TTI.getArithmeticReductionCost(
4558 Opcode: ReductionOpc, Ty: ReductionSrcTy, FMF: std::nullopt, CostKind);
4559 OldCost += TTI.getCastInstrCost(Opcode: CastOpc, Dst: ReductionSrcTy, Src: SrcTy,
4560 CCH: TTI::CastContextHint::None, CostKind,
4561 I: cast<CastInst>(Val: ReductionSrc));
4562 InstructionCost NewCost =
4563 TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: SrcTy, FMF: std::nullopt,
4564 CostKind) +
4565 TTI.getCastInstrCost(Opcode: CastOpc, Dst: ResultTy, Src: ReductionSrcTy->getScalarType(),
4566 CCH: TTI::CastContextHint::None, CostKind);
4567
4568 if (OldCost <= NewCost || !NewCost.isValid())
4569 return false;
4570
4571 Value *NewReduction = Builder.CreateIntrinsic(RetTy: SrcTy->getScalarType(),
4572 ID: II->getIntrinsicID(), Args: {Src});
4573 Value *NewCast = Builder.CreateCast(Op: CastOpc, V: NewReduction, DestTy: ResultTy);
4574 replaceValue(Old&: I, New&: *NewCast);
4575 return true;
4576}
4577
4578/// Fold:
4579/// icmp pred (reduce.{add,or,and,umax,umin}(signbit_extract(x))), C
4580/// into:
4581/// icmp sgt/slt (reduce.{or,umax,and,umin}(x)), -1/0
4582///
4583/// Sign-bit reductions produce values with known semantics:
4584/// - reduce.{or,umax}: 0 if no element is negative, 1 if any is
4585/// - reduce.{and,umin}: 1 if all elements are negative, 0 if any isn't
4586/// - reduce.add: count of negative elements (0 to NumElts)
4587///
4588/// Both lshr and ashr are supported:
4589/// - lshr produces 0 or 1, so reduce.add range is [0, N]
4590/// - ashr produces 0 or -1, so reduce.add range is [-N, 0]
4591///
4592/// The fold generalizes to multiple source vectors combined with the same
4593/// operation as the reduction. For example:
4594/// reduce.or(or(shr A, shr B)) conceptually extends the vector
4595/// For reduce.add, this changes the count to M*N where M is the number of
4596/// source vectors.
4597///
4598/// We transform to a direct sign check on the original vector using
4599/// reduce.{or,umax} or reduce.{and,umin}.
4600///
4601/// In spirit, it's similar to foldSignBitCheck in InstCombine.
4602bool VectorCombine::foldSignBitReductionCmp(Instruction &I) {
4603 CmpPredicate Pred;
4604 IntrinsicInst *ReduceOp;
4605 const APInt *CmpVal;
4606 if (!match(V: &I,
4607 P: m_ICmp(Pred, L: m_OneUse(SubPattern: m_AnyIntrinsic(I&: ReduceOp)), R: m_APInt(Res&: CmpVal))))
4608 return false;
4609
4610 Intrinsic::ID OrigIID = ReduceOp->getIntrinsicID();
4611 switch (OrigIID) {
4612 case Intrinsic::vector_reduce_or:
4613 case Intrinsic::vector_reduce_umax:
4614 case Intrinsic::vector_reduce_and:
4615 case Intrinsic::vector_reduce_umin:
4616 case Intrinsic::vector_reduce_add:
4617 break;
4618 default:
4619 return false;
4620 }
4621
4622 Value *ReductionSrc = ReduceOp->getArgOperand(i: 0);
4623 auto *VecTy = dyn_cast<FixedVectorType>(Val: ReductionSrc->getType());
4624 if (!VecTy)
4625 return false;
4626
4627 unsigned BitWidth = VecTy->getScalarSizeInBits();
4628 if (BitWidth == 1)
4629 return false;
4630
4631 unsigned NumElts = VecTy->getNumElements();
4632
4633 // Determine the expected tree opcode for multi-vector patterns.
4634 // The tree opcode must match the reduction's underlying operation.
4635 //
4636 // TODO: for pairs of equivalent operators, we should match both,
4637 // not only the most common.
4638 Instruction::BinaryOps TreeOpcode;
4639 switch (OrigIID) {
4640 case Intrinsic::vector_reduce_or:
4641 case Intrinsic::vector_reduce_umax:
4642 TreeOpcode = Instruction::Or;
4643 break;
4644 case Intrinsic::vector_reduce_and:
4645 case Intrinsic::vector_reduce_umin:
4646 TreeOpcode = Instruction::And;
4647 break;
4648 case Intrinsic::vector_reduce_add:
4649 TreeOpcode = Instruction::Add;
4650 break;
4651 default:
4652 llvm_unreachable("Unexpected intrinsic");
4653 }
4654
4655 // Collect sign-bit extraction leaves from an associative tree of TreeOpcode.
4656 // The tree conceptually extends the vector being reduced.
4657 SmallVector<Value *, 8> Worklist;
4658 SmallVector<Value *, 8> Sources; // Original vectors (X in shr X, BW-1)
4659 Worklist.push_back(Elt: ReductionSrc);
4660 std::optional<bool> IsAShr;
4661 constexpr unsigned MaxSources = 8;
4662
4663 // Calculate old cost: all shifts + tree ops + reduction
4664 InstructionCost OldCost = TTI.getInstructionCost(U: ReduceOp, CostKind);
4665
4666 while (!Worklist.empty() && Worklist.size() <= MaxSources &&
4667 Sources.size() <= MaxSources) {
4668 Value *V = Worklist.pop_back_val();
4669
4670 // Try to match sign-bit extraction: shr X, (bitwidth-1)
4671 Value *X;
4672 if (match(V, P: m_OneUse(SubPattern: m_Shr(L: m_Value(V&: X), R: m_SpecificInt(V: BitWidth - 1))))) {
4673 auto *Shr = cast<Instruction>(Val: V);
4674
4675 // All shifts must be the same type (all lshr or all ashr)
4676 bool ThisIsAShr = Shr->getOpcode() == Instruction::AShr;
4677 if (!IsAShr)
4678 IsAShr = ThisIsAShr;
4679 else if (*IsAShr != ThisIsAShr)
4680 return false;
4681
4682 Sources.push_back(Elt: X);
4683
4684 // As part of the fold, we remove all of the shifts, so we need to keep
4685 // track of their costs.
4686 OldCost += TTI.getInstructionCost(U: Shr, CostKind);
4687
4688 continue;
4689 }
4690
4691 // Try to extend through a tree node of the expected opcode
4692 Value *A, *B;
4693 if (!match(V, P: m_OneUse(SubPattern: m_BinOp(Opcode: TreeOpcode, L: m_Value(V&: A), R: m_Value(V&: B)))))
4694 return false;
4695
4696 // We are potentially replacing these operations as well, so we add them
4697 // to the costs.
4698 OldCost += TTI.getInstructionCost(U: cast<Instruction>(Val: V), CostKind);
4699
4700 Worklist.push_back(Elt: A);
4701 Worklist.push_back(Elt: B);
4702 }
4703
4704 // Must have at least one source and not exceed limit
4705 if (Sources.empty() || Sources.size() > MaxSources ||
4706 Worklist.size() > MaxSources || !IsAShr)
4707 return false;
4708
4709 unsigned NumSources = Sources.size();
4710
4711 // For reduce.add, the total count must fit as a signed integer.
4712 // Range is [0, M*N] for lshr or [-M*N, 0] for ashr.
4713 if (OrigIID == Intrinsic::vector_reduce_add &&
4714 !isIntN(N: BitWidth, x: NumSources * NumElts))
4715 return false;
4716
4717 // Compute the boundary value when all elements are negative:
4718 // - Per-element contribution: 1 for lshr, -1 for ashr
4719 // - For add: M*N (total elements across all sources); for others: just 1
4720 unsigned Count =
4721 (OrigIID == Intrinsic::vector_reduce_add) ? NumSources * NumElts : 1;
4722 APInt NegativeVal(CmpVal->getBitWidth(), Count);
4723 if (*IsAShr)
4724 NegativeVal.negate();
4725
4726 // Range is [min(0, AllNegVal), max(0, AllNegVal)]
4727 APInt Zero = APInt::getZero(numBits: CmpVal->getBitWidth());
4728 APInt RangeLow = APIntOps::smin(A: Zero, B: NegativeVal);
4729 APInt RangeHigh = APIntOps::smax(A: Zero, B: NegativeVal);
4730
4731 // Determine comparison semantics:
4732 // - IsEq: true for equality test, false for inequality
4733 // - TestsNegative: true if testing against AllNegVal, false for zero
4734 //
4735 // In addition to EQ/NE against 0 or AllNegVal, we support inequalities
4736 // that fold to boundary tests given the narrow value range:
4737 // < RangeHigh -> != RangeHigh
4738 // > RangeHigh-1 -> == RangeHigh
4739 // > RangeLow -> != RangeLow
4740 // < RangeLow+1 -> == RangeLow
4741 //
4742 // For inequalities, we work with signed predicates only. Unsigned predicates
4743 // are canonicalized to signed when the range is non-negative (where they are
4744 // equivalent). When the range includes negative values, unsigned predicates
4745 // would have different semantics due to wrap-around, so we reject them.
4746 if (!ICmpInst::isEquality(P: Pred) && !ICmpInst::isSigned(Pred)) {
4747 if (RangeLow.isNegative())
4748 return false;
4749 Pred = ICmpInst::getSignedPredicate(Pred);
4750 }
4751
4752 bool IsEq;
4753 bool TestsNegative;
4754 if (ICmpInst::isEquality(P: Pred)) {
4755 if (CmpVal->isZero()) {
4756 TestsNegative = false;
4757 } else if (*CmpVal == NegativeVal) {
4758 TestsNegative = true;
4759 } else {
4760 return false;
4761 }
4762 IsEq = Pred == ICmpInst::ICMP_EQ;
4763 } else if (Pred == ICmpInst::ICMP_SLT && *CmpVal == RangeHigh) {
4764 IsEq = false;
4765 TestsNegative = (RangeHigh == NegativeVal);
4766 } else if (Pred == ICmpInst::ICMP_SGT && *CmpVal == RangeHigh - 1) {
4767 IsEq = true;
4768 TestsNegative = (RangeHigh == NegativeVal);
4769 } else if (Pred == ICmpInst::ICMP_SGT && *CmpVal == RangeLow) {
4770 IsEq = false;
4771 TestsNegative = (RangeLow == NegativeVal);
4772 } else if (Pred == ICmpInst::ICMP_SLT && *CmpVal == RangeLow + 1) {
4773 IsEq = true;
4774 TestsNegative = (RangeLow == NegativeVal);
4775 } else {
4776 return false;
4777 }
4778
4779 // For this fold we support four types of checks:
4780 //
4781 // 1. All lanes are negative - AllNeg
4782 // 2. All lanes are non-negative - AllNonNeg
4783 // 3. At least one negative lane - AnyNeg
4784 // 4. At least one non-negative lane - AnyNonNeg
4785 //
4786 // For each case, we can generate the following code:
4787 //
4788 // 1. AllNeg - reduce.and/umin(X) < 0
4789 // 2. AllNonNeg - reduce.or/umax(X) > -1
4790 // 3. AnyNeg - reduce.or/umax(X) < 0
4791 // 4. AnyNonNeg - reduce.and/umin(X) > -1
4792 //
4793 // The table below shows the aggregation of all supported cases
4794 // using these four cases.
4795 //
4796 // Reduction | == 0 | != 0 | == MAX | != MAX
4797 // ------------+-----------+-----------+-----------+-----------
4798 // or/umax | AllNonNeg | AnyNeg | AnyNeg | AllNonNeg
4799 // and/umin | AnyNonNeg | AllNeg | AllNeg | AnyNonNeg
4800 // add | AllNonNeg | AnyNeg | AllNeg | AnyNonNeg
4801 //
4802 // NOTE: MAX = 1 for or/and/umax/umin, and the vector size N for add
4803 //
4804 // For easier codegen and check inversion, we use the following encoding:
4805 //
4806 // 1. Bit-3 === requires or/umax (1) or and/umin (0) check
4807 // 2. Bit-2 === checks < 0 (1) or > -1 (0)
4808 // 3. Bit-1 === universal (1) or existential (0) check
4809 //
4810 // AnyNeg = 0b110: uses or/umax, checks negative, any-check
4811 // AllNonNeg = 0b101: uses or/umax, checks non-neg, all-check
4812 // AnyNonNeg = 0b000: uses and/umin, checks non-neg, any-check
4813 // AllNeg = 0b011: uses and/umin, checks negative, all-check
4814 //
4815 // XOR with 0b011 inverts the check (swaps all/any and neg/non-neg).
4816 //
4817 enum CheckKind : unsigned {
4818 AnyNonNeg = 0b000,
4819 AllNeg = 0b011,
4820 AllNonNeg = 0b101,
4821 AnyNeg = 0b110,
4822 };
4823 // Return true if we fold this check into or/umax and false for and/umin
4824 auto RequiresOr = [](CheckKind C) -> bool { return C & 0b100; };
4825 // Return true if we should check if result is negative and false otherwise
4826 auto IsNegativeCheck = [](CheckKind C) -> bool { return C & 0b010; };
4827 // Logically invert the check
4828 auto Invert = [](CheckKind C) { return CheckKind(C ^ 0b011); };
4829
4830 CheckKind Base;
4831 switch (OrigIID) {
4832 case Intrinsic::vector_reduce_or:
4833 case Intrinsic::vector_reduce_umax:
4834 Base = TestsNegative ? AnyNeg : AllNonNeg;
4835 break;
4836 case Intrinsic::vector_reduce_and:
4837 case Intrinsic::vector_reduce_umin:
4838 Base = TestsNegative ? AllNeg : AnyNonNeg;
4839 break;
4840 case Intrinsic::vector_reduce_add:
4841 Base = TestsNegative ? AllNeg : AllNonNeg;
4842 break;
4843 default:
4844 llvm_unreachable("Unexpected intrinsic");
4845 }
4846
4847 CheckKind Check = IsEq ? Base : Invert(Base);
4848
4849 auto PickCheaper = [&](Intrinsic::ID Arith, Intrinsic::ID MinMax) {
4850 InstructionCost ArithCost =
4851 TTI.getArithmeticReductionCost(Opcode: getArithmeticReductionInstruction(RdxID: Arith),
4852 Ty: VecTy, FMF: std::nullopt, CostKind);
4853 InstructionCost MinMaxCost =
4854 TTI.getMinMaxReductionCost(IID: getMinMaxReductionIntrinsicOp(RdxID: MinMax), Ty: VecTy,
4855 FMF: FastMathFlags(), CostKind);
4856 return ArithCost <= MinMaxCost ? std::make_pair(x&: Arith, y&: ArithCost)
4857 : std::make_pair(x&: MinMax, y&: MinMaxCost);
4858 };
4859
4860 // Choose output reduction based on encoding's MSB
4861 auto [NewIID, NewCost] = RequiresOr(Check)
4862 ? PickCheaper(Intrinsic::vector_reduce_or,
4863 Intrinsic::vector_reduce_umax)
4864 : PickCheaper(Intrinsic::vector_reduce_and,
4865 Intrinsic::vector_reduce_umin);
4866
4867 // Add cost of combining multiple sources with or/and
4868 if (NumSources > 1) {
4869 unsigned CombineOpc =
4870 RequiresOr(Check) ? Instruction::Or : Instruction::And;
4871 NewCost += TTI.getArithmeticInstrCost(Opcode: CombineOpc, Ty: VecTy, CostKind) *
4872 (NumSources - 1);
4873 }
4874
4875 LLVM_DEBUG(dbgs() << "Found sign-bit reduction cmp: " << I << "\n OldCost: "
4876 << OldCost << " vs NewCost: " << NewCost << "\n");
4877
4878 if (NewCost > OldCost)
4879 return false;
4880
4881 // Generate the combined input and reduction
4882 Builder.SetInsertPoint(&I);
4883 Type *ScalarTy = VecTy->getScalarType();
4884
4885 Value *Input;
4886 if (NumSources == 1) {
4887 Input = Sources[0];
4888 } else {
4889 // Combine sources with or/and based on check type
4890 Input = RequiresOr(Check) ? Builder.CreateOr(Ops: Sources)
4891 : Builder.CreateAnd(Ops: Sources);
4892 }
4893
4894 Value *NewReduce = Builder.CreateIntrinsic(RetTy: ScalarTy, ID: NewIID, Args: {Input});
4895 Value *NewCmp = IsNegativeCheck(Check) ? Builder.CreateIsNeg(Arg: NewReduce)
4896 : Builder.CreateIsNotNeg(Arg: NewReduce);
4897 replaceValue(Old&: I, New&: *NewCmp);
4898 return true;
4899}
4900
4901/// Fold a zero test of reduce.or or reduce.umax into a boolean reduction.
4902///
4903/// Vectorization may produce IR that compares the result of a scalar reduction
4904/// with zero. Depending on the target, lowering a reduction and a scalar
4905/// comparison separately can cost more than reducing lane-wise comparison
4906/// results. This fold creates the latter form only when it is not costlier.
4907///
4908/// Before:
4909/// %r = call iT @llvm.vector.reduce.or.vNiT(<N x iT> %x)
4910/// %cmp = icmp ne iT %r, 0
4911///
4912/// After:
4913/// %lane.cmp = icmp ne <N x iT> %x, zeroinitializer
4914/// %cmp = call i1 @llvm.vector.reduce.or.vNi1(<N x i1> %lane.cmp)
4915///
4916/// `reduce.or` and `reduce.umax` are non-zero when at least one lane is
4917/// non-zero. Therefore, `icmp ne` uses the existential `reduce.or` test.
4918/// Conversely, `icmp eq` must check that every lane is zero, so it uses the
4919/// universal `reduce.and` test.
4920///
4921/// Before:
4922/// %r = call iT @llvm.vector.reduce.umax.vNiT(<N x iT> %x)
4923/// %cmp = icmp eq iT %r, 0
4924///
4925/// After:
4926/// %lane.cmp = icmp eq <N x iT> %x, zeroinitializer
4927/// %cmp = call i1 @llvm.vector.reduce.and.vNi1(<N x i1> %lane.cmp)
4928bool VectorCombine::foldReductionZeroTest(Instruction &I) {
4929 CmpPredicate Pred;
4930 Value *Op;
4931
4932 if (!match(V: &I, P: m_c_ICmp(Pred, L: m_Value(V&: Op), R: m_Zero())) ||
4933 !ICmpInst::isEquality(P: Pred))
4934 return false;
4935
4936 auto *II = dyn_cast<IntrinsicInst>(Val: Op);
4937 if (!II || !II->hasOneUse())
4938 return false;
4939
4940 auto ReduceID = II->getIntrinsicID();
4941 if (ReduceID != Intrinsic::vector_reduce_or &&
4942 ReduceID != Intrinsic::vector_reduce_umax)
4943 return false;
4944
4945 Value *Vec = II->getArgOperand(i: 0);
4946 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
4947 if (!VecTy || !VecTy->getElementType()->isIntegerTy())
4948 return false;
4949
4950 // Map the scalar zero test to an any-lane or all-lane boolean reduction.
4951 Intrinsic::ID NewIID = (Pred == ICmpInst::ICMP_NE)
4952 ? Intrinsic::vector_reduce_or
4953 : Intrinsic::vector_reduce_and;
4954
4955 // This is not an unconditional canonicalization: compare the cost of the
4956 // original scalar reduction and compare with the vector compare and i1
4957 // reduction replacement for both reduce.or and reduce.umax.
4958 InstructionCost OldCost = TTI.getInstructionCost(U: II, CostKind) +
4959 TTI.getInstructionCost(U: &I, CostKind);
4960
4961 auto *CmpTy = cast<VectorType>(Val: CmpInst::makeCmpResultType(opnd_type: VecTy));
4962 InstructionCost NewCost =
4963 TTI.getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: VecTy, CondTy: CmpTy, VecPred: Pred, CostKind);
4964 NewCost += TTI.getArithmeticReductionCost(
4965 Opcode: getArithmeticReductionInstruction(RdxID: NewIID), Ty: CmpTy, FMF: std::nullopt, CostKind);
4966
4967 LLVM_DEBUG(dbgs() << "Found a reduction zero test: " << I << "\n OldCost: "
4968 << OldCost << " vs NewCost: " << NewCost << "\n");
4969
4970 if (!OldCost.isValid() || !NewCost.isValid() || NewCost > OldCost)
4971 return false;
4972
4973 Builder.SetInsertPoint(&I);
4974 Value *NewCmp = Builder.CreateICmp(P: Pred, LHS: Vec, RHS: Constant::getNullValue(Ty: VecTy));
4975 Value *NewReduce = Builder.CreateIntrinsic(ID: NewIID, OverloadTypes: {CmpTy}, Args: {NewCmp});
4976 replaceValue(Old&: I, New&: *NewReduce);
4977 return true;
4978}
4979
4980/// vector.reduce.OP f(X_i) == 0 -> vector.reduce.OP X_i == 0
4981///
4982/// We can prove it for cases when:
4983///
4984/// 1. OP X_i == 0 <=> \forall i \in [1, N] X_i == 0
4985/// 1'. OP X_i == 0 <=> \exists j \in [1, N] X_j == 0
4986/// 2. f(x) == 0 <=> x == 0
4987///
4988/// From 1 and 2 (or 1' and 2), we can infer that
4989///
4990/// OP f(X_i) == 0 <=> OP X_i == 0.
4991///
4992/// (1)
4993/// OP f(X_i) == 0 <=> \forall i \in [1, N] f(X_i) == 0
4994/// (2)
4995/// <=> \forall i \in [1, N] X_i == 0
4996/// (1)
4997/// <=> OP(X_i) == 0
4998///
4999/// For some of the OP's and f's, we need to have domain constraints on X
5000/// to ensure properties 1 (or 1') and 2.
5001bool VectorCombine::foldICmpEqZeroVectorReduce(Instruction &I) {
5002 CmpPredicate Pred;
5003 Value *Op;
5004 if (!match(V: &I, P: m_ICmp(Pred, L: m_Value(V&: Op), R: m_Zero())) ||
5005 !ICmpInst::isEquality(P: Pred))
5006 return false;
5007
5008 auto *II = dyn_cast<IntrinsicInst>(Val: Op);
5009 if (!II)
5010 return false;
5011
5012 switch (II->getIntrinsicID()) {
5013 case Intrinsic::vector_reduce_add:
5014 case Intrinsic::vector_reduce_or:
5015 case Intrinsic::vector_reduce_umin:
5016 case Intrinsic::vector_reduce_umax:
5017 case Intrinsic::vector_reduce_smin:
5018 case Intrinsic::vector_reduce_smax:
5019 break;
5020 default:
5021 return false;
5022 }
5023
5024 Value *InnerOp = II->getArgOperand(i: 0);
5025
5026 // TODO: fixed vector type might be too restrictive
5027 if (!II->hasOneUse() || !isa<FixedVectorType>(Val: InnerOp->getType()))
5028 return false;
5029
5030 Value *X = nullptr;
5031
5032 // Check for zero-preserving operations where f(x) = 0 <=> x = 0
5033 //
5034 // 1. f(x) = shl nuw x, y for arbitrary y
5035 // 2. f(x) = mul nuw x, c for defined c != 0
5036 // 3. f(x) = zext x
5037 // 4. f(x) = sext x
5038 // 5. f(x) = neg x
5039 //
5040 if (!(match(V: InnerOp, P: m_NUWShl(L: m_Value(V&: X), R: m_Value())) || // Case 1
5041 match(V: InnerOp, P: m_NUWMul(L: m_Value(V&: X), R: m_NonZeroInt())) || // Case 2
5042 match(V: InnerOp, P: m_ZExt(Op: m_Value(V&: X))) || // Case 3
5043 match(V: InnerOp, P: m_SExt(Op: m_Value(V&: X))) || // Case 4
5044 match(V: InnerOp, P: m_Neg(V: m_Value(V&: X))) // Case 5
5045 ))
5046 return false;
5047
5048 SimplifyQuery S = SQ.getWithInstruction(I: &I);
5049 auto *XTy = cast<FixedVectorType>(Val: X->getType());
5050
5051 // Check for domain constraints for all supported reductions.
5052 //
5053 // a. OR X_i - has property 1 for every X
5054 // b. UMAX X_i - has property 1 for every X
5055 // c. UMIN X_i - has property 1' for every X
5056 // d. SMAX X_i - has property 1 for X >= 0
5057 // e. SMIN X_i - has property 1' for X >= 0
5058 // f. ADD X_i - has property 1 for X >= 0 && ADD X_i doesn't sign wrap
5059 //
5060 // In order for the proof to work, we need 1 (or 1') to be true for both
5061 // OP f(X_i) and OP X_i and that's why below we check constraints twice.
5062 //
5063 // NOTE: ADD X_i holds property 1 for a mirror case as well, i.e. when
5064 // X <= 0 && ADD X_i doesn't sign wrap. However, due to the nature
5065 // of known bits, we can't reasonably hold knowledge of "either 0
5066 // or negative".
5067 switch (II->getIntrinsicID()) {
5068 case Intrinsic::vector_reduce_add: {
5069 // We need to check that both X_i and f(X_i) have enough leading
5070 // zeros to not overflow.
5071 KnownBits KnownX = computeKnownBits(V: X, Q: S);
5072 KnownBits KnownFX = computeKnownBits(V: InnerOp, Q: S);
5073 unsigned NumElems = XTy->getNumElements();
5074 // Adding N elements loses at most ceil(log2(N)) leading bits.
5075 unsigned LostBits = Log2_32_Ceil(Value: NumElems);
5076 unsigned LeadingZerosX = KnownX.countMinLeadingZeros();
5077 unsigned LeadingZerosFX = KnownFX.countMinLeadingZeros();
5078 // Need at least one leading zero left after summation to ensure no overflow
5079 if (LeadingZerosX <= LostBits || LeadingZerosFX <= LostBits)
5080 return false;
5081
5082 // We are not checking whether X or f(X) are positive explicitly because
5083 // we implicitly checked for it when we checked if both cases have enough
5084 // leading zeros to not wrap addition.
5085 break;
5086 }
5087 case Intrinsic::vector_reduce_smin:
5088 case Intrinsic::vector_reduce_smax:
5089 // Check whether X >= 0 and f(X) >= 0
5090 if (!isKnownNonNegative(V: InnerOp, SQ: S) || !isKnownNonNegative(V: X, SQ: S))
5091 return false;
5092
5093 break;
5094 default:
5095 break;
5096 };
5097
5098 LLVM_DEBUG(dbgs() << "Found a reduction to 0 comparison with removable op: "
5099 << *II << "\n");
5100
5101 // For zext/sext, check if the transform is profitable using cost model.
5102 // For other operations (shl, mul, neg), we're removing an instruction
5103 // while keeping the same reduction type, so it's always profitable.
5104 if (isa<ZExtInst>(Val: InnerOp) || isa<SExtInst>(Val: InnerOp)) {
5105 auto *FXTy = cast<FixedVectorType>(Val: InnerOp->getType());
5106 Intrinsic::ID IID = II->getIntrinsicID();
5107
5108 InstructionCost ExtCost = TTI.getCastInstrCost(
5109 Opcode: cast<CastInst>(Val: InnerOp)->getOpcode(), Dst: FXTy, Src: XTy,
5110 CCH: TTI::CastContextHint::None, CostKind, I: cast<CastInst>(Val: InnerOp));
5111
5112 InstructionCost OldReduceCost, NewReduceCost;
5113 switch (IID) {
5114 case Intrinsic::vector_reduce_add:
5115 case Intrinsic::vector_reduce_or:
5116 OldReduceCost = TTI.getArithmeticReductionCost(
5117 Opcode: getArithmeticReductionInstruction(RdxID: IID), Ty: FXTy, FMF: std::nullopt, CostKind);
5118 NewReduceCost = TTI.getArithmeticReductionCost(
5119 Opcode: getArithmeticReductionInstruction(RdxID: IID), Ty: XTy, FMF: std::nullopt, CostKind);
5120 break;
5121 case Intrinsic::vector_reduce_umin:
5122 case Intrinsic::vector_reduce_umax:
5123 case Intrinsic::vector_reduce_smin:
5124 case Intrinsic::vector_reduce_smax:
5125 OldReduceCost = TTI.getMinMaxReductionCost(
5126 IID: getMinMaxReductionIntrinsicOp(RdxID: IID), Ty: FXTy, FMF: FastMathFlags(), CostKind);
5127 NewReduceCost = TTI.getMinMaxReductionCost(
5128 IID: getMinMaxReductionIntrinsicOp(RdxID: IID), Ty: XTy, FMF: FastMathFlags(), CostKind);
5129 break;
5130 default:
5131 llvm_unreachable("Unexpected reduction");
5132 }
5133
5134 InstructionCost OldCost = OldReduceCost + ExtCost;
5135 InstructionCost NewCost =
5136 NewReduceCost + (InnerOp->hasOneUse() ? 0 : ExtCost);
5137
5138 LLVM_DEBUG(dbgs() << "Found a removable extension before reduction: "
5139 << *InnerOp << "\n OldCost: " << OldCost
5140 << " vs NewCost: " << NewCost << "\n");
5141
5142 // We consider transformation to still be potentially beneficial even
5143 // when the costs are the same because we might remove a use from f(X)
5144 // and unlock other optimizations. Equal costs would just mean that we
5145 // didn't make it worse in the worst case.
5146 if (NewCost > OldCost)
5147 return false;
5148 }
5149
5150 // Since we support zext and sext as f, we might change the scalar type
5151 // of the intrinsic.
5152 Type *Ty = XTy->getScalarType();
5153 Value *NewReduce = Builder.CreateIntrinsic(RetTy: Ty, ID: II->getIntrinsicID(), Args: {X});
5154 Value *NewCmp =
5155 Builder.CreateICmp(P: Pred, LHS: NewReduce, RHS: ConstantInt::getNullValue(Ty));
5156 replaceValue(Old&: I, New&: *NewCmp);
5157 return true;
5158}
5159
5160/// Fold comparisons of reduce.or/reduce.and with reduce.umax/reduce.umin
5161/// based on cost, preserving the comparison semantics.
5162///
5163/// We use two fundamental properties for each pair:
5164///
5165/// 1. or(X) == 0 <=> umax(X) == 0
5166/// 2. or(X) == 1 <=> umax(X) == 1
5167/// 3. sign(or(X)) == sign(umax(X))
5168///
5169/// 1. and(X) == -1 <=> umin(X) == -1
5170/// 2. and(X) == -2 <=> umin(X) == -2
5171/// 3. sign(and(X)) == sign(umin(X))
5172///
5173/// From these we can infer the following transformations:
5174/// a. or(X) ==/!= 0 <-> umax(X) ==/!= 0
5175/// b. or(X) s< 0 <-> umax(X) s< 0
5176/// c. or(X) s> -1 <-> umax(X) s> -1
5177/// d. or(X) s< 1 <-> umax(X) s< 1
5178/// e. or(X) ==/!= 1 <-> umax(X) ==/!= 1
5179/// f. or(X) s< 2 <-> umax(X) s< 2
5180/// g. and(X) ==/!= -1 <-> umin(X) ==/!= -1
5181/// h. and(X) s< 0 <-> umin(X) s< 0
5182/// i. and(X) s> -1 <-> umin(X) s> -1
5183/// j. and(X) s> -2 <-> umin(X) s> -2
5184/// k. and(X) ==/!= -2 <-> umin(X) ==/!= -2
5185/// l. and(X) s> -3 <-> umin(X) s> -3
5186///
5187bool VectorCombine::foldEquivalentReductionCmp(Instruction &I) {
5188 CmpPredicate Pred;
5189 Value *ReduceOp;
5190 const APInt *CmpVal;
5191 if (!match(V: &I, P: m_ICmp(Pred, L: m_Value(V&: ReduceOp), R: m_APInt(Res&: CmpVal))))
5192 return false;
5193
5194 auto *II = dyn_cast<IntrinsicInst>(Val: ReduceOp);
5195 if (!II || !II->hasOneUse())
5196 return false;
5197
5198 const auto IsValidOrUmaxCmp = [&]() {
5199 // or === umax for i1
5200 if (CmpVal->getBitWidth() == 1)
5201 return true;
5202
5203 // Cases a and e
5204 bool IsEquality =
5205 (CmpVal->isZero() || CmpVal->isOne()) && ICmpInst::isEquality(P: Pred);
5206 // Case c
5207 bool IsPositive = CmpVal->isAllOnes() && Pred == ICmpInst::ICMP_SGT;
5208 // Cases b, d, and f
5209 bool IsNegative = (CmpVal->isZero() || CmpVal->isOne() || *CmpVal == 2) &&
5210 Pred == ICmpInst::ICMP_SLT;
5211 return IsEquality || IsPositive || IsNegative;
5212 };
5213
5214 const auto IsValidAndUminCmp = [&]() {
5215 // and === umin for i1
5216 if (CmpVal->getBitWidth() == 1)
5217 return true;
5218
5219 const auto LeadingOnes = CmpVal->countl_one();
5220
5221 // Cases g and k
5222 bool IsEquality =
5223 (CmpVal->isAllOnes() || LeadingOnes + 1 == CmpVal->getBitWidth()) &&
5224 ICmpInst::isEquality(P: Pred);
5225 // Case h
5226 bool IsNegative = CmpVal->isZero() && Pred == ICmpInst::ICMP_SLT;
5227 // Cases i, j, and l
5228 bool IsPositive =
5229 // if the number has at least N - 2 leading ones
5230 // and the two LSBs are:
5231 // - 1 x 1 -> -1
5232 // - 1 x 0 -> -2
5233 // - 0 x 1 -> -3
5234 LeadingOnes + 2 >= CmpVal->getBitWidth() &&
5235 ((*CmpVal)[0] || (*CmpVal)[1]) && Pred == ICmpInst::ICMP_SGT;
5236 return IsEquality || IsNegative || IsPositive;
5237 };
5238
5239 Intrinsic::ID OriginalIID = II->getIntrinsicID();
5240 Intrinsic::ID AlternativeIID;
5241
5242 // Check if this is a valid comparison pattern and determine the alternate
5243 // reduction intrinsic.
5244 switch (OriginalIID) {
5245 case Intrinsic::vector_reduce_or:
5246 if (!IsValidOrUmaxCmp())
5247 return false;
5248 AlternativeIID = Intrinsic::vector_reduce_umax;
5249 break;
5250 case Intrinsic::vector_reduce_umax:
5251 if (!IsValidOrUmaxCmp())
5252 return false;
5253 AlternativeIID = Intrinsic::vector_reduce_or;
5254 break;
5255 case Intrinsic::vector_reduce_and:
5256 if (!IsValidAndUminCmp())
5257 return false;
5258 AlternativeIID = Intrinsic::vector_reduce_umin;
5259 break;
5260 case Intrinsic::vector_reduce_umin:
5261 if (!IsValidAndUminCmp())
5262 return false;
5263 AlternativeIID = Intrinsic::vector_reduce_and;
5264 break;
5265 default:
5266 return false;
5267 }
5268
5269 Value *X = II->getArgOperand(i: 0);
5270 auto *VecTy = dyn_cast<FixedVectorType>(Val: X->getType());
5271 if (!VecTy)
5272 return false;
5273
5274 const auto GetReductionCost = [&](Intrinsic::ID IID) -> InstructionCost {
5275 unsigned ReductionOpc = getArithmeticReductionInstruction(RdxID: IID);
5276 if (ReductionOpc != Instruction::ICmp)
5277 return TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: VecTy, FMF: std::nullopt,
5278 CostKind);
5279 return TTI.getMinMaxReductionCost(IID: getMinMaxReductionIntrinsicOp(RdxID: IID), Ty: VecTy,
5280 FMF: FastMathFlags(), CostKind);
5281 };
5282
5283 InstructionCost OrigCost = GetReductionCost(OriginalIID);
5284 InstructionCost AltCost = GetReductionCost(AlternativeIID);
5285
5286 LLVM_DEBUG(dbgs() << "Found equivalent reduction cmp: " << I
5287 << "\n OrigCost: " << OrigCost
5288 << " vs AltCost: " << AltCost << "\n");
5289
5290 if (AltCost >= OrigCost)
5291 return false;
5292
5293 Builder.SetInsertPoint(&I);
5294 Type *ScalarTy = VecTy->getScalarType();
5295 Value *NewReduce = Builder.CreateIntrinsic(RetTy: ScalarTy, ID: AlternativeIID, Args: {X});
5296 Value *NewCmp =
5297 Builder.CreateICmp(P: Pred, LHS: NewReduce, RHS: ConstantInt::get(Ty: ScalarTy, V: *CmpVal));
5298
5299 replaceValue(Old&: I, New&: *NewCmp);
5300 return true;
5301}
5302
5303/// Used by foldReduceAddCmpZero to check if we can prove that a value is
5304/// non-positive.
5305/// KnownBits cannot see sext <? x i1> as non-positive: each top bit equals a
5306/// single unknown input bit, which a per-bit lattice cannot track. The fold's
5307/// target shape is popcount-style sums of <N x i1> valid/invalid masks (e.g.
5308/// ray-intersection hits) tested for any-hit.
5309/// Previous attempts to approximate the known bits of such expressions were
5310/// using a fully recursive value tracking approach to infer a constant range
5311/// but ultimately turned to be too expensive in compile time.
5312static bool isKnownNonPositive(const Value *V, const SimplifyQuery &SQ,
5313 unsigned Depth = 0) {
5314 constexpr unsigned MaxLocalDepth = 2;
5315 if (Depth > MaxLocalDepth)
5316 return false;
5317
5318 auto NumSignBits = [&](const Value *X) {
5319 return ComputeNumSignBits(Op: X, DL: SQ.DL, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT);
5320 };
5321 if (NumSignBits(V) == V->getType()->getScalarSizeInBits())
5322 return true;
5323
5324 Value *A, *B;
5325 if (match(V, P: m_Add(L: m_Value(V&: A), R: m_Value(V&: B))))
5326 return NumSignBits(A) >= 2 && NumSignBits(B) >= 2 &&
5327 isKnownNonPositive(V: A, SQ, Depth: Depth + 1) &&
5328 isKnownNonPositive(V: B, SQ, Depth: Depth + 1);
5329
5330 return computeKnownBits(V, Q: SQ).isNonPositive();
5331}
5332
5333/// Fold (icmp pred (reduce.add X), 0) to (icmp pred' (reduce.or X), 0) when X
5334/// has lanes known to all be non-negative or all non-positive, so that
5335/// sum == 0 iff every lane is 0. Falls back to reduce.umax if reduce.or is
5336/// more expensive on the target.
5337bool VectorCombine::foldReduceAddCmpZero(Instruction &I) {
5338 CmpPredicate Pred;
5339 Value *Vec;
5340 if (!match(V: &I, P: m_ICmp(Pred,
5341 L: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::vector_reduce_add>(
5342 Ops: m_Value(V&: Vec))),
5343 R: m_Zero())))
5344 return false;
5345
5346 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
5347 if (!VecTy || VecTy->getNumElements() < 2)
5348 return false;
5349
5350 SimplifyQuery Q = SQ.getWithInstruction(I: &I);
5351 bool IsNonNegative = isKnownNonNegative(V: Vec, SQ: Q);
5352 bool IsNonPositive = !IsNonNegative && isKnownNonPositive(V: Vec, SQ: Q);
5353 if (!IsNonNegative && !IsNonPositive)
5354 return false;
5355
5356 // Summing NumElts lanes can consume up to log2(NumElts) sign bits. Require
5357 // strictly more headroom than that so the sum cannot wrap to zero.
5358 unsigned NumElts = VecTy->getNumElements();
5359 unsigned NumSignBits = ComputeNumSignBits(Op: Vec, DL: *DL, AC: SQ.AC, CtxI: &I, DT: &DT);
5360 if (Log2_32(Value: NumElts) >= NumSignBits)
5361 return false;
5362
5363 ICmpInst::Predicate NewPred;
5364 switch (Pred) {
5365 case ICmpInst::ICMP_EQ:
5366 case ICmpInst::ICMP_ULE:
5367 case ICmpInst::ICMP_SLE:
5368 case ICmpInst::ICMP_SGE:
5369 NewPred = ICmpInst::ICMP_EQ;
5370 break;
5371 case ICmpInst::ICMP_NE:
5372 case ICmpInst::ICMP_UGT:
5373 case ICmpInst::ICMP_SGT:
5374 case ICmpInst::ICMP_SLT:
5375 NewPred = ICmpInst::ICMP_NE;
5376 break;
5377 default:
5378 return false;
5379 }
5380
5381 // SGT and SLE on a non-positive tree, and SLT and SGE on a non-negative
5382 // tree, are tautologies (always true or always false). Leave those to
5383 // InstCombine rather than mapping them here. Remaining signed inequalities
5384 // also need one extra sign bit so the sum cannot flip sign.
5385 if (!IsNonNegative &&
5386 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE))
5387 return false;
5388 if (!IsNonPositive &&
5389 (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE))
5390 return false;
5391 if ((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE ||
5392 Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) &&
5393 Log2_32(Value: NumElts) >= NumSignBits - 1)
5394 return false;
5395
5396 InstructionCost OrigCost = TTI.getArithmeticReductionCost(
5397 Opcode: Instruction::Add, Ty: VecTy, FMF: std::nullopt, CostKind);
5398 InstructionCost OrCost = TTI.getArithmeticReductionCost(
5399 Opcode: Instruction::Or, Ty: VecTy, FMF: std::nullopt, CostKind);
5400 InstructionCost UmaxCost = TTI.getMinMaxReductionCost(
5401 IID: Intrinsic::umax, Ty: VecTy, FMF: FastMathFlags(), CostKind);
5402 if (!OrCost.isValid() && !UmaxCost.isValid())
5403 return false;
5404 bool UseOr = OrCost.isValid() && (!UmaxCost.isValid() || OrCost <= UmaxCost);
5405 InstructionCost AltCost = UseOr ? OrCost : UmaxCost;
5406 if (AltCost > OrigCost)
5407 return false;
5408
5409 Builder.SetInsertPoint(&I);
5410 Value *NewReduce = UseOr ? Builder.CreateOrReduce(Src: Vec)
5411 : Builder.CreateIntrinsic(
5412 ID: Intrinsic::vector_reduce_umax, OverloadTypes: {VecTy}, Args: {Vec});
5413 Value *NewCmp = Builder.CreateICmp(
5414 P: NewPred, LHS: NewReduce, RHS: ConstantInt::getNullValue(Ty: VecTy->getScalarType()));
5415 replaceValue(Old&: I, New&: *NewCmp);
5416 return true;
5417}
5418
5419/// Returns true if this ShuffleVectorInst eventually feeds into a
5420/// vector reduction intrinsic (e.g., vector_reduce_add) by only following
5421/// chains of shuffles and binary operators (in any combination/order).
5422/// The search does not go deeper than the given Depth.
5423static bool feedsIntoVectorReduction(ShuffleVectorInst *SVI) {
5424 constexpr unsigned MaxVisited = 32;
5425 SmallPtrSet<Instruction *, 8> Visited;
5426 SmallVector<Instruction *, 4> WorkList;
5427 bool FoundReduction = false;
5428
5429 WorkList.push_back(Elt: SVI);
5430 while (!WorkList.empty()) {
5431 Instruction *I = WorkList.pop_back_val();
5432 for (User *U : I->users()) {
5433 auto *UI = cast<Instruction>(Val: U);
5434 if (!UI || !Visited.insert(Ptr: UI).second)
5435 continue;
5436 if (Visited.size() > MaxVisited)
5437 return false;
5438 if (auto *II = dyn_cast<IntrinsicInst>(Val: UI)) {
5439 // More than one reduction reached
5440 if (FoundReduction)
5441 return false;
5442 switch (II->getIntrinsicID()) {
5443 case Intrinsic::vector_reduce_add:
5444 case Intrinsic::vector_reduce_mul:
5445 case Intrinsic::vector_reduce_and:
5446 case Intrinsic::vector_reduce_or:
5447 case Intrinsic::vector_reduce_xor:
5448 case Intrinsic::vector_reduce_smin:
5449 case Intrinsic::vector_reduce_smax:
5450 case Intrinsic::vector_reduce_umin:
5451 case Intrinsic::vector_reduce_umax:
5452 FoundReduction = true;
5453 continue;
5454 default:
5455 return false;
5456 }
5457 }
5458
5459 if (!isa<BinaryOperator>(Val: UI) && !isa<ShuffleVectorInst>(Val: UI))
5460 return false;
5461
5462 WorkList.emplace_back(Args&: UI);
5463 }
5464 }
5465 return FoundReduction;
5466}
5467
5468/// This method looks for groups of shuffles acting on binops, of the form:
5469/// %x = shuffle ...
5470/// %y = shuffle ...
5471/// %a = binop %x, %y
5472/// %b = binop %x, %y
5473/// shuffle %a, %b, selectmask
5474/// We may, especially if the shuffle is wider than legal, be able to convert
5475/// the shuffle to a form where only parts of a and b need to be computed. On
5476/// architectures with no obvious "select" shuffle, this can reduce the total
5477/// number of operations if the target reports them as cheaper.
5478bool VectorCombine::foldSelectShuffle(Instruction &I, bool FromReduction) {
5479 auto *SVI = cast<ShuffleVectorInst>(Val: &I);
5480 auto *VT = cast<FixedVectorType>(Val: I.getType());
5481 auto *Op0 = dyn_cast<Instruction>(Val: SVI->getOperand(i_nocapture: 0));
5482 auto *Op1 = dyn_cast<Instruction>(Val: SVI->getOperand(i_nocapture: 1));
5483 if (!Op0 || !Op1 || Op0 == Op1 || !Op0->isBinaryOp() || !Op1->isBinaryOp() ||
5484 VT != Op0->getType())
5485 return false;
5486
5487 auto *SVI0A = dyn_cast<Instruction>(Val: Op0->getOperand(i: 0));
5488 auto *SVI0B = dyn_cast<Instruction>(Val: Op0->getOperand(i: 1));
5489 auto *SVI1A = dyn_cast<Instruction>(Val: Op1->getOperand(i: 0));
5490 auto *SVI1B = dyn_cast<Instruction>(Val: Op1->getOperand(i: 1));
5491 SmallPtrSet<Instruction *, 4> InputShuffles({SVI0A, SVI0B, SVI1A, SVI1B});
5492 auto checkSVNonOpUses = [&](Instruction *I) {
5493 if (!I || I->getOperand(i: 0)->getType() != VT)
5494 return true;
5495 return any_of(Range: I->users(), P: [&](User *U) {
5496 return U != Op0 && U != Op1 &&
5497 !(isa<ShuffleVectorInst>(Val: U) &&
5498 (InputShuffles.contains(Ptr: cast<Instruction>(Val: U)) ||
5499 isInstructionTriviallyDead(I: cast<Instruction>(Val: U))));
5500 });
5501 };
5502 if (checkSVNonOpUses(SVI0A) || checkSVNonOpUses(SVI0B) ||
5503 checkSVNonOpUses(SVI1A) || checkSVNonOpUses(SVI1B))
5504 return false;
5505
5506 // Collect all the uses that are shuffles that we can transform together. We
5507 // may not have a single shuffle, but a group that can all be transformed
5508 // together profitably.
5509 SmallVector<ShuffleVectorInst *> Shuffles;
5510 auto collectShuffles = [&](Instruction *I) {
5511 for (auto *U : I->users()) {
5512 auto *SV = dyn_cast<ShuffleVectorInst>(Val: U);
5513 if (!SV || SV->getType() != VT)
5514 return false;
5515 if ((SV->getOperand(i_nocapture: 0) != Op0 && SV->getOperand(i_nocapture: 0) != Op1) ||
5516 (SV->getOperand(i_nocapture: 1) != Op0 && SV->getOperand(i_nocapture: 1) != Op1))
5517 return false;
5518 if (!llvm::is_contained(Range&: Shuffles, Element: SV))
5519 Shuffles.push_back(Elt: SV);
5520 }
5521 return true;
5522 };
5523 if (!collectShuffles(Op0) || !collectShuffles(Op1))
5524 return false;
5525 // From a reduction, we need to be processing a single shuffle, otherwise the
5526 // other uses will not be lane-invariant.
5527 if (FromReduction && Shuffles.size() > 1)
5528 return false;
5529
5530 // Add any shuffle uses for the shuffles we have found, to include them in our
5531 // cost calculations.
5532 if (!FromReduction) {
5533 for (size_t Idx = 0, E = Shuffles.size(); Idx != E; ++Idx) {
5534 for (auto *U : Shuffles[Idx]->users()) {
5535 ShuffleVectorInst *SSV = dyn_cast<ShuffleVectorInst>(Val: U);
5536 if (SSV && isa<UndefValue>(Val: SSV->getOperand(i_nocapture: 1)) && SSV->getType() == VT)
5537 Shuffles.push_back(Elt: SSV);
5538 }
5539 }
5540 }
5541
5542 // For each of the output shuffles, we try to sort all the first vector
5543 // elements to the beginning, followed by the second array elements at the
5544 // end. If the binops are legalized to smaller vectors, this may reduce total
5545 // number of binops. We compute the ReconstructMask mask needed to convert
5546 // back to the original lane order.
5547 SmallVector<std::pair<int, int>> V1, V2;
5548 SmallVector<SmallVector<int>> OrigReconstructMasks;
5549 int MaxV1Elt = 0, MaxV2Elt = 0;
5550 unsigned NumElts = VT->getNumElements();
5551 for (ShuffleVectorInst *SVN : Shuffles) {
5552 SmallVector<int> Mask;
5553 SVN->getShuffleMask(Result&: Mask);
5554
5555 // Check the operands are the same as the original, or reversed (in which
5556 // case we need to commute the mask).
5557 Value *SVOp0 = SVN->getOperand(i_nocapture: 0);
5558 Value *SVOp1 = SVN->getOperand(i_nocapture: 1);
5559 if (isa<UndefValue>(Val: SVOp1)) {
5560 auto *SSV = cast<ShuffleVectorInst>(Val: SVOp0);
5561 SVOp0 = SSV->getOperand(i_nocapture: 0);
5562 SVOp1 = SSV->getOperand(i_nocapture: 1);
5563 for (int &Elem : Mask) {
5564 if (Elem >= static_cast<int>(SSV->getShuffleMask().size()))
5565 return false;
5566 Elem = Elem < 0 ? Elem : SSV->getMaskValue(Elt: Elem);
5567 }
5568 }
5569 if (SVOp0 == Op1 && SVOp1 == Op0) {
5570 std::swap(a&: SVOp0, b&: SVOp1);
5571 ShuffleVectorInst::commuteShuffleMask(Mask, InVecNumElts: NumElts);
5572 }
5573 if (SVOp0 != Op0 || SVOp1 != Op1)
5574 return false;
5575
5576 // Calculate the reconstruction mask for this shuffle, as the mask needed to
5577 // take the packed values from Op0/Op1 and reconstructing to the original
5578 // order.
5579 SmallVector<int> ReconstructMask;
5580 for (unsigned I = 0; I < Mask.size(); I++) {
5581 if (Mask[I] < 0) {
5582 ReconstructMask.push_back(Elt: -1);
5583 } else if (Mask[I] < static_cast<int>(NumElts)) {
5584 MaxV1Elt = std::max(a: MaxV1Elt, b: Mask[I]);
5585 auto It = find_if(Range&: V1, P: [&](const std::pair<int, int> &A) {
5586 return Mask[I] == A.first;
5587 });
5588 if (It != V1.end())
5589 ReconstructMask.push_back(Elt: It - V1.begin());
5590 else {
5591 ReconstructMask.push_back(Elt: V1.size());
5592 V1.emplace_back(Args&: Mask[I], Args: V1.size());
5593 }
5594 } else {
5595 MaxV2Elt = std::max<int>(a: MaxV2Elt, b: Mask[I] - NumElts);
5596 auto It = find_if(Range&: V2, P: [&](const std::pair<int, int> &A) {
5597 return Mask[I] - static_cast<int>(NumElts) == A.first;
5598 });
5599 if (It != V2.end())
5600 ReconstructMask.push_back(Elt: NumElts + It - V2.begin());
5601 else {
5602 ReconstructMask.push_back(Elt: NumElts + V2.size());
5603 V2.emplace_back(Args: Mask[I] - NumElts, Args: NumElts + V2.size());
5604 }
5605 }
5606 }
5607
5608 // For reductions, we know that the lane ordering out doesn't alter the
5609 // result. In-order can help simplify the shuffle away.
5610 if (FromReduction)
5611 sort(C&: ReconstructMask);
5612 OrigReconstructMasks.push_back(Elt: std::move(ReconstructMask));
5613 }
5614
5615 // If the Maximum element used from V1 and V2 are not larger than the new
5616 // vectors, the vectors are already packes and performing the optimization
5617 // again will likely not help any further. This also prevents us from getting
5618 // stuck in a cycle in case the costs do not also rule it out.
5619 if (V1.empty() || V2.empty() ||
5620 (MaxV1Elt == static_cast<int>(V1.size()) - 1 &&
5621 MaxV2Elt == static_cast<int>(V2.size()) - 1))
5622 return false;
5623
5624 // GetBaseMaskValue takes one of the inputs, which may either be a shuffle, a
5625 // shuffle of another shuffle, or not a shuffle (that is treated like a
5626 // identity shuffle).
5627 auto GetBaseMaskValue = [&](Instruction *I, int M) {
5628 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5629 if (!SV)
5630 return M;
5631 if (isa<UndefValue>(Val: SV->getOperand(i_nocapture: 1)))
5632 if (auto *SSV = dyn_cast<ShuffleVectorInst>(Val: SV->getOperand(i_nocapture: 0)))
5633 if (InputShuffles.contains(Ptr: SSV))
5634 return SSV->getMaskValue(Elt: SV->getMaskValue(Elt: M));
5635 return SV->getMaskValue(Elt: M);
5636 };
5637
5638 // Attempt to sort the inputs my ascending mask values to make simpler input
5639 // shuffles and push complex shuffles down to the uses. We sort on the first
5640 // of the two input shuffle orders, to try and get at least one input into a
5641 // nice order.
5642 auto SortBase = [&](Instruction *A, std::pair<int, int> X,
5643 std::pair<int, int> Y) {
5644 int MXA = GetBaseMaskValue(A, X.first);
5645 int MYA = GetBaseMaskValue(A, Y.first);
5646 return MXA < MYA;
5647 };
5648 stable_sort(Range&: V1, C: [&](std::pair<int, int> A, std::pair<int, int> B) {
5649 return SortBase(SVI0A, A, B);
5650 });
5651 stable_sort(Range&: V2, C: [&](std::pair<int, int> A, std::pair<int, int> B) {
5652 return SortBase(SVI1A, A, B);
5653 });
5654 // Calculate our ReconstructMasks from the OrigReconstructMasks and the
5655 // modified order of the input shuffles.
5656 SmallVector<SmallVector<int>> ReconstructMasks;
5657 for (const auto &Mask : OrigReconstructMasks) {
5658 SmallVector<int> ReconstructMask;
5659 for (int M : Mask) {
5660 auto FindIndex = [](const SmallVector<std::pair<int, int>> &V, int M) {
5661 auto It = find_if(Range: V, P: [M](auto A) { return A.second == M; });
5662 assert(It != V.end() && "Expected all entries in Mask");
5663 return std::distance(first: V.begin(), last: It);
5664 };
5665 if (M < 0)
5666 ReconstructMask.push_back(Elt: -1);
5667 else if (M < static_cast<int>(NumElts)) {
5668 ReconstructMask.push_back(Elt: FindIndex(V1, M));
5669 } else {
5670 ReconstructMask.push_back(Elt: NumElts + FindIndex(V2, M));
5671 }
5672 }
5673 ReconstructMasks.push_back(Elt: std::move(ReconstructMask));
5674 }
5675
5676 // Calculate the masks needed for the new input shuffles, which get padded
5677 // with undef
5678 SmallVector<int> V1A, V1B, V2A, V2B;
5679 for (unsigned I = 0; I < V1.size(); I++) {
5680 V1A.push_back(Elt: GetBaseMaskValue(SVI0A, V1[I].first));
5681 V1B.push_back(Elt: GetBaseMaskValue(SVI0B, V1[I].first));
5682 }
5683 for (unsigned I = 0; I < V2.size(); I++) {
5684 V2A.push_back(Elt: GetBaseMaskValue(SVI1A, V2[I].first));
5685 V2B.push_back(Elt: GetBaseMaskValue(SVI1B, V2[I].first));
5686 }
5687 while (V1A.size() < NumElts) {
5688 V1A.push_back(Elt: PoisonMaskElem);
5689 V1B.push_back(Elt: PoisonMaskElem);
5690 }
5691 while (V2A.size() < NumElts) {
5692 V2A.push_back(Elt: PoisonMaskElem);
5693 V2B.push_back(Elt: PoisonMaskElem);
5694 }
5695
5696 auto AddShuffleCost = [&](InstructionCost C, Instruction *I) {
5697 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5698 if (!SV)
5699 return C;
5700 return C + TTI.getShuffleCost(Kind: isa<UndefValue>(Val: SV->getOperand(i_nocapture: 1))
5701 ? TTI::SK_PermuteSingleSrc
5702 : TTI::SK_PermuteTwoSrc,
5703 DstTy: VT, SrcTy: VT, CostKind, Mask: SV->getShuffleMask());
5704 };
5705 auto AddShuffleMaskCost = [&](InstructionCost C, ArrayRef<int> Mask) {
5706 return C +
5707 TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: VT, SrcTy: VT, CostKind, Mask);
5708 };
5709
5710 unsigned ElementSize = VT->getElementType()->getPrimitiveSizeInBits();
5711 unsigned MaxVectorSize =
5712 TTI.getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector);
5713 unsigned MaxElementsInVector = MaxVectorSize / ElementSize;
5714 if (MaxElementsInVector == 0)
5715 return false;
5716 // When there are multiple shufflevector operations on the same input,
5717 // especially when the vector length is larger than the register size,
5718 // identical shuffle patterns may occur across different groups of elements.
5719 // To avoid overestimating the cost by counting these repeated shuffles more
5720 // than once, we only account for unique shuffle patterns. This adjustment
5721 // prevents inflated costs in the cost model for wide vectors split into
5722 // several register-sized groups.
5723 std::set<SmallVector<int, 4>> UniqueShuffles;
5724 auto AddShuffleMaskAdjustedCost = [&](InstructionCost C, ArrayRef<int> Mask) {
5725 // Compute the cost for performing the shuffle over the full vector.
5726 auto ShuffleCost =
5727 TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: VT, SrcTy: VT, CostKind, Mask);
5728 unsigned NumFullVectors = Mask.size() / MaxElementsInVector;
5729 if (NumFullVectors < 2)
5730 return C + ShuffleCost;
5731 SmallVector<int, 4> SubShuffle(MaxElementsInVector);
5732 unsigned NumUniqueGroups = 0;
5733 unsigned NumGroups = Mask.size() / MaxElementsInVector;
5734 // For each group of MaxElementsInVector contiguous elements,
5735 // collect their shuffle pattern and insert into the set of unique patterns.
5736 for (unsigned I = 0; I < NumFullVectors; ++I) {
5737 for (unsigned J = 0; J < MaxElementsInVector; ++J)
5738 SubShuffle[J] = Mask[MaxElementsInVector * I + J];
5739 if (UniqueShuffles.insert(x: SubShuffle).second)
5740 NumUniqueGroups += 1;
5741 }
5742 return C + ShuffleCost * NumUniqueGroups / NumGroups;
5743 };
5744 auto AddShuffleAdjustedCost = [&](InstructionCost C, Instruction *I) {
5745 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5746 if (!SV)
5747 return C;
5748 SmallVector<int, 16> Mask;
5749 SV->getShuffleMask(Result&: Mask);
5750 return AddShuffleMaskAdjustedCost(C, Mask);
5751 };
5752 // Check that input consists of ShuffleVectors applied to the same input
5753 auto AllShufflesHaveSameOperands =
5754 [](SmallPtrSetImpl<Instruction *> &InputShuffles) {
5755 if (InputShuffles.size() < 2)
5756 return false;
5757 ShuffleVectorInst *FirstSV =
5758 dyn_cast<ShuffleVectorInst>(Val: *InputShuffles.begin());
5759 if (!FirstSV)
5760 return false;
5761
5762 Value *In0 = FirstSV->getOperand(i_nocapture: 0), *In1 = FirstSV->getOperand(i_nocapture: 1);
5763 return std::all_of(
5764 first: std::next(x: InputShuffles.begin()), last: InputShuffles.end(),
5765 pred: [&](Instruction *I) {
5766 ShuffleVectorInst *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5767 return SV && SV->getOperand(i_nocapture: 0) == In0 && SV->getOperand(i_nocapture: 1) == In1;
5768 });
5769 };
5770
5771 // Get the costs of the shuffles + binops before and after with the new
5772 // shuffle masks.
5773 InstructionCost CostBefore =
5774 TTI.getArithmeticInstrCost(Opcode: Op0->getOpcode(), Ty: VT, CostKind) +
5775 TTI.getArithmeticInstrCost(Opcode: Op1->getOpcode(), Ty: VT, CostKind);
5776 CostBefore += std::accumulate(first: Shuffles.begin(), last: Shuffles.end(),
5777 init: InstructionCost(0), binary_op: AddShuffleCost);
5778 if (AllShufflesHaveSameOperands(InputShuffles)) {
5779 UniqueShuffles.clear();
5780 CostBefore += std::accumulate(first: InputShuffles.begin(), last: InputShuffles.end(),
5781 init: InstructionCost(0), binary_op: AddShuffleAdjustedCost);
5782 } else {
5783 CostBefore += std::accumulate(first: InputShuffles.begin(), last: InputShuffles.end(),
5784 init: InstructionCost(0), binary_op: AddShuffleCost);
5785 }
5786
5787 // The new binops will be unused for lanes past the used shuffle lengths.
5788 // These types attempt to get the correct cost for that from the target.
5789 FixedVectorType *Op0SmallVT =
5790 FixedVectorType::get(ElementType: VT->getScalarType(), NumElts: V1.size());
5791 FixedVectorType *Op1SmallVT =
5792 FixedVectorType::get(ElementType: VT->getScalarType(), NumElts: V2.size());
5793 InstructionCost CostAfter =
5794 TTI.getArithmeticInstrCost(Opcode: Op0->getOpcode(), Ty: Op0SmallVT, CostKind) +
5795 TTI.getArithmeticInstrCost(Opcode: Op1->getOpcode(), Ty: Op1SmallVT, CostKind);
5796 UniqueShuffles.clear();
5797 CostAfter += std::accumulate(first: ReconstructMasks.begin(), last: ReconstructMasks.end(),
5798 init: InstructionCost(0), binary_op: AddShuffleMaskAdjustedCost);
5799 std::set<SmallVector<int>> OutputShuffleMasks({V1A, V1B, V2A, V2B});
5800 CostAfter +=
5801 std::accumulate(first: OutputShuffleMasks.begin(), last: OutputShuffleMasks.end(),
5802 init: InstructionCost(0), binary_op: AddShuffleMaskCost);
5803
5804 LLVM_DEBUG(dbgs() << "Found a binop select shuffle pattern: " << I << "\n");
5805 LLVM_DEBUG(dbgs() << " CostBefore: " << CostBefore
5806 << " vs CostAfter: " << CostAfter << "\n");
5807 if (CostBefore < CostAfter ||
5808 (CostBefore == CostAfter && !feedsIntoVectorReduction(SVI)))
5809 return false;
5810
5811 // The cost model has passed, create the new instructions.
5812 auto GetShuffleOperand = [&](Instruction *I, unsigned Op) -> Value * {
5813 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5814 if (!SV)
5815 return I;
5816 if (isa<UndefValue>(Val: SV->getOperand(i_nocapture: 1)))
5817 if (auto *SSV = dyn_cast<ShuffleVectorInst>(Val: SV->getOperand(i_nocapture: 0)))
5818 if (InputShuffles.contains(Ptr: SSV))
5819 return SSV->getOperand(i_nocapture: Op);
5820 return SV->getOperand(i_nocapture: Op);
5821 };
5822 Builder.SetInsertPoint(*SVI0A->getInsertionPointAfterDef());
5823 Value *NSV0A = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI0A, 0),
5824 V2: GetShuffleOperand(SVI0A, 1), Mask: V1A);
5825 Builder.SetInsertPoint(*SVI0B->getInsertionPointAfterDef());
5826 Value *NSV0B = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI0B, 0),
5827 V2: GetShuffleOperand(SVI0B, 1), Mask: V1B);
5828 Builder.SetInsertPoint(*SVI1A->getInsertionPointAfterDef());
5829 Value *NSV1A = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI1A, 0),
5830 V2: GetShuffleOperand(SVI1A, 1), Mask: V2A);
5831 Builder.SetInsertPoint(*SVI1B->getInsertionPointAfterDef());
5832 Value *NSV1B = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI1B, 0),
5833 V2: GetShuffleOperand(SVI1B, 1), Mask: V2B);
5834 Builder.SetInsertPoint(Op0);
5835 Value *NOp0 = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Op0->getOpcode(),
5836 LHS: NSV0A, RHS: NSV0B);
5837 if (auto *I = dyn_cast<Instruction>(Val: NOp0))
5838 I->copyIRFlags(V: Op0, IncludeWrapFlags: true);
5839 Builder.SetInsertPoint(Op1);
5840 Value *NOp1 = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Op1->getOpcode(),
5841 LHS: NSV1A, RHS: NSV1B);
5842 if (auto *I = dyn_cast<Instruction>(Val: NOp1))
5843 I->copyIRFlags(V: Op1, IncludeWrapFlags: true);
5844
5845 for (int S = 0, E = ReconstructMasks.size(); S != E; S++) {
5846 Builder.SetInsertPoint(Shuffles[S]);
5847 Value *NSV = Builder.CreateShuffleVector(V1: NOp0, V2: NOp1, Mask: ReconstructMasks[S]);
5848 replaceValue(Old&: *Shuffles[S], New&: *NSV, Erase: false);
5849 }
5850
5851 return true;
5852}
5853
5854/// Check if instruction depends on ZExt and this ZExt can be moved after the
5855/// instruction. Move ZExt if it is profitable. For example:
5856/// logic(zext(x),y) -> zext(logic(x,trunc(y)))
5857/// lshr((zext(x),y) -> zext(lshr(x,trunc(y)))
5858/// Cost model calculations takes into account if zext(x) has other users and
5859/// whether it can be propagated through them too.
5860bool VectorCombine::shrinkType(Instruction &I) {
5861 Value *ZExted, *OtherOperand;
5862 if (!match(V: &I, P: m_c_BitwiseLogic(L: m_ZExt(Op: m_Value(V&: ZExted)),
5863 R: m_Value(V&: OtherOperand))) &&
5864 !match(V: &I, P: m_LShr(L: m_ZExt(Op: m_Value(V&: ZExted)), R: m_Value(V&: OtherOperand))))
5865 return false;
5866
5867 Value *ZExtOperand = I.getOperand(i: I.getOperand(i: 0) == OtherOperand ? 1 : 0);
5868
5869 auto *BigTy = cast<FixedVectorType>(Val: I.getType());
5870 auto *SmallTy = cast<FixedVectorType>(Val: ZExted->getType());
5871 unsigned BW = SmallTy->getElementType()->getPrimitiveSizeInBits();
5872
5873 if (I.getOpcode() == Instruction::LShr) {
5874 // Check that the shift amount is less than the number of bits in the
5875 // smaller type. Otherwise, the smaller lshr will return a poison value.
5876 KnownBits ShAmtKB = computeKnownBits(V: I.getOperand(i: 1), DL: *DL);
5877 if (ShAmtKB.getMaxValue().uge(RHS: BW))
5878 return false;
5879 } else {
5880 // Check that the expression overall uses at most the same number of bits as
5881 // ZExted
5882 KnownBits KB = computeKnownBits(V: &I, DL: *DL);
5883 if (KB.countMaxActiveBits() > BW)
5884 return false;
5885 }
5886
5887 // Calculate costs of leaving current IR as it is and moving ZExt operation
5888 // later, along with adding truncates if needed
5889 InstructionCost ZExtCost = TTI.getCastInstrCost(
5890 Opcode: Instruction::ZExt, Dst: BigTy, Src: SmallTy,
5891 CCH: TargetTransformInfo::CastContextHint::None, CostKind);
5892 InstructionCost CurrentCost = ZExtCost;
5893 InstructionCost ShrinkCost = 0;
5894
5895 // Calculate total cost and check that we can propagate through all ZExt users
5896 for (User *U : ZExtOperand->users()) {
5897 auto *UI = cast<Instruction>(Val: U);
5898 if (UI == &I) {
5899 CurrentCost +=
5900 TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: BigTy, CostKind);
5901 ShrinkCost +=
5902 TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: SmallTy, CostKind);
5903 ShrinkCost += ZExtCost;
5904 continue;
5905 }
5906
5907 if (!Instruction::isBinaryOp(Opcode: UI->getOpcode()))
5908 return false;
5909
5910 // Check if we can propagate ZExt through its other users
5911 KnownBits KB = computeKnownBits(V: UI, DL: *DL);
5912 if (KB.countMaxActiveBits() > BW)
5913 return false;
5914
5915 CurrentCost += TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: BigTy, CostKind);
5916 ShrinkCost +=
5917 TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: SmallTy, CostKind);
5918 ShrinkCost += ZExtCost;
5919 }
5920
5921 // If the other instruction operand is not a constant, we'll need to
5922 // generate a truncate instruction. So we have to adjust cost
5923 if (!isa<Constant>(Val: OtherOperand))
5924 ShrinkCost += TTI.getCastInstrCost(
5925 Opcode: Instruction::Trunc, Dst: SmallTy, Src: BigTy,
5926 CCH: TargetTransformInfo::CastContextHint::None, CostKind);
5927
5928 // If the cost of shrinking types and leaving the IR is the same, we'll lean
5929 // towards modifying the IR because shrinking opens opportunities for other
5930 // shrinking optimisations.
5931 if (ShrinkCost > CurrentCost)
5932 return false;
5933
5934 Builder.SetInsertPoint(&I);
5935 Value *Op0 = ZExted;
5936 Value *Op1 = Builder.CreateTrunc(V: OtherOperand, DestTy: SmallTy);
5937 // Keep the order of operands the same
5938 if (I.getOperand(i: 0) == OtherOperand)
5939 std::swap(a&: Op0, b&: Op1);
5940 Value *NewBinOp =
5941 Builder.CreateBinOp(Opc: (Instruction::BinaryOps)I.getOpcode(), LHS: Op0, RHS: Op1);
5942 if (auto *NewBinOpI = dyn_cast<Instruction>(Val: NewBinOp)) {
5943 NewBinOpI->copyIRFlags(V: &I);
5944 NewBinOpI->copyMetadata(SrcInst: I);
5945 }
5946 Value *NewZExtr = Builder.CreateZExt(V: NewBinOp, DestTy: BigTy);
5947 replaceValue(Old&: I, New&: *NewZExtr);
5948 return true;
5949}
5950
5951/// insert (DstVec, (extract SrcVec, ExtIdx), InsIdx) -->
5952/// shuffle (DstVec, SrcVec, Mask)
5953bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
5954 Value *DstVec, *SrcVec;
5955 uint64_t ExtIdx, InsIdx;
5956 if (!match(V: &I,
5957 P: m_InsertElt(Val: m_Value(V&: DstVec),
5958 Elt: m_ExtractElt(Val: m_Value(V&: SrcVec), Idx: m_ConstantInt(V&: ExtIdx)),
5959 Idx: m_ConstantInt(V&: InsIdx))))
5960 return false;
5961
5962 auto *DstVecTy = dyn_cast<FixedVectorType>(Val: I.getType());
5963 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcVec->getType());
5964 // We can try combining vectors with different element sizes.
5965 if (!DstVecTy || !SrcVecTy ||
5966 SrcVecTy->getElementType() != DstVecTy->getElementType())
5967 return false;
5968
5969 unsigned NumDstElts = DstVecTy->getNumElements();
5970 unsigned NumSrcElts = SrcVecTy->getNumElements();
5971 if (InsIdx >= NumDstElts || ExtIdx >= NumSrcElts || NumDstElts == 1)
5972 return false;
5973
5974 // Insertion into poison is a cheaper single operand shuffle.
5975 TargetTransformInfo::ShuffleKind SK;
5976 SmallVector<int> Mask(NumDstElts, PoisonMaskElem);
5977
5978 bool NeedExpOrNarrow = NumSrcElts != NumDstElts;
5979 bool NeedDstSrcSwap = isa<PoisonValue>(Val: DstVec) && !isa<UndefValue>(Val: SrcVec);
5980 if (NeedDstSrcSwap) {
5981 SK = TargetTransformInfo::SK_PermuteSingleSrc;
5982 Mask[InsIdx] = ExtIdx % NumDstElts;
5983 std::swap(a&: DstVec, b&: SrcVec);
5984 } else {
5985 SK = TargetTransformInfo::SK_PermuteTwoSrc;
5986 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
5987 Mask[InsIdx] = (ExtIdx % NumDstElts) + NumDstElts;
5988 }
5989
5990 // Cost
5991 auto *Ins = cast<InsertElementInst>(Val: &I);
5992 auto *Ext = cast<ExtractElementInst>(Val: I.getOperand(i: 1));
5993 InstructionCost InsCost =
5994 TTI.getVectorInstrCost(I: *Ins, Val: DstVecTy, CostKind, Index: InsIdx);
5995 InstructionCost ExtCost =
5996 TTI.getVectorInstrCost(I: *Ext, Val: DstVecTy, CostKind, Index: ExtIdx);
5997 InstructionCost OldCost = ExtCost + InsCost;
5998
5999 InstructionCost NewCost = 0;
6000 SmallVector<int> ExtToVecMask;
6001 if (!NeedExpOrNarrow) {
6002 // Ignore 'free' identity insertion shuffle.
6003 // TODO: getShuffleCost should return TCC_Free for Identity shuffles.
6004 if (!ShuffleVectorInst::isIdentityMask(Mask, NumSrcElts))
6005 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: DstVecTy, CostKind, Mask, Index: 0,
6006 SubTp: nullptr, Args: {DstVec, SrcVec});
6007 } else {
6008 // When creating a length-changing-vector, always try to keep the relevant
6009 // element in an equivalent position, so that bulk shuffles are more likely
6010 // to be useful.
6011 ExtToVecMask.assign(NumElts: NumDstElts, Elt: PoisonMaskElem);
6012 ExtToVecMask[ExtIdx % NumDstElts] = ExtIdx;
6013 // Add cost for expanding or narrowing
6014 NewCost = TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
6015 DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask: ExtToVecMask);
6016 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: DstVecTy, CostKind, Mask);
6017 }
6018
6019 if (!Ext->hasOneUse())
6020 NewCost += ExtCost;
6021
6022 LLVM_DEBUG(dbgs() << "Found a insert/extract shuffle-like pair: " << I
6023 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
6024 << "\n");
6025
6026 if (OldCost < NewCost)
6027 return false;
6028
6029 if (NeedExpOrNarrow) {
6030 if (!NeedDstSrcSwap)
6031 SrcVec = Builder.CreateShuffleVector(V: SrcVec, Mask: ExtToVecMask);
6032 else
6033 DstVec = Builder.CreateShuffleVector(V: DstVec, Mask: ExtToVecMask);
6034 }
6035
6036 // Canonicalize undef param to RHS to help further folds.
6037 if (isa<UndefValue>(Val: DstVec) && !isa<UndefValue>(Val: SrcVec)) {
6038 ShuffleVectorInst::commuteShuffleMask(Mask, InVecNumElts: NumDstElts);
6039 std::swap(a&: DstVec, b&: SrcVec);
6040 }
6041
6042 Value *Shuf = Builder.CreateShuffleVector(V1: DstVec, V2: SrcVec, Mask);
6043 replaceValue(Old&: I, New&: *Shuf);
6044
6045 return true;
6046}
6047
6048/// Try to replace a chain of insertelements of parts of the same scalar with a
6049/// bitcast and a shuffle (little endian):
6050/// insert (insert poison, (trunc (lshr X, 32)), 0), (trunc X), 1 -->
6051/// shuffle (bitcast X to <2 x i32>), poison, <1, 0>
6052bool VectorCombine::foldInsertScalarPartsToShuffle(Instruction &I) {
6053 auto *VecTy = dyn_cast<FixedVectorType>(Val: I.getType());
6054 if (!VecTy)
6055 return false;
6056
6057 // Start from the last insertelement of the chain.
6058 if (I.hasOneUse() && isa<InsertElementInst>(Val: I.user_back()))
6059 return false;
6060
6061 Type *EltTy = VecTy->getElementType();
6062 if ((!EltTy->isIntegerTy() && !EltTy->isIEEELikeFPTy()) ||
6063 !DL->typeSizeEqualsStoreSize(Ty: EltTy))
6064 return false;
6065 unsigned EltBits = EltTy->getPrimitiveSizeInBits();
6066 unsigned NumElts = VecTy->getNumElements();
6067
6068 Value *Src = nullptr;
6069 unsigned NumSrcElts = 0;
6070 SmallVector<int> Mask(NumElts, PoisonMaskElem);
6071 APInt DemandedElts = APInt::getZero(numBits: NumElts);
6072 InstructionCost OldCost = 0;
6073 Value *Vec = &I;
6074 while (auto *Ins = dyn_cast<InsertElementInst>(Val: Vec)) {
6075 if (Ins != &I && !Ins->hasOneUse())
6076 return false;
6077 uint64_t Idx;
6078 if (!match(V: Ins->getOperand(i_nocapture: 2), P: m_ConstantInt(V&: Idx)) || Idx >= NumElts)
6079 return false;
6080 Vec = Ins->getOperand(i_nocapture: 0);
6081 // A later insert to the same element overrides this one.
6082 if (DemandedElts[Idx])
6083 continue;
6084 DemandedElts.setBit(Idx);
6085
6086 // Match (bitcast (trunc (lshr X, ShAmt))), the bitcast and shift being
6087 // optional.
6088 Value *Elt = Ins->getOperand(i_nocapture: 1);
6089 Value *Trunc = Elt;
6090 match(V: Trunc, P: m_BitCast(Op: m_Value(V&: Trunc)));
6091 Value *X;
6092 if (!match(V: Trunc, P: m_Trunc(Op: m_Value(V&: X))) || !X->getType()->isIntegerTy() ||
6093 Trunc->getType()->getPrimitiveSizeInBits() != EltBits)
6094 return false;
6095 Value *Shift = nullptr;
6096 uint64_t ShAmt = 0;
6097 if (match(V: X, P: m_LShr(L: m_Value(), R: m_ConstantInt(V&: ShAmt)))) {
6098 Shift = X;
6099 X = cast<Instruction>(Val: Shift)->getOperand(i: 0);
6100 }
6101
6102 if (!Src) {
6103 unsigned SrcBits = X->getType()->getIntegerBitWidth();
6104 if (SrcBits % EltBits)
6105 return false;
6106 Src = X;
6107 NumSrcElts = SrcBits / EltBits;
6108 } else if (X != Src) {
6109 return false;
6110 }
6111 uint64_t Part = ShAmt / EltBits;
6112 if (ShAmt % EltBits || Part >= NumSrcElts)
6113 return false;
6114 Mask[Idx] = DL->isBigEndian() ? NumSrcElts - 1 - Part : Part;
6115
6116 // The scalar ops die with the chain if it is their only user.
6117 for (Value *V : {Elt == Trunc ? nullptr : Elt, Trunc, Shift}) {
6118 if (!V)
6119 continue;
6120 if (!V->hasOneUse())
6121 break;
6122 OldCost += TTI.getInstructionCost(U: cast<Instruction>(Val: V), CostKind);
6123 }
6124 }
6125 // Elements that are not inserted become poison, so the base must be poison
6126 // unless every element is inserted.
6127 if (!Src || (!isa<PoisonValue>(Val: Vec) && !DemandedElts.isAllOnes()))
6128 return false;
6129 // Inserting a single part is a scalar insert or a splat, whose canonical
6130 // insertelement (+ splat shuffle) form is better left alone.
6131 if (all_equal(
6132 Range: make_filter_range(Range&: Mask, Pred: [](int M) { return M != PoisonMaskElem; })))
6133 return false;
6134
6135 OldCost += TTI.getScalarizationOverhead(Ty: VecTy, DemandedElts, /*Insert=*/true,
6136 /*Extract=*/false, CostKind);
6137
6138 auto *SrcVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: NumSrcElts);
6139 InstructionCost NewCost =
6140 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: SrcVecTy, Src: Src->getType(),
6141 CCH: TTI::CastContextHint::None, CostKind);
6142 bool IsIdentity = NumSrcElts == NumElts &&
6143 ShuffleVectorInst::isIdentityMask(Mask, NumSrcElts);
6144 if (!IsIdentity)
6145 NewCost += TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: VecTy, SrcTy: SrcVecTy,
6146 CostKind, Mask);
6147
6148 LLVM_DEBUG(dbgs() << "Found an insertelement chain of scalar parts: " << I
6149 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
6150 << "\n");
6151 if (!OldCost.isValid() || !NewCost.isValid() || NewCost >= OldCost)
6152 return false;
6153
6154 Value *Cast = Builder.CreateBitCast(V: Src, DestTy: SrcVecTy);
6155 Value *Shuf = IsIdentity ? Cast : Builder.CreateShuffleVector(V: Cast, Mask);
6156 replaceValue(Old&: I, New&: *Shuf);
6157 return true;
6158}
6159
6160/// Return the number of data operands of \p Inst.
6161static unsigned getNumDataOperands(const Instruction *Inst) {
6162 if (auto *CB = dyn_cast<CallBase>(Val: Inst))
6163 return CB->arg_size(); // Exclude callee operand and bundles.
6164 return Inst->getNumOperands();
6165}
6166
6167/// Return true if \p Inst is an elementwise operation that can be rebuilt at a
6168/// wider element count.
6169static bool isSupportedElementwise(Instruction *Inst) {
6170 auto *ResultTy = dyn_cast<VectorType>(Val: Inst->getType());
6171 if (!ResultTy || !isSafeToSpeculativelyExecute(I: Inst))
6172 return false;
6173
6174 if (auto *II = dyn_cast<IntrinsicInst>(Val: Inst)) {
6175 if (II->hasOperandBundles() ||
6176 !isTriviallyVectorizable(ID: II->getIntrinsicID()))
6177 return false;
6178 } else if (!isa<BinaryOperator, UnaryOperator, CastInst, CmpInst, SelectInst,
6179 FreezeInst>(Val: Inst)) {
6180 return false;
6181 }
6182
6183 // Reject operations that change the element-count.
6184 // E.g., bitcast <vscale x 4 x i16> %v to <vscale x 8 x i8>
6185 for (unsigned Op = 0, E = getNumDataOperands(Inst); Op != E; ++Op) {
6186 auto *OperandTy = dyn_cast<VectorType>(Val: Inst->getOperand(i: Op)->getType());
6187 if (OperandTy &&
6188 OperandTy->getElementCount() != ResultTy->getElementCount())
6189 return false;
6190 }
6191
6192 return true;
6193}
6194
6195/// Return the common splat value of \p Values.
6196static Value *getCommonSplatValue(ArrayRef<Value *> Values) {
6197 auto GetSplatOrScalar = [](Value *V) {
6198 return isa<VectorType>(Val: V->getType()) ? getSplatValue(V) : V;
6199 };
6200
6201 Value *CommonValue = GetSplatOrScalar(Values.front());
6202 if (!CommonValue)
6203 return nullptr;
6204 for (Value *V : Values.drop_front())
6205 if (GetSplatOrScalar(V) != CommonValue)
6206 return nullptr;
6207 return CommonValue;
6208}
6209
6210/// Return the common deinterleave intrinsic if \p Members are its extracts in
6211/// field order.
6212static IntrinsicInst *getCommonDeinterleavedSource(ArrayRef<Value *> Members) {
6213 IntrinsicInst *Deinterleave = nullptr;
6214 for (const auto &[Index, Member] : enumerate(First&: Members)) {
6215 auto *Extract = dyn_cast<ExtractValueInst>(Val: Member);
6216 if (!Extract || Extract->getNumIndices() != 1 ||
6217 *Extract->idx_begin() != Index)
6218 return nullptr;
6219
6220 auto *Current = dyn_cast<IntrinsicInst>(Val: Extract->getAggregateOperand());
6221 if (!Current || (Deinterleave && Current != Deinterleave))
6222 return nullptr;
6223 Deinterleave = Current;
6224 }
6225 unsigned Factor = Members.size();
6226 if (Deinterleave->hasOperandBundles() ||
6227 getDeinterleaveIntrinsicFactor(ID: Deinterleave->getIntrinsicID()) !=
6228 Factor ||
6229 !Deinterleave->hasNUndroppableUses(N: Factor))
6230 return nullptr;
6231 return Deinterleave;
6232}
6233
6234/// Return the operand at \p OperandIndex of each \p Members.
6235static SmallVector<Value *, 8> getInstrOperandsAtIdx(ArrayRef<Value *> Members,
6236 unsigned OperandIndex) {
6237 SmallVector<Value *, 8> Operands;
6238 for (Value *Member : Members)
6239 Operands.push_back(Elt: cast<Instruction>(Val: Member)->getOperand(i: OperandIndex));
6240 return Operands;
6241}
6242
6243/// Check whether the tree of elementwise operations each feeding \p Members
6244/// can be rebuilt at the interleaved width.
6245static bool canWidenDeinterleavedOperations(const VectorizeOptions &Opts,
6246 ArrayRef<Value *> Members,
6247 unsigned &NumScanned) {
6248 unsigned Factor = Members.size();
6249 if (getCommonDeinterleavedSource(Members))
6250 return true;
6251 if (NumScanned + Factor > Opts.vector_combine_max_scan_instrs)
6252 return false;
6253 NumScanned += Factor;
6254
6255 auto *FirstInst = dyn_cast<Instruction>(Val: Members.front());
6256 if (!FirstInst || !isSupportedElementwise(Inst: FirstInst) ||
6257 !FirstInst->getSingleUndroppableUse())
6258 return false;
6259
6260 for (Value *Member : Members.drop_front()) {
6261 auto *Inst = dyn_cast<Instruction>(Val: Member);
6262 if (!Inst || !Inst->getSingleUndroppableUse() ||
6263 !FirstInst->isSameOperationAs(I: Inst, flags: Instruction::CompareCallTargets))
6264 return false;
6265 }
6266
6267 // Scalars operands should be equal among all members.
6268 // Vector operands should be a common splat value or can be widened.
6269 for (unsigned Op = 0, E = getNumDataOperands(Inst: FirstInst); Op != E; ++Op) {
6270 SmallVector<Value *, 8> Operands = getInstrOperandsAtIdx(Members, OperandIndex: Op);
6271 if (!getCommonSplatValue(Values: Operands) &&
6272 !canWidenDeinterleavedOperations(Opts, Members: Operands, NumScanned))
6273 return false;
6274 }
6275 return true;
6276}
6277
6278static Value *createWideInstruction(Instruction *NarrowInst,
6279 ArrayRef<Value *> NewOperands,
6280 VectorType *WideResultTy,
6281 IRBuilderBase &Builder) {
6282 if (isa<BinaryOperator, UnaryOperator>(Val: NarrowInst))
6283 return Builder.CreateNAryOp(Opc: NarrowInst->getOpcode(), Ops: NewOperands);
6284 if (auto *Cast = dyn_cast<CastInst>(Val: NarrowInst))
6285 return Builder.CreateCast(Op: Cast->getOpcode(), V: NewOperands[0], DestTy: WideResultTy);
6286 if (auto *Cmp = dyn_cast<CmpInst>(Val: NarrowInst))
6287 return Builder.CreateCmp(Pred: Cmp->getPredicate(), LHS: NewOperands[0],
6288 RHS: NewOperands[1]);
6289 if (isa<SelectInst>(Val: NarrowInst))
6290 return Builder.CreateSelect(
6291 C: NewOperands[0], True: NewOperands[1], False: NewOperands[2], /*Name=*/"",
6292 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : NarrowInst);
6293 if (isa<FreezeInst>(Val: NarrowInst))
6294 return Builder.CreateFreeze(V: NewOperands[0]);
6295 if (auto *II = dyn_cast<IntrinsicInst>(Val: NarrowInst))
6296 return Builder.CreateIntrinsic(RetTy: WideResultTy, ID: II->getIntrinsicID(),
6297 Args: NewOperands);
6298 llvm_unreachable("Unsupported instruction");
6299}
6300
6301static Value *widenDeinterleavedOperations(ArrayRef<Value *> Members,
6302 ElementCount WideEC,
6303 IRBuilderBase &Builder) {
6304 if (auto *Deinterleave = getCommonDeinterleavedSource(Members)) {
6305 Value *Source = Deinterleave->getArgOperand(i: 0);
6306 assert(cast<VectorType>(Source->getType())->getElementCount() == WideEC &&
6307 "deinterleave source must have the interleaved element count");
6308 return Source;
6309 }
6310
6311 auto *NarrowInst = cast<Instruction>(Val: Members.front());
6312 unsigned NumOperands = getNumDataOperands(Inst: NarrowInst);
6313 SmallVector<Value *, 4> NewOperands;
6314 NewOperands.reserve(N: NumOperands);
6315 for (unsigned Op = 0; Op != NumOperands; ++Op) {
6316 SmallVector<Value *, 8> Operands = getInstrOperandsAtIdx(Members, OperandIndex: Op);
6317 Value *NewOperand = nullptr;
6318 if ((NewOperand = getCommonSplatValue(Values: Operands))) {
6319 if (isa<VectorType>(Val: Operands.front()->getType())) {
6320 Builder.SetCurrentDebugLocation(NarrowInst->getDebugLoc());
6321 NewOperand = Builder.CreateVectorSplat(EC: WideEC, V: NewOperand);
6322 } else {
6323 assert(all_equal(Operands) && "expected all operands to be equal");
6324 }
6325 } else {
6326 NewOperand = widenDeinterleavedOperations(Members: Operands, WideEC, Builder);
6327 }
6328 NewOperands.push_back(Elt: NewOperand);
6329 }
6330
6331 Builder.SetCurrentDebugLocation(NarrowInst->getDebugLoc());
6332 auto *WideResultTy =
6333 VectorType::get(ElementType: NarrowInst->getType()->getScalarType(), EC: WideEC);
6334 Value *NewValue =
6335 createWideInstruction(NarrowInst, NewOperands, WideResultTy, Builder);
6336 if (auto *NewInst = dyn_cast<Instruction>(Val: NewValue)) {
6337 propagateIRFlags(I: NewInst, VL: Members);
6338 propagateMetadata(I: NewInst, VL: Members);
6339 }
6340 return NewValue;
6341}
6342
6343/// Fold away vector.deinterleave/interleave intrinsics with matching trees of
6344/// elementwise operations between them.
6345///
6346/// For example:
6347/// ```
6348/// %d = call { <2 x i16>, <2 x i16> } @deinterleave2.v4i16(<4 x i16> %v)
6349/// %f0 = extractvalue { <2 x i16>, <2 x i16> } %d, 0
6350/// %f1 = extractvalue { <2 x i16>, <2 x i16> } %d, 1
6351///
6352/// %u0 = add <2 x i16> %f0, splat (i16 3)
6353/// %u1 = add <2 x i16> %f1, splat (i16 3)
6354///
6355/// %r = call <4 x i16> @interleave2.v4i16(<2 x i16> %u0, <2 x i16> %u1)
6356/// ```
6357/// Folds to:
6358/// ```
6359/// %r = add <4 x i16> %v, splat (i16 3)
6360/// ```
6361/// And with two sources:
6362/// ```
6363/// %da = call { <2 x i16>, <2 x i16> } @deinterleave2.v4i16(<4 x i16> %a)
6364/// %a0 = extractvalue { <2 x i16>, <2 x i16> } %da, 0
6365/// %a1 = extractvalue { <2 x i16>, <2 x i16> } %da, 1
6366/// %db = call { <2 x i16>, <2 x i16> } @deinterleave2.v4i16(<4 x i16> %b)
6367/// %b0 = extractvalue { <2 x i16>, <2 x i16> } %db, 0
6368/// %b1 = extractvalue { <2 x i16>, <2 x i16> } %db, 1
6369///
6370/// %m0 = mul <2 x i16> %a0, %b0
6371/// %m1 = mul <2 x i16> %a1, %b1
6372///
6373/// %r = call <4 x i16> @interleave2.v4i16(<2 x i16> %m0, <2 x i16> %m1)
6374/// ```
6375/// Folds to:
6376/// ```
6377/// %r = mul <4 x i16> %a, %b
6378/// ```
6379bool VectorCombine::foldInterleaveOfDeinterleaveChains(Instruction &I) {
6380 auto *Interleave = dyn_cast<IntrinsicInst>(Val: &I);
6381 if (!Interleave)
6382 return false;
6383
6384 unsigned Factor = getInterleaveIntrinsicFactor(ID: Interleave->getIntrinsicID());
6385 if (!Factor || Interleave->hasOperandBundles())
6386 return false;
6387
6388 SmallVector<Value *, 8> RootMembers(Interleave->args());
6389 unsigned NumScanned = 0;
6390 if (!canWidenDeinterleavedOperations(Opts, Members: RootMembers, NumScanned))
6391 return false;
6392
6393 ElementCount WideEC = cast<VectorType>(Val: I.getType())->getElementCount();
6394 Builder.SetInsertPoint(Interleave);
6395 Value *WideValue = widenDeinterleavedOperations(Members: RootMembers, WideEC, Builder);
6396 assert(WideValue->getType() == Interleave->getType());
6397 replaceValue(Old&: *Interleave, New&: *WideValue);
6398 return true;
6399}
6400
6401/// If we're interleaving 2 constant splats, for instance `<vscale x 8 x i32>
6402/// <splat of 666>` and `<vscale x 8 x i32> <splat of 777>`, we can create a
6403/// larger splat `<vscale x 8 x i64> <splat of ((777 << 32) | 666)>` first
6404/// before casting it back into `<vscale x 16 x i32>`.
6405bool VectorCombine::foldInterleaveIntrinsics(Instruction &I) {
6406 if (foldInterleaveOfDeinterleaveChains(I))
6407 return true;
6408
6409 const APInt *SplatVal0, *SplatVal1;
6410 if (!match(V: &I, P: m_Intrinsic<Intrinsic::vector_interleave2>(
6411 Ops: m_APInt(Res&: SplatVal0), Ops: m_APInt(Res&: SplatVal1))))
6412 return false;
6413
6414 LLVM_DEBUG(dbgs() << "VC: Folding interleave2 with two splats: " << I
6415 << "\n");
6416
6417 auto *VTy =
6418 cast<VectorType>(Val: cast<IntrinsicInst>(Val&: I).getArgOperand(i: 0)->getType());
6419 auto *ExtVTy = VectorType::getExtendedElementVectorType(VTy);
6420 unsigned Width = VTy->getElementType()->getIntegerBitWidth();
6421
6422 // Just in case the cost of interleave2 intrinsic and bitcast are both
6423 // invalid, in which case we want to bail out, we use <= rather
6424 // than < here. Even they both have valid and equal costs, it's probably
6425 // not a good idea to emit a high-cost constant splat.
6426 if (TTI.getInstructionCost(U: &I, CostKind) <=
6427 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: I.getType(), Src: ExtVTy,
6428 CCH: TTI::CastContextHint::None, CostKind)) {
6429 LLVM_DEBUG(dbgs() << "VC: The cost to cast from " << *ExtVTy << " to "
6430 << *I.getType() << " is too high.\n");
6431 return false;
6432 }
6433
6434 APInt NewSplatVal = SplatVal1->zext(width: Width * 2);
6435 NewSplatVal <<= Width;
6436 NewSplatVal |= SplatVal0->zext(width: Width * 2);
6437 auto *NewSplat = ConstantVector::getSplat(
6438 EC: ExtVTy->getElementCount(), Elt: ConstantInt::get(Context&: F.getContext(), V: NewSplatVal));
6439
6440 Builder.SetInsertPoint(&I);
6441 replaceValue(Old&: I, New&: *Builder.CreateBitCast(V: NewSplat, DestTy: I.getType()));
6442 return true;
6443}
6444
6445/// Given this sequence:
6446/// ```
6447/// %d = llvm.vector.deinterleave2 <vscale x 16 x i32> %v
6448/// %f0 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d, 0
6449/// %f1 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d, 1
6450///
6451/// %low0 = and <vscale x 8 x i32> %f0, splat (i32 65535)
6452/// %low1 = shl <vscale x 8 x i32> %f1, splat (i32 16)
6453/// %merge0 = or disjoint <vscale x 8 x i32> %low0, %low1
6454///
6455/// %high0 = and <vscale x 8 x i32> %f1, splat (i32 -65536)
6456/// %high1 = lshr <vscale x 8 x i32> %f0, splat (i32 16)
6457/// %merge1 = or disjoint <vscale x 8 x i32> %high0, %high1
6458/// ```
6459/// It is actually just de-interleaving a 16-bit vector with double the
6460/// vector length. More generally speaking, it's de-interleaving on a vector
6461/// with half the element width as the original vector.
6462///
6463/// Therefore, we can turn it into:
6464/// ```
6465/// %narrow.v = bitcast <vscale x 16 x i32> %v to <vscale x 32 x i16>
6466/// %d = llvm.vector.deinterleave2 <vscale x 32 x i16> %narrow.v
6467/// %f0 = extractvalue { <vscale x 16 x i16>, <vscale x 16 x i16> } %d, 0
6468/// %f1 = extractvalue { <vscale x 16 x i16>, <vscale x 16 x i16> } %d, 1
6469///
6470/// %merge0 = bitcast <vscale x 16 x i16> %f0 to <vscale x 8 x i32>
6471/// %merge1 = bitcast <vscale x 16 x i16> %f1 to <vscale x 8 x i32>
6472/// ```
6473bool VectorCombine::foldDeinterleaveIntrinsics(Instruction &I) {
6474 // This pattern involves bitcast that is not compatible with big endian.
6475 if (DL->isBigEndian())
6476 return false;
6477
6478 using namespace PatternMatch;
6479 Value *DeinterleavedVal;
6480 if (!match(V: &I, P: m_Deinterleave2(Op: m_Value(V&: DeinterleavedVal))))
6481 return false;
6482
6483 VectorType *VecTy = cast<VectorType>(Val: DeinterleavedVal->getType());
6484 IntegerType *ElementTy = dyn_cast<IntegerType>(Val: VecTy->getElementType());
6485 if (!ElementTy)
6486 return false;
6487 unsigned ElementWidth = ElementTy->getBitWidth();
6488 if (ElementWidth < 2 || !isPowerOf2_32(Value: ElementWidth))
6489 return false;
6490 unsigned HalfElementWidth = ElementWidth / 2;
6491
6492 if (!I.hasNUses(N: 2))
6493 return false;
6494 std::array<ExtractValueInst *, 2> OrigFields{};
6495 for (User *Usr : I.users()) {
6496 auto *E = dyn_cast<ExtractValueInst>(Val: Usr);
6497 // The deinterleave result can only be used by extractions.
6498 if (!E || E->getNumIndices() != 1)
6499 return false;
6500 unsigned Idx = *E->idx_begin();
6501 // A single field cannot be extracted more than once.
6502 if (Idx >= 2 || OrigFields[Idx] || !E->hasNUses(N: 2))
6503 return false;
6504 OrigFields[Idx] = E;
6505 }
6506
6507 // Find the merge instruction (i.e. OR) first.
6508 SmallVector<Instruction *, 2> MergeInsts;
6509 for (auto *FieldUsr : OrigFields[0]->users()) {
6510 if (!FieldUsr->hasOneUse() || !isa<Instruction>(Val: FieldUsr->user_back()))
6511 return false;
6512 MergeInsts.push_back(Elt: cast<Instruction>(Val: FieldUsr->user_back()));
6513 }
6514 assert(MergeInsts.size() == 2);
6515
6516 // Pattern match bottom-up from the merge instructions.
6517 auto MatchMerge = [&](void) -> bool {
6518 APInt LoMask = APInt::getLowBitsSet(numBits: ElementWidth, loBitsSet: HalfElementWidth);
6519 APInt HiMask = APInt::getHighBitsSet(numBits: ElementWidth, hiBitsSet: HalfElementWidth);
6520 return match(V: MergeInsts[0],
6521 P: m_c_Or(L: m_And(L: m_Specific(V: OrigFields[0]), R: m_SpecificInt(V: LoMask)),
6522 R: m_Shl(L: m_Specific(V: OrigFields[1]),
6523 R: m_SpecificInt(V: HalfElementWidth)))) &&
6524 match(V: MergeInsts[1],
6525 P: m_c_Or(L: m_And(L: m_Specific(V: OrigFields[1]), R: m_SpecificInt(V: HiMask)),
6526 R: m_LShr(L: m_Specific(V: OrigFields[0]),
6527 R: m_SpecificInt(V: HalfElementWidth))));
6528 };
6529 if (!MatchMerge()) {
6530 std::swap(a&: MergeInsts[0], b&: MergeInsts[1]);
6531 if (!MatchMerge())
6532 return false;
6533 }
6534
6535 // Profitability check.
6536 InstructionCost OldCost =
6537 TTI.getInstructionCost(U: MergeInsts[0], CostKind) +
6538 TTI.getInstructionCost(U: cast<Instruction>(Val: MergeInsts[0]->getOperand(i: 0)),
6539 CostKind) +
6540 TTI.getInstructionCost(U: cast<Instruction>(Val: MergeInsts[0]->getOperand(i: 1)),
6541 CostKind);
6542 // There are two fields (assuming SHL has the same cost as LSHR).
6543 OldCost *= 2;
6544
6545 auto *NewFieldTy = VecTy->getWithNewBitWidth(NewBitWidth: HalfElementWidth);
6546 auto *NewVecTy =
6547 VectorType::getDoubleElementsVectorType(VTy: cast<VectorType>(Val: NewFieldTy));
6548 InstructionCost NewCost =
6549 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: VecTy, Src: NewVecTy,
6550 CCH: TTI::CastContextHint::None, CostKind) +
6551 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: NewFieldTy,
6552 Src: MergeInsts[0]->getType(), CCH: TTI::CastContextHint::None,
6553 CostKind) *
6554 2;
6555 if (OldCost <= NewCost || !NewCost.isValid()) {
6556 LLVM_DEBUG(
6557 dbgs() << "VC: New deinterleave2 sequence cost (" << NewCost << ")"
6558 << " is higher than that of the old one (" << OldCost << ")\n");
6559 return false;
6560 }
6561
6562 // Do the replacement.
6563 Builder.SetInsertPoint(&I);
6564 Value *NewVecCast = Builder.CreateBitCast(V: DeinterleavedVal, DestTy: NewVecTy);
6565 Value *NewDeinterleave = Builder.CreateIntrinsic(
6566 ID: Intrinsic::vector_deinterleave2, OverloadTypes: {NewVecTy}, Args: {NewVecCast});
6567 for (auto [Idx, MergeInst] : enumerate(First&: MergeInsts)) {
6568 Value *NewField = Builder.CreateExtractValue(Agg: NewDeinterleave, Idxs: Idx);
6569 NewField = Builder.CreateBitCast(V: NewField, DestTy: MergeInst->getType());
6570 replaceValue(Old&: *MergeInst, New&: *NewField);
6571 }
6572
6573 return true;
6574}
6575
6576bool VectorCombine::foldBitcastOfVPLoad(Instruction &I) {
6577 const DataLayout &DL = I.getDataLayout();
6578 auto *Cast = dyn_cast<CastInst>(Val: &I);
6579 if (!Cast || !Cast->isNoopCast(DL) || !isa<VectorType>(Val: Cast->getDestTy()))
6580 return false;
6581
6582 // Fold away bit casts of the loaded value by loading the desired type,
6583 // if the mask is all-ones.
6584 Value *EVL;
6585 auto *II = dyn_cast<VPIntrinsic>(Val: I.getOperand(i: 0));
6586 if (!II || !match(V: II, P: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::vp_load>(
6587 Ops: m_Value(), Ops: m_AllOnes(), Ops: m_Value(V&: EVL)))))
6588 return false;
6589
6590 VectorType *OrigVecTy = cast<VectorType>(Val: II->getType());
6591 Align OrigAlign =
6592 DL.getValueOrABITypeAlignment(Alignment: II->getPointerAlignment(), Ty: OrigVecTy);
6593 ElementCount OrigVecCnt = OrigVecTy->getElementCount();
6594 VectorType *NewVecTy = cast<VectorType>(Val: Cast->getDestTy());
6595 ElementCount NewVecCnt = NewVecTy->getElementCount();
6596
6597 // Right now we only support cases where the NewVec is longer, because for
6598 // cases where it's shorter, we have to be sure that EVL can be exactly
6599 // divided, otherwise it might yield incorrect results or even page faults
6600 // (if we round-up during the division).
6601 if (!(OrigVecCnt.isScalable() == NewVecCnt.isScalable() &&
6602 NewVecCnt.hasKnownScalarFactor(RHS: OrigVecCnt)))
6603 return false;
6604
6605 InstructionCost OldCost =
6606 TTI.getMemIntrinsicInstrCost(MICA: {Intrinsic::vp_load, OrigVecTy,
6607 II->getMemoryPointerParam(), false,
6608 OrigAlign},
6609 CostKind) +
6610 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: Cast->getType(), Src: OrigVecTy,
6611 CCH: TTI::CastContextHint::None, CostKind);
6612 InstructionCost NewCost = TTI.getMemIntrinsicInstrCost(
6613 MICA: {Intrinsic::vp_load, NewVecTy, II->getMemoryPointerParam(), false,
6614 OrigAlign},
6615 CostKind);
6616 LLVM_DEBUG(dbgs() << "foldBitcastOfVPLoad: OldCost=" << OldCost
6617 << " NewCost=" << NewCost << "\n");
6618 if (NewCost > OldCost || !NewCost.isValid())
6619 return false;
6620
6621 Builder.SetInsertPoint(II);
6622 unsigned Factor = NewVecCnt.getKnownScalarFactor(RHS: OrigVecCnt);
6623 Value *NewEVL = Builder.CreateNUWMul(LHS: EVL, RHS: Builder.getInt32(C: Factor));
6624 Value *NewMask = Builder.CreateVectorSplat(EC: NewVecCnt, V: Builder.getTrue());
6625 CallInst *NewVP = Builder.CreateIntrinsicWithoutFolding(
6626 RetTy: NewVecTy, ID: Intrinsic::vp_load,
6627 Args: {II->getMemoryPointerParam(), NewMask, NewEVL});
6628 // Preserve the original alignment.
6629 NewVP->addParamAttrs(
6630 ArgNo: 0, B: AttrBuilder(II->getContext()).addAlignmentAttr(Align: OrigAlign));
6631 replaceValue(Old&: *Cast, New&: *NewVP);
6632 return true;
6633}
6634/// Fold the following cases into a single byte-level bit-reverse operation
6635/// and accepts bswap and bitreverse intrinsics:
6636/// bswap(bitreverse(x)) --> bitcast(bitreverse(bitcast(x)))
6637/// bitreverse(bswap(x)) <--> bitcast(bitreverse(bitcast(x)))
6638/// The direction of the fold is cost-model driven.
6639/// Also supports:
6640/// bitcast(bitreverse(bitcast(x))) --> bitreverse(fshl(x))
6641bool VectorCombine::foldBitOrderReverseAndSwap(Instruction &I) {
6642 Value *X;
6643
6644 if (match(V: &I, P: m_BitCast(Op: m_BitReverse(Op0: m_BitCast(Op: m_Value(V&: X)))))) {
6645 Type *Ty = X->getType();
6646 Type *VecTy = I.getOperand(i: 0)->getType();
6647 // Detect the case when bitreversing every octet in X individually. Then we
6648 // can use bswap to reorder the octets before doing a single bitreverse.
6649 bool CanUseBswap =
6650 Ty->isIntegerTy() && Ty == I.getType() && isa<FixedVectorType>(Val: VecTy) &&
6651 cast<FixedVectorType>(Val: VecTy)->getElementType()->isIntegerTy(BitWidth: 8) &&
6652 Ty->getIntegerBitWidth() % 16 == 0;
6653 // Detect the case when bitreversing upper and lower half of X
6654 // individually. Then we can use fshl as a rotate operation, to swap the
6655 // halves before doing a single bitreverse.
6656 bool CanUseFshl =
6657 Ty->isIntegerTy() && Ty == I.getType() && isa<FixedVectorType>(Val: VecTy) &&
6658 cast<FixedVectorType>(Val: VecTy)->getElementType()->isIntegerTy() &&
6659 cast<FixedVectorType>(Val: VecTy)->getNumElements() == 2;
6660 if (CanUseBswap || CanUseFshl) {
6661 auto *InnerCall = dyn_cast<Instruction>(Val: I.getOperand(i: 0));
6662 if (!InnerCall)
6663 return false;
6664 auto *InnerBitCast = dyn_cast<BitCastInst>(Val: InnerCall->getOperand(i: 0));
6665 if (!InnerBitCast)
6666 return false;
6667 Constant *HalfBW = ConstantInt::get(Ty, V: Ty->getIntegerBitWidth() / 2);
6668 InstructionCost OldCost = TTI.getInstructionCost(U: InnerBitCast, CostKind) +
6669 TTI.getInstructionCost(U: InnerCall, CostKind) +
6670 TTI.getInstructionCost(U: &I, CostKind);
6671 IntrinsicCostAttributes ICABSwap(Intrinsic::bswap, Ty, {Ty});
6672 IntrinsicCostAttributes ICABFshl(Intrinsic::fshl, Ty, {X, X, HalfBW},
6673 {Ty, Ty, Ty});
6674 IntrinsicCostAttributes ICABRev(Intrinsic::bitreverse, Ty, {Ty});
6675 InstructionCost NewCost =
6676 TTI.getIntrinsicInstrCost(ICA: CanUseBswap ? ICABSwap : ICABFshl,
6677 CostKind) +
6678 TTI.getIntrinsicInstrCost(ICA: ICABRev, CostKind);
6679 if (!InnerCall->hasOneUse())
6680 NewCost += TTI.getInstructionCost(U: InnerCall, CostKind) +
6681 TTI.getInstructionCost(U: InnerBitCast, CostKind);
6682 else if (!InnerBitCast->hasOneUse())
6683 NewCost += TTI.getInstructionCost(U: InnerBitCast, CostKind);
6684 LLVM_DEBUG(dbgs() << "Found bitreverse vector roundtrip: " << I
6685 << "\n OldCost: " << OldCost
6686 << " vs NewCost: " << NewCost << "\n");
6687 if (NewCost.isValid() && NewCost < OldCost) {
6688 Builder.SetInsertPoint(&I);
6689 Value *Swap =
6690 CanUseBswap
6691 ? Builder.CreateUnaryIntrinsic(ID: Intrinsic::bswap, Op: X)
6692 : Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::fshl, Args: {X, X, HalfBW});
6693 Value *BRev = Builder.CreateUnaryIntrinsic(ID: Intrinsic::bitreverse, Op: Swap);
6694 replaceValue(Old&: I, New&: *BRev);
6695 return true;
6696 }
6697 }
6698 }
6699
6700 if (!match(V: &I, P: m_BitReverse(Op0: m_BSwap(Op0: m_Value(V&: X)))) &&
6701 !match(V: &I, P: m_BSwap(Op0: m_BitReverse(Op0: m_Value(V&: X)))))
6702 return false;
6703 Type *Ty = I.getType();
6704 Type *I8Ty = Builder.getInt8Ty();
6705 TypeSize ElementSize = DL->getTypeStoreSize(Ty);
6706 ElementCount NewVecCnt = ElementCount::get(MinVal: ElementSize.getKnownMinValue(),
6707 Scalable: ElementSize.isScalable());
6708 Type *NewVecTy = VectorType::get(ElementType: I8Ty, EC: NewVecCnt);
6709 auto *II = cast<IntrinsicInst>(Val: &I);
6710 auto *InnerII = cast<IntrinsicInst>(Val: II->getArgOperand(i: 0));
6711 // OldCost = cost of bitreverse/bswap + cost of bswap/bitreverse
6712 InstructionCost OldCost = TTI.getInstructionCost(U: II, CostKind) +
6713 TTI.getInstructionCost(U: InnerII, CostKind);
6714 // NewCost = cost of bitcast to byte vector +
6715 // cost of bitreverse/bswap on byte vector +
6716 // cost of bitcast back to original type
6717 InstructionCost CastToVecCost = TTI.getCastInstrCost(
6718 Opcode: Instruction::BitCast, Dst: NewVecTy, Src: Ty, CCH: TTI::CastContextHint::None, CostKind);
6719 InstructionCost CastToOrigCost = TTI.getCastInstrCost(
6720 Opcode: Instruction::BitCast, Dst: Ty, Src: NewVecTy, CCH: TTI::CastContextHint::None, CostKind);
6721 IntrinsicCostAttributes ICANew(Intrinsic::bitreverse, NewVecTy, {NewVecTy});
6722 InstructionCost NewIntrinsicCost =
6723 TTI.getIntrinsicInstrCost(ICA: ICANew, CostKind);
6724 InstructionCost NewCost = CastToVecCost + NewIntrinsicCost + CastToOrigCost;
6725 if (!InnerII->hasOneUse())
6726 NewCost += TTI.getInstructionCost(U: InnerII, CostKind);
6727 LLVM_DEBUG(dbgs() << "Found bitorder reverse and swap: " << I
6728 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
6729 << "\n");
6730 if (!NewCost.isValid() || NewCost >= OldCost)
6731 return false;
6732 // Perform transform: bitcast(arg, <N x i8>), bitreverse, bitcast back
6733 Builder.SetInsertPoint(II);
6734 Value *CastToVec = Builder.CreateBitCast(V: X, DestTy: NewVecTy);
6735 Value *NewCall =
6736 Builder.CreateUnaryIntrinsic(ID: Intrinsic::bitreverse, Op: CastToVec);
6737 Value *CastToOrig = Builder.CreateBitCast(V: NewCall, DestTy: Ty);
6738 replaceValue(Old&: I, New&: *CastToOrig);
6739 return true;
6740}
6741
6742/// Given the maximum shuffle index and load vector type, compute the number of
6743/// elements for the shrunk load, rounding up to the next full vector register
6744/// boundary to avoid scalar remainders that legalize poorly.
6745static unsigned getAlignedNumElements(unsigned MaxIdx, FixedVectorType *LoadTy,
6746 const TargetTransformInfo &TTI,
6747 const DataLayout &DL) {
6748 unsigned RawNumElements = MaxIdx + 1u;
6749 Type *ElemTy = LoadTy->getElementType();
6750 // Skip alignment for illegal element types.
6751 if (!TTI.isTypeLegal(Ty: ElemTy))
6752 return RawNumElements;
6753
6754 TypeSize ElemSize = DL.getTypeSizeInBits(Ty: ElemTy);
6755 if (ElemSize.isScalable() || ElemSize.isZero())
6756 return RawNumElements;
6757
6758 TypeSize RegSize =
6759 TTI.getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector);
6760 if (RegSize.isScalable() || RegSize.isZero())
6761 return RawNumElements;
6762
6763 unsigned ElemsPerReg = RegSize.getFixedValue() / ElemSize.getFixedValue();
6764 // If the load already fits in a register, keep the exact size.
6765 // Otherwise round up to the next full register boundary.
6766 if (ElemsPerReg == 0 || RawNumElements <= ElemsPerReg)
6767 return RawNumElements;
6768
6769 return alignTo(Value: RawNumElements, Align: ElemsPerReg);
6770}
6771
6772// Attempt to shrink loads that are only used by shufflevector instructions.
6773bool VectorCombine::shrinkLoadForShuffles(Instruction &I) {
6774 auto *OldLoad = dyn_cast<LoadInst>(Val: &I);
6775 if (!OldLoad || !OldLoad->isSimple())
6776 return false;
6777
6778 auto *OldLoadTy = dyn_cast<FixedVectorType>(Val: OldLoad->getType());
6779 if (!OldLoadTy)
6780 return false;
6781
6782 unsigned const OldNumElements = OldLoadTy->getNumElements();
6783
6784 // Search all uses of load. If all uses are shufflevector instructions, and
6785 // the second operands are all poison values, find the minimum and maximum
6786 // indices of the vector elements referenced by all shuffle masks.
6787 // Otherwise return `std::nullopt`.
6788 using IndexRange = std::pair<int, int>;
6789 auto GetIndexRangeInShuffles = [&]() -> std::optional<IndexRange> {
6790 IndexRange OutputRange = IndexRange(OldNumElements, -1);
6791 for (llvm::Use &Use : I.uses()) {
6792 // Ensure all uses match the required pattern.
6793 User *Shuffle = Use.getUser();
6794 ArrayRef<int> Mask;
6795
6796 if (!match(V: Shuffle,
6797 P: m_Shuffle(v1: m_Specific(V: OldLoad), v2: m_Undef(), mask: m_Mask(Mask))))
6798 return std::nullopt;
6799
6800 // Ignore shufflevector instructions that have no uses.
6801 if (Shuffle->use_empty())
6802 continue;
6803
6804 // Find the min and max indices used by the shufflevector instruction.
6805 for (int Index : Mask) {
6806 if (Index >= 0 && Index < static_cast<int>(OldNumElements)) {
6807 OutputRange.first = std::min(a: Index, b: OutputRange.first);
6808 OutputRange.second = std::max(a: Index, b: OutputRange.second);
6809 }
6810 }
6811 }
6812
6813 if (OutputRange.second < OutputRange.first)
6814 return std::nullopt;
6815
6816 return OutputRange;
6817 };
6818
6819 // Get the range of vector elements used by shufflevector instructions.
6820 if (std::optional<IndexRange> Indices = GetIndexRangeInShuffles()) {
6821 unsigned const NewNumElements =
6822 getAlignedNumElements(MaxIdx: Indices->second, LoadTy: OldLoadTy, TTI, DL: *DL);
6823
6824 // If the range of vector elements is smaller than the full load, attempt
6825 // to create a smaller load.
6826 if (NewNumElements < OldNumElements) {
6827 Builder.SetInsertPoint(&I);
6828 Builder.SetCurrentDebugLocation(I.getDebugLoc());
6829
6830 // Calculate costs of old and new ops.
6831 Type *ElemTy = OldLoadTy->getElementType();
6832 FixedVectorType *NewLoadTy = FixedVectorType::get(ElementType: ElemTy, NumElts: NewNumElements);
6833 Value *PtrOp = OldLoad->getPointerOperand();
6834
6835 InstructionCost OldCost = TTI.getMemoryOpCost(
6836 Opcode: Instruction::Load, Src: OldLoad->getType(), Alignment: OldLoad->getAlign(),
6837 AddressSpace: OldLoad->getPointerAddressSpace(), CostKind);
6838 InstructionCost NewCost =
6839 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: NewLoadTy, Alignment: OldLoad->getAlign(),
6840 AddressSpace: OldLoad->getPointerAddressSpace(), CostKind);
6841
6842 using UseEntry = std::pair<ShuffleVectorInst *, std::vector<int>>;
6843 SmallVector<UseEntry, 4u> NewUses;
6844 unsigned const MaxIndex = NewNumElements * 2u;
6845
6846 for (llvm::Use &Use : I.uses()) {
6847 auto *Shuffle = cast<ShuffleVectorInst>(Val: Use.getUser());
6848
6849 // Ignore shufflevector instructions that have no uses.
6850 if (Shuffle->use_empty())
6851 continue;
6852
6853 ArrayRef<int> OldMask = Shuffle->getShuffleMask();
6854
6855 // Create entry for new use.
6856 NewUses.push_back(Elt: {Shuffle, OldMask});
6857
6858 // Validate mask indices.
6859 for (int Index : OldMask) {
6860 if (Index >= static_cast<int>(MaxIndex))
6861 return false;
6862 }
6863
6864 // Update costs.
6865 OldCost +=
6866 TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: Shuffle->getType(),
6867 SrcTy: OldLoadTy, CostKind, Mask: OldMask);
6868 NewCost +=
6869 TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: Shuffle->getType(),
6870 SrcTy: NewLoadTy, CostKind, Mask: OldMask);
6871 }
6872
6873 LLVM_DEBUG(
6874 dbgs() << "Found a load used only by shufflevector instructions: "
6875 << I << "\n OldCost: " << OldCost
6876 << " vs NewCost: " << NewCost << "\n");
6877
6878 if (OldCost < NewCost || !NewCost.isValid())
6879 return false;
6880
6881 // Create new load of smaller vector.
6882 auto *NewLoad = cast<LoadInst>(
6883 Val: Builder.CreateAlignedLoad(Ty: NewLoadTy, Ptr: PtrOp, Align: OldLoad->getAlign()));
6884 NewLoad->copyMetadata(SrcInst: I);
6885
6886 // Replace all uses.
6887 for (UseEntry &Use : NewUses) {
6888 ShuffleVectorInst *Shuffle = Use.first;
6889 std::vector<int> &NewMask = Use.second;
6890
6891 Builder.SetInsertPoint(Shuffle);
6892 Builder.SetCurrentDebugLocation(Shuffle->getDebugLoc());
6893 Value *NewShuffle = Builder.CreateShuffleVector(
6894 V1: NewLoad, V2: PoisonValue::get(T: NewLoadTy), Mask: NewMask);
6895
6896 replaceValue(Old&: *Shuffle, New&: *NewShuffle, Erase: false);
6897 }
6898
6899 return true;
6900 }
6901 }
6902 return false;
6903}
6904
6905// Attempt to narrow a phi of shufflevector instructions where the two incoming
6906// values have the same operands but different masks. If the two shuffle masks
6907// are offsets of one another we can use one branch to rotate the incoming
6908// vector and perform one larger shuffle after the phi.
6909bool VectorCombine::shrinkPhiOfShuffles(Instruction &I) {
6910 auto *Phi = dyn_cast<PHINode>(Val: &I);
6911 if (!Phi || Phi->getNumIncomingValues() != 2u)
6912 return false;
6913
6914 Value *Op = nullptr;
6915 ArrayRef<int> Mask0;
6916 ArrayRef<int> Mask1;
6917
6918 if (!match(V: Phi->getOperand(i_nocapture: 0u),
6919 P: m_OneUse(SubPattern: m_Shuffle(v1: m_Value(V&: Op), v2: m_Poison(), mask: m_Mask(Mask0)))) ||
6920 !match(V: Phi->getOperand(i_nocapture: 1u),
6921 P: m_OneUse(SubPattern: m_Shuffle(v1: m_Specific(V: Op), v2: m_Poison(), mask: m_Mask(Mask1)))))
6922 return false;
6923
6924 auto *Shuf = cast<ShuffleVectorInst>(Val: Phi->getOperand(i_nocapture: 0u));
6925
6926 // Ensure result vectors are wider than the argument vector.
6927 auto *InputVT = cast<FixedVectorType>(Val: Op->getType());
6928 auto *ResultVT = cast<FixedVectorType>(Val: Shuf->getType());
6929 auto const InputNumElements = InputVT->getNumElements();
6930
6931 if (InputNumElements >= ResultVT->getNumElements())
6932 return false;
6933
6934 // Take the difference of the two shuffle masks at each index. Ignore poison
6935 // values at the same index in both masks.
6936 SmallVector<int, 16> NewMask;
6937 NewMask.reserve(N: Mask0.size());
6938
6939 for (auto [M0, M1] : zip(t&: Mask0, u&: Mask1)) {
6940 if (M0 >= 0 && M1 >= 0)
6941 NewMask.push_back(Elt: M0 - M1);
6942 else if (M0 == -1 && M1 == -1)
6943 continue;
6944 else
6945 return false;
6946 }
6947
6948 // Ensure all elements of the new mask are equal. If the difference between
6949 // the incoming mask elements is the same, the two must be constant offsets
6950 // of one another.
6951 if (NewMask.empty() || !all_equal(Range&: NewMask))
6952 return false;
6953
6954 // Create new mask using difference of the two incoming masks.
6955 int MaskOffset = NewMask[0u];
6956 unsigned Index = (InputNumElements + MaskOffset) % InputNumElements;
6957 NewMask.clear();
6958
6959 for (unsigned I = 0u; I < InputNumElements; ++I) {
6960 NewMask.push_back(Elt: Index);
6961 Index = (Index + 1u) % InputNumElements;
6962 }
6963
6964 // Calculate costs for worst cases and compare.
6965 auto const Kind = TTI::SK_PermuteSingleSrc;
6966 auto OldCost =
6967 std::max(a: TTI.getShuffleCost(Kind, DstTy: ResultVT, SrcTy: InputVT, CostKind, Mask: Mask0),
6968 b: TTI.getShuffleCost(Kind, DstTy: ResultVT, SrcTy: InputVT, CostKind, Mask: Mask1));
6969 auto NewCost = TTI.getShuffleCost(Kind, DstTy: InputVT, SrcTy: InputVT, CostKind, Mask: NewMask) +
6970 TTI.getShuffleCost(Kind, DstTy: ResultVT, SrcTy: InputVT, CostKind, Mask: Mask1);
6971
6972 LLVM_DEBUG(dbgs() << "Found a phi of mergeable shuffles: " << I
6973 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
6974 << "\n");
6975
6976 if (NewCost > OldCost)
6977 return false;
6978
6979 // Create new shuffles and narrowed phi.
6980 Builder.SetInsertPoint(Shuf);
6981 Builder.SetCurrentDebugLocation(Shuf->getDebugLoc());
6982 auto *PoisonVal = PoisonValue::get(T: InputVT);
6983 auto *NewShuf0 = Builder.CreateShuffleVector(V1: Op, V2: PoisonVal, Mask: NewMask);
6984
6985 Builder.SetInsertPoint(Phi);
6986 Builder.SetCurrentDebugLocation(Phi->getDebugLoc());
6987 auto *NewPhi = Builder.CreatePHI(Ty: NewShuf0->getType(), NumReservedValues: 2u);
6988 NewPhi->addIncoming(V: NewShuf0, BB: Phi->getIncomingBlock(i: 0u));
6989 NewPhi->addIncoming(V: Op, BB: Phi->getIncomingBlock(i: 1u));
6990
6991 Builder.SetInsertPoint(*NewPhi->getInsertionPointAfterDef());
6992 PoisonVal = PoisonValue::get(T: NewPhi->getType());
6993 auto *NewShuf1 = Builder.CreateShuffleVector(V1: NewPhi, V2: PoisonVal, Mask: Mask1);
6994
6995 replaceValue(Old&: *Phi, New&: *NewShuf1);
6996 return true;
6997}
6998
6999/// This is the entry point for all transforms. Pass manager differences are
7000/// handled in the callers of this function.
7001bool VectorCombine::run() {
7002 if (Opts.disable_vector_combine)
7003 return false;
7004
7005 // Don't attempt vectorization if the target does not support vectors.
7006 if (!TTI.getNumberOfRegisters(ClassID: TTI.getRegisterClassForType(/*Vector*/ true)))
7007 return false;
7008
7009 LLVM_DEBUG(dbgs() << "\n\nVECTORCOMBINE on " << F.getName() << "\n");
7010
7011 auto FoldInst = [this](Instruction &I) {
7012 Builder.SetInsertPoint(&I);
7013 bool IsVectorType = isa<VectorType>(Val: I.getType());
7014 bool IsFixedVectorType = isa<FixedVectorType>(Val: I.getType());
7015 auto Opcode = I.getOpcode();
7016
7017 LLVM_DEBUG(dbgs() << "VC: Visiting: " << I << '\n');
7018
7019 // These folds should be beneficial regardless of when this pass is run
7020 // in the optimization pipeline.
7021 // The type checking is for run-time efficiency. We can avoid wasting time
7022 // dispatching to folding functions if there's no chance of matching.
7023 if (IsFixedVectorType) {
7024 switch (Opcode) {
7025 case Instruction::InsertElement:
7026 if (vectorizeLoadInsert(I))
7027 return true;
7028 break;
7029 case Instruction::ShuffleVector:
7030 if (widenSubvectorLoad(I))
7031 return true;
7032 break;
7033 default:
7034 break;
7035 }
7036 }
7037
7038 // This transform works with scalable and fixed vectors
7039 // TODO: Identify and allow other scalable transforms
7040 if (IsVectorType) {
7041 if (scalarizeOpOrCmp(I))
7042 return true;
7043 if (scalarizeLoad(I))
7044 return true;
7045 if (scalarizeExtExtract(I))
7046 return true;
7047 if (foldInterleaveIntrinsics(I))
7048 return true;
7049 if (foldBitcastOfVPLoad(I))
7050 return true;
7051 }
7052
7053 if (foldDeinterleaveIntrinsics(I))
7054 return true;
7055
7056 if (Opcode == Instruction::Store)
7057 if (foldInsertElementsToStores(I))
7058 return true;
7059
7060 // If this is an early pipeline invocation of this pass, we are done.
7061 if (TryEarlyFoldsOnly)
7062 return false;
7063
7064 if (Opcode == Instruction::Call)
7065 if (foldBitOrderReverseAndSwap(I))
7066 return true;
7067 if (Opcode == Instruction::BitCast)
7068 if (foldBitOrderReverseAndSwap(I))
7069 return true;
7070
7071 // Otherwise, try folds that improve codegen but may interfere with
7072 // early IR canonicalizations.
7073 // The type checking is for run-time efficiency. We can avoid wasting time
7074 // dispatching to folding functions if there's no chance of matching.
7075 if (IsFixedVectorType) {
7076 switch (Opcode) {
7077 case Instruction::InsertElement:
7078 if (foldInsExtFNeg(I))
7079 return true;
7080 if (foldInsExtBinop(I))
7081 return true;
7082 if (foldInsExtVectorToShuffle(I))
7083 return true;
7084 if (foldInsertScalarPartsToShuffle(I))
7085 return true;
7086 break;
7087 case Instruction::ShuffleVector:
7088 if (foldPermuteOfBinops(I))
7089 return true;
7090 if (foldShuffleOfBinops(I))
7091 return true;
7092 if (foldShuffleOfSelects(I))
7093 return true;
7094 if (foldShuffleOfCastops(I))
7095 return true;
7096 if (foldShuffleOfShuffles(I))
7097 return true;
7098 if (foldPermuteOfIntrinsic(I))
7099 return true;
7100 if (foldShufflesOfLengthChangingShuffles(I))
7101 return true;
7102 if (foldShuffleOfIntrinsics(I))
7103 return true;
7104 if (foldSelectShuffle(I))
7105 return true;
7106 if (foldShuffleToIdentity(I))
7107 return true;
7108 break;
7109 case Instruction::Load:
7110 if (shrinkLoadForShuffles(I))
7111 return true;
7112 break;
7113 case Instruction::BitCast:
7114 if (foldBitcastShuffle(I))
7115 return true;
7116 if (foldSelectsFromBitcast(I))
7117 return true;
7118 break;
7119 case Instruction::And:
7120 case Instruction::Or:
7121 case Instruction::Xor:
7122 if (foldBitOpOfCastops(I))
7123 return true;
7124 if (foldBitOpOfCastConstant(I))
7125 return true;
7126 break;
7127 case Instruction::PHI:
7128 if (shrinkPhiOfShuffles(I))
7129 return true;
7130 break;
7131 default:
7132 if (shrinkType(I))
7133 return true;
7134 break;
7135 }
7136 } else {
7137 switch (Opcode) {
7138 case Instruction::Call:
7139 if (foldShuffleFromReductions(I))
7140 return true;
7141 if (foldCastFromReductions(I))
7142 return true;
7143 break;
7144 case Instruction::ExtractElement:
7145 if (foldShuffleChainsToReduce(I))
7146 return true;
7147 break;
7148 case Instruction::ICmp:
7149 if (foldSignBitReductionCmp(I))
7150 return true;
7151 if (foldICmpEqZeroVectorReduce(I))
7152 return true;
7153 if (foldReductionZeroTest(I))
7154 return true;
7155 if (foldEquivalentReductionCmp(I))
7156 return true;
7157 if (foldReduceAddCmpZero(I))
7158 return true;
7159 [[fallthrough]];
7160 case Instruction::FCmp:
7161 if (foldExtractExtract(I))
7162 return true;
7163 break;
7164 case Instruction::Or:
7165 if (foldConcatOfBoolMasks(I))
7166 return true;
7167 [[fallthrough]];
7168 default:
7169 if (Instruction::isBinaryOp(Opcode)) {
7170 if (foldExtractExtract(I))
7171 return true;
7172 if (foldExtractedCmps(I))
7173 return true;
7174 if (foldBinopOfReductions(I))
7175 return true;
7176 }
7177 break;
7178 }
7179 }
7180 return false;
7181 };
7182
7183 bool MadeChange = false;
7184 for (BasicBlock &BB : F) {
7185 // Ignore unreachable basic blocks.
7186 if (!DT.isReachableFromEntry(A: &BB))
7187 continue;
7188 // Use early increment range so that we can erase instructions in loop.
7189 // make_early_inc_range is not applicable here, as the next iterator may
7190 // be invalidated by RecursivelyDeleteTriviallyDeadInstructions.
7191 // We manually maintain the next instruction and update it when it is about
7192 // to be deleted.
7193 Instruction *I = &BB.front();
7194 while (I) {
7195 NextInst = I->getNextNode();
7196 MadeChange |= FoldInst(*I);
7197 I = NextInst;
7198 }
7199 }
7200
7201 NextInst = nullptr;
7202
7203 while (!Worklist.isEmpty()) {
7204 // Push deferred instructions in reverse order, so that they'll end up
7205 // popped from the worklist in-order.
7206 while (Instruction *I = Worklist.popDeferred()) {
7207 if (isInstructionTriviallyDead(I)) {
7208 eraseInstruction(I&: *I);
7209 continue;
7210 }
7211
7212 Worklist.push(I);
7213 }
7214
7215 Instruction *I = Worklist.removeOne();
7216 if (!I)
7217 continue;
7218
7219 if (isInstructionTriviallyDead(I)) {
7220 eraseInstruction(I&: *I);
7221 continue;
7222 }
7223
7224 MadeChange |= FoldInst(*I);
7225 }
7226
7227 return MadeChange;
7228}
7229
7230PreservedAnalyses VectorCombinePass::run(Function &F,
7231 FunctionAnalysisManager &FAM) {
7232 auto &AC = FAM.getResult<AssumptionAnalysis>(IR&: F);
7233 TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(IR&: F);
7234 DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(IR&: F);
7235 AAResults &AA = FAM.getResult<AAManager>(IR&: F);
7236 const DataLayout *DL = &F.getDataLayout();
7237 TTI::TargetCostKind CostKind =
7238 F.hasOptSize() ? TTI::TCK_CodeSize : TTI::TCK_RecipThroughput;
7239 VectorCombine Combiner(
7240 F.getContext().getLibraryOptions().get<VectorizeOptions>(), F, TTI, DT,
7241 AA, AC, DL, CostKind, TryEarlyFoldsOnly);
7242 if (!Combiner.run())
7243 return PreservedAnalyses::all();
7244 PreservedAnalyses PA;
7245 PA.preserveSet<CFGAnalyses>();
7246 return PA;
7247}
7248