1//===------- VectorCombine.cpp - Optimize partial vector operations -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass optimizes scalar/vector interactions using target cost models. The
10// transforms implemented here may not fit in traditional loop-based or SLP
11// vectorization passes.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Vectorize/VectorCombine.h"
16#include "VectorizeOptions.h"
17#include "llvm/ADT/DenseMap.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/ScopeExit.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/Analysis/AssumptionCache.h"
23#include "llvm/Analysis/BasicAliasAnalysis.h"
24#include "llvm/Analysis/GlobalsModRef.h"
25#include "llvm/Analysis/InstSimplifyFolder.h"
26#include "llvm/Analysis/Loads.h"
27#include "llvm/Analysis/TargetFolder.h"
28#include "llvm/Analysis/TargetTransformInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/IR/Dominators.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/IRBuilder.h"
34#include "llvm/IR/Instructions.h"
35#include "llvm/IR/PatternMatch.h"
36#include "llvm/IR/ProfDataUtils.h"
37#include "llvm/Support/KnownBits.h"
38#include "llvm/Support/MathExtras.h"
39#include "llvm/Transforms/Utils/Local.h"
40#include "llvm/Transforms/Utils/LoopUtils.h"
41#include <numeric>
42#include <optional>
43#include <queue>
44#include <set>
45
46#define DEBUG_TYPE "vector-combine"
47#include "llvm/Transforms/Utils/InstructionWorklist.h"
48
49using namespace llvm;
50using namespace llvm::PatternMatch;
51
52STATISTIC(NumVecLoad, "Number of vector loads formed");
53STATISTIC(NumVecCmp, "Number of vector compares formed");
54STATISTIC(NumVecBO, "Number of vector binops formed");
55STATISTIC(NumVecCmpBO, "Number of vector compare + binop formed");
56STATISTIC(NumShufOfBitcast, "Number of shuffles moved after bitcast");
57STATISTIC(NumScalarOps, "Number of scalar unary + binary ops formed");
58STATISTIC(NumScalarCmp, "Number of scalar compares formed");
59STATISTIC(NumScalarIntrinsic, "Number of scalar intrinsic calls formed");
60
61static const unsigned InvalidIndex = std::numeric_limits<unsigned>::max();
62
63namespace {
64class VectorCombine {
65public:
66 VectorCombine(const VectorizeOptions &Opts, Function &F,
67 const TargetTransformInfo &TTI, const DominatorTree &DT,
68 AAResults &AA, AssumptionCache &AC, const DataLayout *DL,
69 TTI::TargetCostKind CostKind, bool TryEarlyFoldsOnly)
70 : Opts(Opts), F(F),
71 Builder(*F.getParent(), InstSimplifyFolder(*DL),
72 IRBuilderCallbackInserter([this](Instruction *I) {
73 // Automatically add newly created instructions
74 // into the deferred queue.
75 Worklist.add(I);
76 })),
77 TTI(TTI), DT(DT), AA(AA), DL(DL), CostKind(CostKind),
78 SQ(*DL, /*TLI=*/nullptr, &DT, &AC),
79 TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
80
81 bool run();
82
83private:
84 const VectorizeOptions &Opts;
85 Function &F;
86 IRBuilder<InstSimplifyFolder, IRBuilderCallbackInserter> Builder;
87 const TargetTransformInfo &TTI;
88 const DominatorTree &DT;
89 AAResults &AA;
90 const DataLayout *DL;
91 TTI::TargetCostKind CostKind;
92 const SimplifyQuery SQ;
93
94 /// If true, only perform beneficial early IR transforms. Do not introduce new
95 /// vector operations.
96 bool TryEarlyFoldsOnly;
97
98 InstructionWorklist Worklist;
99
100 /// Next instruction to iterate. It will be updated when it is erased by
101 /// RecursivelyDeleteTriviallyDeadInstructions.
102 Instruction *NextInst;
103
104 // TODO: Direct calls from the top-level "run" loop use a plain "Instruction"
105 // parameter. That should be updated to specific sub-classes because the
106 // run loop was changed to dispatch on opcode.
107 bool vectorizeLoadInsert(Instruction &I);
108 bool widenSubvectorLoad(Instruction &I);
109 ExtractElementInst *getShuffleExtract(ExtractElementInst *Ext0,
110 ExtractElementInst *Ext1,
111 unsigned PreferredExtractIndex) const;
112 bool isExtractExtractCheap(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
113 const Instruction &I,
114 ExtractElementInst *&ConvertToShuffle,
115 unsigned PreferredExtractIndex);
116 Value *foldExtExtCmp(Value *V0, Value *V1, Value *ExtIndex, Instruction &I);
117 Value *foldExtExtBinop(Value *V0, Value *V1, Value *ExtIndex, Instruction &I);
118 bool foldExtractExtract(Instruction &I);
119 bool foldInsExtFNeg(Instruction &I);
120 bool foldInsExtBinop(Instruction &I);
121 bool foldInsExtVectorToShuffle(Instruction &I);
122 bool foldInsertScalarPartsToShuffle(Instruction &I);
123 bool foldBitOpOfCastops(Instruction &I);
124 bool foldBitOpOfCastConstant(Instruction &I);
125 bool foldBitcastShuffle(Instruction &I);
126 bool scalarizeOpOrCmp(Instruction &I);
127 bool foldExtractedCmps(Instruction &I);
128 bool foldSelectsFromBitcast(Instruction &I);
129 bool foldBinopOfReductions(Instruction &I);
130 bool foldInsertElementsToStores(Instruction &I);
131 bool scalarizeLoad(Instruction &I);
132 bool scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy, Value *Ptr);
133 bool scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy, Value *Ptr);
134 bool scalarizeExtExtract(Instruction &I);
135 bool foldConcatOfBoolMasks(Instruction &I);
136 bool foldPermuteOfBinops(Instruction &I);
137 bool foldShuffleOfBinops(Instruction &I);
138 bool foldShuffleOfSelects(Instruction &I);
139 bool foldShuffleOfCastops(Instruction &I);
140 bool foldShuffleOfShuffles(Instruction &I);
141 bool foldShuffleOfAdjacentLoads(Instruction &I);
142 bool foldPermuteOfIntrinsic(Instruction &I);
143 bool foldShufflesOfLengthChangingShuffles(Instruction &I);
144 bool foldShuffleOfIntrinsics(Instruction &I);
145 bool foldShuffleToIdentity(Instruction &I);
146 bool foldShuffleFromReductions(Instruction &I);
147 bool foldShuffleChainsToReduce(Instruction &I);
148 bool foldCastFromReductions(Instruction &I);
149 bool foldSignBitReductionCmp(Instruction &I);
150 bool foldReductionZeroTest(Instruction &I);
151 bool foldICmpEqZeroVectorReduce(Instruction &I);
152 bool foldEquivalentReductionCmp(Instruction &I);
153 bool foldReduceAddCmpZero(Instruction &I);
154 bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
155 bool foldInterleaveIntrinsics(Instruction &I);
156 bool foldDeinterleaveIntrinsics(Instruction &I);
157 bool foldBitcastOfVPLoad(Instruction &I);
158 bool foldBitOrderReverseAndSwap(Instruction &I);
159 bool shrinkType(Instruction &I);
160 bool shrinkLoadForShuffles(Instruction &I);
161 bool shrinkPhiOfShuffles(Instruction &I);
162 bool foldInterleaveOfDeinterleaveChains(Instruction &I);
163
164 void replaceValue(Instruction &Old, Value &New, bool Erase = true) {
165 LLVM_DEBUG(dbgs() << "VC: Replacing: " << Old << '\n');
166 LLVM_DEBUG(dbgs() << " With: " << New << '\n');
167 Old.replaceAllUsesWith(V: &New);
168 if (auto *NewI = dyn_cast<Instruction>(Val: &New)) {
169 New.takeName(V: &Old);
170 Worklist.pushUsersToWorkList(I&: *NewI);
171 Worklist.pushValue(V: NewI);
172 }
173 if (Erase && isInstructionTriviallyDead(I: &Old)) {
174 eraseInstruction(I&: Old);
175 } else {
176 Worklist.push(I: &Old);
177 }
178 }
179
180 void eraseInstruction(Instruction &I) {
181 LLVM_DEBUG(dbgs() << "VC: Erasing: " << I << '\n');
182 SmallVector<Value *> Ops(I.operands());
183 Worklist.remove(I: &I);
184 I.eraseFromParent();
185
186 // Push remaining users of the operands and then the operand itself - allows
187 // further folds that were hindered by OneUse limits.
188 SmallPtrSet<Value *, 4> Visited;
189 for (Value *Op : Ops) {
190 if (!Visited.contains(Ptr: Op)) {
191 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
192 if (RecursivelyDeleteTriviallyDeadInstructions(
193 V: OpI, TLI: nullptr, MSSAU: nullptr, AboutToDeleteCallback: [&](Value *V) {
194 if (auto *I = dyn_cast<Instruction>(Val: V)) {
195 LLVM_DEBUG(dbgs() << "VC: Erased: " << *I << '\n');
196 Worklist.remove(I);
197 if (I == NextInst)
198 NextInst = NextInst->getNextNode();
199 Visited.insert(Ptr: I);
200 }
201 }))
202 continue;
203 Worklist.pushUsersToWorkList(I&: *OpI);
204 Worklist.pushValue(V: OpI);
205 }
206 }
207 }
208 }
209};
210} // namespace
211
212/// Return the source operand of a potentially bitcasted value. If there is no
213/// bitcast, return the input value itself.
214static Value *peekThroughBitcasts(Value *V) {
215 while (auto *BitCast = dyn_cast<BitCastInst>(Val: V))
216 V = BitCast->getOperand(i_nocapture: 0);
217 return V;
218}
219
220/// Helper to peek through bitcasts to the same value.
221static bool isEquivBitcast(Value *X, Value *Y) {
222 return X->getType() == Y->getType() &&
223 peekThroughBitcasts(V: X) == peekThroughBitcasts(V: Y);
224}
225
226static bool canWidenLoad(LoadInst *Load, const TargetTransformInfo &TTI) {
227 // Do not widen load if atomic/volatile or under asan/hwasan/memtag/tsan.
228 // The widened load may load data from dirty regions or create data races
229 // non-existent in the source.
230 if (!Load || !Load->isSimple() || !Load->hasOneUse() ||
231 Load->getFunction()->hasFnAttribute(Kind: Attribute::SanitizeMemTag) ||
232 mustSuppressSpeculation(LI: *Load))
233 return false;
234
235 // We are potentially transforming byte-sized (8-bit) memory accesses, so make
236 // sure we have all of our type-based constraints in place for this target.
237 Type *ScalarTy = Load->getType()->getScalarType();
238 uint64_t ScalarSize = ScalarTy->getPrimitiveSizeInBits();
239 unsigned MinVectorSize = TTI.getMinVectorRegisterBitWidth();
240 if (!ScalarSize || !MinVectorSize || MinVectorSize % ScalarSize != 0 ||
241 ScalarSize % 8 != 0)
242 return false;
243
244 return true;
245}
246
247bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
248 // Match insert into fixed vector of scalar value.
249 // TODO: Handle non-zero insert index.
250 Value *Scalar;
251 if (!match(V: &I,
252 P: m_InsertElt(Val: m_Poison(), Elt: m_OneUse(SubPattern: m_Value(V&: Scalar)), Idx: m_ZeroInt())))
253 return false;
254
255 // Optionally match an extract from another vector.
256 Value *X;
257 bool HasExtract = match(V: Scalar, P: m_ExtractElt(Val: m_Value(V&: X), Idx: m_ZeroInt()));
258 if (!HasExtract)
259 X = Scalar;
260
261 auto *Load = dyn_cast<LoadInst>(Val: X);
262 if (!canWidenLoad(Load, TTI))
263 return false;
264
265 Type *ScalarTy = Scalar->getType();
266 uint64_t ScalarSize = ScalarTy->getPrimitiveSizeInBits();
267 unsigned MinVectorSize = TTI.getMinVectorRegisterBitWidth();
268
269 // Check safety of replacing the scalar load with a larger vector load.
270 // We use minimal alignment (maximum flexibility) because we only care about
271 // the dereferenceable region. When calculating cost and creating a new op,
272 // we may use a larger value based on alignment attributes.
273 Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts();
274 assert(isa<PointerType>(SrcPtr->getType()) && "Expected a pointer type");
275
276 unsigned MinVecNumElts = MinVectorSize / ScalarSize;
277 auto *MinVecTy = VectorType::get(ElementType: ScalarTy, NumElements: MinVecNumElts, Scalable: false);
278 unsigned OffsetEltIndex = 0;
279 Align Alignment = Load->getAlign();
280 // Ignore frees because the existing load proves that the object has not been
281 // freed. Allocations cannot shrink, so this is sufficient.
282 if (!isSafeToLoadUnconditionally(V: SrcPtr, Ty: MinVecTy, Alignment: Align(1),
283 SQ: SQ.getWithInstruction(I: Load),
284 /*IgnoreFree=*/true)) {
285 // It is not safe to load directly from the pointer, but we can still peek
286 // through gep offsets and check if it safe to load from a base address with
287 // updated alignment. If it is, we can shuffle the element(s) into place
288 // after loading.
289 unsigned OffsetBitWidth = DL->getIndexTypeSizeInBits(Ty: SrcPtr->getType());
290 APInt Offset(OffsetBitWidth, 0);
291 SrcPtr = SrcPtr->stripAndAccumulateInBoundsConstantOffsets(DL: *DL, Offset);
292
293 // We want to shuffle the result down from a high element of a vector, so
294 // the offset must be positive.
295 if (Offset.isNegative())
296 return false;
297
298 // The offset must be a multiple of the scalar element to shuffle cleanly
299 // in the element's size.
300 uint64_t ScalarSizeInBytes = ScalarSize / 8;
301 if (Offset.urem(RHS: ScalarSizeInBytes) != 0)
302 return false;
303
304 // If we load MinVecNumElts, will our target element still be loaded?
305 APInt OffsetEltIndexAP = Offset.udiv(RHS: ScalarSizeInBytes);
306 if (OffsetEltIndexAP.uge(RHS: MinVecNumElts))
307 return false;
308 OffsetEltIndex = OffsetEltIndexAP.getZExtValue();
309
310 if (!isSafeToLoadUnconditionally(V: SrcPtr, Ty: MinVecTy, Alignment: Align(1),
311 SQ: SQ.getWithInstruction(I: Load),
312 /*IgnoreFree=*/true))
313 return false;
314
315 // Update alignment with offset value. Note that the offset could be negated
316 // to more accurately represent "(new) SrcPtr - Offset = (old) SrcPtr", but
317 // negation does not change the result of the alignment calculation.
318 Alignment = commonAlignment(A: Alignment, Offset: Offset.getZExtValue());
319 }
320
321 // Original pattern: insertelt undef, load [free casts of] PtrOp, 0
322 // Use the greater of the alignment on the load or its source pointer.
323 Alignment = std::max(a: SrcPtr->getPointerAlignment(DL: *DL), b: Alignment);
324 Type *LoadTy = Load->getType();
325 unsigned AS = Load->getPointerAddressSpace();
326 InstructionCost OldCost =
327 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: LoadTy, Alignment, AddressSpace: AS, CostKind);
328 APInt DemandedElts = APInt::getOneBitSet(numBits: MinVecNumElts, BitNo: 0);
329 OldCost +=
330 TTI.getScalarizationOverhead(Ty: MinVecTy, DemandedElts,
331 /* Insert */ true, Extract: HasExtract, CostKind);
332
333 // New pattern: load VecPtr
334 InstructionCost NewCost =
335 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: MinVecTy, Alignment, AddressSpace: AS, CostKind);
336 // Optionally, we are shuffling the loaded vector element(s) into place.
337 // For the mask set everything but element 0 to undef to prevent poison from
338 // propagating from the extra loaded memory. This will also optionally
339 // shrink/grow the vector from the loaded size to the output size.
340 // We assume this operation has no cost in codegen if there was no offset.
341 // Note that we could use freeze to avoid poison problems, but then we might
342 // still need a shuffle to change the vector size.
343 auto *Ty = cast<FixedVectorType>(Val: I.getType());
344 unsigned OutputNumElts = Ty->getNumElements();
345 SmallVector<int, 16> Mask(OutputNumElts, PoisonMaskElem);
346 assert(OffsetEltIndex < MinVecNumElts && "Address offset too big");
347 Mask[0] = OffsetEltIndex;
348 if (OffsetEltIndex)
349 NewCost += TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: Ty, SrcTy: MinVecTy,
350 CostKind, Mask);
351
352 // We can aggressively convert to the vector form because the backend can
353 // invert this transform if it does not result in a performance win.
354 if (OldCost < NewCost || !NewCost.isValid())
355 return false;
356
357 // It is safe and potentially profitable to load a vector directly:
358 // inselt undef, load Scalar, 0 --> load VecPtr
359 Builder.SetInsertPoint(Load);
360 Value *CastedPtr =
361 Builder.CreatePointerBitCastOrAddrSpaceCast(V: SrcPtr, DestTy: Builder.getPtrTy(AddrSpace: AS));
362 Value *VecLd = Builder.CreateAlignedLoad(Ty: MinVecTy, Ptr: CastedPtr, Align: Alignment);
363 VecLd = Builder.CreateShuffleVector(V: VecLd, Mask);
364
365 replaceValue(Old&: I, New&: *VecLd);
366 ++NumVecLoad;
367 return true;
368}
369
370/// If we are loading a vector and then inserting it into a larger vector with
371/// undefined elements, try to load the larger vector and eliminate the insert.
372/// This removes a shuffle in IR and may allow combining of other loaded values.
373bool VectorCombine::widenSubvectorLoad(Instruction &I) {
374 // Match subvector insert of fixed vector.
375 auto *Shuf = cast<ShuffleVectorInst>(Val: &I);
376 if (!Shuf->isIdentityWithPadding())
377 return false;
378
379 // Allow a non-canonical shuffle mask that is choosing elements from op1.
380 unsigned NumOpElts =
381 cast<FixedVectorType>(Val: Shuf->getOperand(i_nocapture: 0)->getType())->getNumElements();
382 unsigned OpIndex = any_of(Range: Shuf->getShuffleMask(), P: [&NumOpElts](int M) {
383 return M >= (int)(NumOpElts);
384 });
385
386 auto *Load = dyn_cast<LoadInst>(Val: Shuf->getOperand(i_nocapture: OpIndex));
387 if (!canWidenLoad(Load, TTI))
388 return false;
389
390 // We use minimal alignment (maximum flexibility) because we only care about
391 // the dereferenceable region. When calculating cost and creating a new op,
392 // we may use a larger value based on alignment attributes.
393 auto *Ty = cast<FixedVectorType>(Val: I.getType());
394 Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts();
395 assert(isa<PointerType>(SrcPtr->getType()) && "Expected a pointer type");
396 Align Alignment = Load->getAlign();
397 // Ignore frees because the existing load proves that the object has not been
398 // freed. Allocations cannot shrink, so this is sufficient.
399 if (!isSafeToLoadUnconditionally(V: SrcPtr, Ty, Alignment: Align(1),
400 SQ: SQ.getWithInstruction(I: Load),
401 /*IgnoreFree=*/true))
402 return false;
403
404 Alignment = std::max(a: SrcPtr->getPointerAlignment(DL: *DL), b: Alignment);
405 Type *LoadTy = Load->getType();
406 unsigned AS = Load->getPointerAddressSpace();
407
408 // Original pattern: insert_subvector (load PtrOp)
409 // This conservatively assumes that the cost of a subvector insert into an
410 // undef value is 0. We could add that cost if the cost model accurately
411 // reflects the real cost of that operation.
412 InstructionCost OldCost =
413 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: LoadTy, Alignment, AddressSpace: AS, CostKind);
414
415 // New pattern: load PtrOp
416 InstructionCost NewCost =
417 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment, AddressSpace: AS, CostKind);
418
419 // We can aggressively convert to the vector form because the backend can
420 // invert this transform if it does not result in a performance win.
421 if (OldCost < NewCost || !NewCost.isValid())
422 return false;
423
424 Builder.SetInsertPoint(Load);
425 Value *CastedPtr =
426 Builder.CreatePointerBitCastOrAddrSpaceCast(V: SrcPtr, DestTy: Builder.getPtrTy(AddrSpace: AS));
427 Value *VecLd = Builder.CreateAlignedLoad(Ty, Ptr: CastedPtr, Align: Alignment);
428 replaceValue(Old&: I, New&: *VecLd);
429 ++NumVecLoad;
430 return true;
431}
432
433/// Determine which, if any, of the inputs should be replaced by a shuffle
434/// followed by extract from a different index.
435ExtractElementInst *VectorCombine::getShuffleExtract(
436 ExtractElementInst *Ext0, ExtractElementInst *Ext1,
437 unsigned PreferredExtractIndex = InvalidIndex) const {
438 auto *Index0C = dyn_cast<ConstantInt>(Val: Ext0->getIndexOperand());
439 auto *Index1C = dyn_cast<ConstantInt>(Val: Ext1->getIndexOperand());
440 assert(Index0C && Index1C && "Expected constant extract indexes");
441
442 unsigned Index0 = Index0C->getZExtValue();
443 unsigned Index1 = Index1C->getZExtValue();
444
445 // If the extract indexes are identical, no shuffle is needed.
446 if (Index0 == Index1)
447 return nullptr;
448
449 Type *VecTy = Ext0->getVectorOperand()->getType();
450 assert(VecTy == Ext1->getVectorOperand()->getType() && "Need matching types");
451 InstructionCost Cost0 =
452 TTI.getVectorInstrCost(I: *Ext0, Val: VecTy, CostKind, Index: Index0);
453 InstructionCost Cost1 =
454 TTI.getVectorInstrCost(I: *Ext1, Val: VecTy, CostKind, Index: Index1);
455
456 // If both costs are invalid no shuffle is needed
457 if (!Cost0.isValid() && !Cost1.isValid())
458 return nullptr;
459
460 // We are extracting from 2 different indexes, so one operand must be shuffled
461 // before performing a vector operation and/or extract. The more expensive
462 // extract will be replaced by a shuffle.
463 if (Cost0 > Cost1)
464 return Ext0;
465 if (Cost1 > Cost0)
466 return Ext1;
467
468 // If the costs are equal and there is a preferred extract index, shuffle the
469 // opposite operand.
470 if (PreferredExtractIndex == Index0)
471 return Ext1;
472 if (PreferredExtractIndex == Index1)
473 return Ext0;
474
475 // Otherwise, replace the extract with the higher index.
476 return Index0 > Index1 ? Ext0 : Ext1;
477}
478
479/// Compare the relative costs of 2 extracts followed by scalar operation vs.
480/// vector operation(s) followed by extract. Return true if the existing
481/// instructions are cheaper than a vector alternative. Otherwise, return false
482/// and if one of the extracts should be transformed to a shufflevector, set
483/// \p ConvertToShuffle to that extract instruction.
484bool VectorCombine::isExtractExtractCheap(ExtractElementInst *Ext0,
485 ExtractElementInst *Ext1,
486 const Instruction &I,
487 ExtractElementInst *&ConvertToShuffle,
488 unsigned PreferredExtractIndex) {
489 auto *Ext0IndexC = dyn_cast<ConstantInt>(Val: Ext0->getIndexOperand());
490 auto *Ext1IndexC = dyn_cast<ConstantInt>(Val: Ext1->getIndexOperand());
491 assert(Ext0IndexC && Ext1IndexC && "Expected constant extract indexes");
492
493 unsigned Opcode = I.getOpcode();
494 Value *Ext0Src = Ext0->getVectorOperand();
495 Value *Ext1Src = Ext1->getVectorOperand();
496 Type *ScalarTy = Ext0->getType();
497 auto *VecTy = cast<VectorType>(Val: Ext0Src->getType());
498 InstructionCost ScalarOpCost, VectorOpCost;
499
500 // Get cost estimates for scalar and vector versions of the operation.
501 bool IsBinOp = Instruction::isBinaryOp(Opcode);
502 if (IsBinOp) {
503 ScalarOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: ScalarTy, CostKind);
504 VectorOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
505 } else {
506 assert((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
507 "Expected a compare");
508 CmpInst::Predicate Pred = cast<CmpInst>(Val: I).getPredicate();
509 ScalarOpCost = TTI.getCmpSelInstrCost(
510 Opcode, ValTy: ScalarTy, CondTy: CmpInst::makeCmpResultType(opnd_type: ScalarTy), VecPred: Pred, CostKind);
511 VectorOpCost = TTI.getCmpSelInstrCost(
512 Opcode, ValTy: VecTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VecTy), VecPred: Pred, CostKind);
513 }
514
515 // Get cost estimates for the extract elements. These costs will factor into
516 // both sequences.
517 unsigned Ext0Index = Ext0IndexC->getZExtValue();
518 unsigned Ext1Index = Ext1IndexC->getZExtValue();
519
520 InstructionCost Extract0Cost =
521 TTI.getVectorInstrCost(I: *Ext0, Val: VecTy, CostKind, Index: Ext0Index);
522 InstructionCost Extract1Cost =
523 TTI.getVectorInstrCost(I: *Ext1, Val: VecTy, CostKind, Index: Ext1Index);
524
525 // A more expensive extract will always be replaced by a splat shuffle.
526 // For example, if Ext0 is more expensive:
527 // opcode (extelt V0, Ext0), (ext V1, Ext1) -->
528 // extelt (opcode (splat V0, Ext0), V1), Ext1
529 // TODO: Evaluate whether that always results in lowest cost. Alternatively,
530 // check the cost of creating a broadcast shuffle and shuffling both
531 // operands to element 0.
532 unsigned BestExtIndex = Extract0Cost > Extract1Cost ? Ext0Index : Ext1Index;
533 unsigned BestInsIndex = Extract0Cost > Extract1Cost ? Ext1Index : Ext0Index;
534 InstructionCost CheapExtractCost = std::min(a: Extract0Cost, b: Extract1Cost);
535
536 // Extra uses of the extracts mean that we include those costs in the
537 // vector total because those instructions will not be eliminated.
538 InstructionCost OldCost, NewCost;
539 if (Ext0Src == Ext1Src && Ext0Index == Ext1Index) {
540 // Handle a special case. If the 2 extracts are identical, adjust the
541 // formulas to account for that. The extra use charge allows for either the
542 // CSE'd pattern or an unoptimized form with identical values:
543 // opcode (extelt V, C), (extelt V, C) --> extelt (opcode V, V), C
544 bool HasUseTax = Ext0 == Ext1 ? !Ext0->hasNUses(N: 2)
545 : !Ext0->hasOneUse() || !Ext1->hasOneUse();
546 OldCost = CheapExtractCost + ScalarOpCost;
547 NewCost = VectorOpCost + CheapExtractCost + HasUseTax * CheapExtractCost;
548 } else {
549 // Handle the general case. Each extract is actually a different value:
550 // opcode (extelt V0, C0), (extelt V1, C1) --> extelt (opcode V0, V1), C
551 OldCost = Extract0Cost + Extract1Cost + ScalarOpCost;
552 NewCost = VectorOpCost + CheapExtractCost +
553 !Ext0->hasOneUse() * Extract0Cost +
554 !Ext1->hasOneUse() * Extract1Cost;
555 }
556
557 ConvertToShuffle = getShuffleExtract(Ext0, Ext1, PreferredExtractIndex);
558 if (ConvertToShuffle) {
559 if (IsBinOp && Opts.disable_binop_extract_shuffle)
560 return true;
561
562 // If we are extracting from 2 different indexes, then one operand must be
563 // shuffled before performing the vector operation. The shuffle mask is
564 // poison except for 1 lane that is being translated to the remaining
565 // extraction lane. Therefore, it is a splat shuffle. Ex:
566 // ShufMask = { poison, poison, 0, poison }
567 // TODO: The cost model has an option for a "broadcast" shuffle
568 // (splat-from-element-0), but no option for a more general splat.
569 if (auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: VecTy)) {
570 SmallVector<int> ShuffleMask(FixedVecTy->getNumElements(),
571 PoisonMaskElem);
572 ShuffleMask[BestInsIndex] = BestExtIndex;
573 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
574 DstTy: VecTy, SrcTy: VecTy, CostKind, Mask: ShuffleMask, Index: 0,
575 SubTp: nullptr, Args: {ConvertToShuffle});
576 } else {
577 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
578 DstTy: VecTy, SrcTy: VecTy, CostKind, Mask: {}, Index: 0, SubTp: nullptr,
579 Args: {ConvertToShuffle});
580 }
581 }
582
583 LLVM_DEBUG(dbgs() << "Found a binop of extractions: " << I << "\n OldCost: "
584 << OldCost << " vs NewCost: " << NewCost << "\n");
585
586 // Aggressively form a vector op if the cost is equal because the transform
587 // may enable further optimization.
588 // Codegen can reverse this transform (scalarize) if it was not profitable.
589 return OldCost < NewCost;
590}
591
592/// Create a shuffle that translates (shifts) 1 element from the input vector
593/// to a new element location.
594static Value *createShiftShuffle(Value *Vec, unsigned OldIndex,
595 unsigned NewIndex, IRBuilderBase &Builder) {
596 // The shuffle mask is poison except for 1 lane that is being translated
597 // to the new element index. Example for OldIndex == 2 and NewIndex == 0:
598 // ShufMask = { 2, poison, poison, poison }
599 auto *VecTy = cast<FixedVectorType>(Val: Vec->getType());
600 SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
601 ShufMask[NewIndex] = OldIndex;
602 return Builder.CreateShuffleVector(V: Vec, Mask: ShufMask, Name: "shift");
603}
604
605/// Given an extract element instruction with constant index operand, shuffle
606/// the source vector (shift the scalar element) to a NewIndex for extraction.
607/// Return null if the input can be constant folded, so that we are not creating
608/// unnecessary instructions.
609static Value *translateExtract(ExtractElementInst *ExtElt, unsigned NewIndex,
610 IRBuilderBase &Builder) {
611 // Shufflevectors can only be created for fixed-width vectors.
612 Value *X = ExtElt->getVectorOperand();
613 if (!isa<FixedVectorType>(Val: X->getType()))
614 return nullptr;
615
616 // If the extract can be constant-folded, this code is unsimplified. Defer
617 // to other passes to handle that.
618 Value *C = ExtElt->getIndexOperand();
619 assert(isa<ConstantInt>(C) && "Expected a constant index operand");
620 if (isa<Constant>(Val: X))
621 return nullptr;
622
623 Value *Shuf = createShiftShuffle(Vec: X, OldIndex: cast<ConstantInt>(Val: C)->getZExtValue(),
624 NewIndex, Builder);
625 return Shuf;
626}
627
628/// Try to reduce extract element costs by converting scalar compares to vector
629/// compares followed by extract.
630/// cmp (ext0 V0, ExtIndex), (ext1 V1, ExtIndex)
631Value *VectorCombine::foldExtExtCmp(Value *V0, Value *V1, Value *ExtIndex,
632 Instruction &I) {
633 assert(isa<CmpInst>(&I) && "Expected a compare");
634
635 // cmp Pred (extelt V0, ExtIndex), (extelt V1, ExtIndex)
636 // --> extelt (cmp Pred V0, V1), ExtIndex
637 ++NumVecCmp;
638 CmpInst::Predicate Pred = cast<CmpInst>(Val: &I)->getPredicate();
639 Value *VecCmp = Builder.CreateCmp(Pred, LHS: V0, RHS: V1);
640 return Builder.CreateExtractElement(Vec: VecCmp, Idx: ExtIndex, Name: "foldExtExtCmp");
641}
642
643/// Try to reduce extract element costs by converting scalar binops to vector
644/// binops followed by extract.
645/// bo (ext0 V0, ExtIndex), (ext1 V1, ExtIndex)
646Value *VectorCombine::foldExtExtBinop(Value *V0, Value *V1, Value *ExtIndex,
647 Instruction &I) {
648 assert(isa<BinaryOperator>(&I) && "Expected a binary operator");
649
650 // bo (extelt V0, ExtIndex), (extelt V1, ExtIndex)
651 // --> extelt (bo V0, V1), ExtIndex
652 ++NumVecBO;
653 Value *VecBO = Builder.CreateBinOp(Opc: cast<BinaryOperator>(Val: &I)->getOpcode(), LHS: V0,
654 RHS: V1, Name: "foldExtExtBinop");
655
656 // All IR flags are safe to back-propagate because any potential poison
657 // created in unused vector elements is discarded by the extract.
658 if (auto *VecBOInst = dyn_cast<Instruction>(Val: VecBO))
659 VecBOInst->copyIRFlags(V: &I);
660
661 return Builder.CreateExtractElement(Vec: VecBO, Idx: ExtIndex, Name: "foldExtExtBinop");
662}
663
664/// Match an instruction with extracted vector operands.
665bool VectorCombine::foldExtractExtract(Instruction &I) {
666 // It is not safe to transform things like div, urem, etc. because we may
667 // create undefined behavior when executing those on unknown vector elements.
668 if (!isSafeToSpeculativelyExecute(I: &I))
669 return false;
670
671 Instruction *I0, *I1;
672 CmpPredicate Pred = CmpInst::BAD_ICMP_PREDICATE;
673 if (!match(V: &I, P: m_Cmp(Pred, L: m_Instruction(I&: I0), R: m_Instruction(I&: I1))) &&
674 !match(V: &I, P: m_BinOp(L: m_Instruction(I&: I0), R: m_Instruction(I&: I1))))
675 return false;
676
677 Value *V0, *V1;
678 uint64_t C0, C1;
679 if (!match(V: I0, P: m_ExtractElt(Val: m_Value(V&: V0), Idx: m_ConstantInt(V&: C0))) ||
680 !match(V: I1, P: m_ExtractElt(Val: m_Value(V&: V1), Idx: m_ConstantInt(V&: C1))) ||
681 V0->getType() != V1->getType())
682 return false;
683
684 // For fixed-width vectors, reject out-of-bounds extract indexes
685 if (auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: V0->getType())) {
686 unsigned NumElts = FixedVecTy->getNumElements();
687 if (C0 >= NumElts || C1 >= NumElts)
688 return false;
689 }
690
691 // If the scalar value 'I' is going to be re-inserted into a vector, then try
692 // to create an extract to that same element. The extract/insert can be
693 // reduced to a "select shuffle".
694 // TODO: If we add a larger pattern match that starts from an insert, this
695 // probably becomes unnecessary.
696 auto *Ext0 = cast<ExtractElementInst>(Val: I0);
697 auto *Ext1 = cast<ExtractElementInst>(Val: I1);
698 uint64_t InsertIndex = InvalidIndex;
699 if (I.hasOneUse())
700 match(V: I.user_back(),
701 P: m_InsertElt(Val: m_Value(), Elt: m_Value(), Idx: m_ConstantInt(V&: InsertIndex)));
702
703 ExtractElementInst *ExtractToChange;
704 if (isExtractExtractCheap(Ext0, Ext1, I, ConvertToShuffle&: ExtractToChange, PreferredExtractIndex: InsertIndex))
705 return false;
706
707 Value *ExtOp0 = Ext0->getVectorOperand();
708 Value *ExtOp1 = Ext1->getVectorOperand();
709
710 if (ExtractToChange) {
711 unsigned CheapExtractIdx = ExtractToChange == Ext0 ? C1 : C0;
712 Value *NewExtOp =
713 translateExtract(ExtElt: ExtractToChange, NewIndex: CheapExtractIdx, Builder);
714 if (!NewExtOp)
715 return false;
716 if (ExtractToChange == Ext0)
717 ExtOp0 = NewExtOp;
718 else
719 ExtOp1 = NewExtOp;
720 }
721
722 Value *ExtIndex = ExtractToChange == Ext0 ? Ext1->getIndexOperand()
723 : Ext0->getIndexOperand();
724 Value *NewExt = Pred != CmpInst::BAD_ICMP_PREDICATE
725 ? foldExtExtCmp(V0: ExtOp0, V1: ExtOp1, ExtIndex, I)
726 : foldExtExtBinop(V0: ExtOp0, V1: ExtOp1, ExtIndex, I);
727 replaceValue(Old&: I, New&: *NewExt);
728 return true;
729}
730
731/// Try to replace an extract + scalar fneg + insert with a vector fneg +
732/// shuffle.
733bool VectorCombine::foldInsExtFNeg(Instruction &I) {
734 // Match an insert (op (extract)) pattern.
735 Value *DstVec;
736 uint64_t ExtIdx, InsIdx;
737 Instruction *FNeg;
738 if (!match(V: &I, P: m_InsertElt(Val: m_Value(V&: DstVec), Elt: m_OneUse(SubPattern: m_Instruction(I&: FNeg)),
739 Idx: m_ConstantInt(V&: InsIdx))))
740 return false;
741
742 // Note: This handles the canonical fneg instruction and "fsub -0.0, X".
743 Value *SrcVec;
744 Instruction *Extract;
745 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
746 if (!match(V: FNeg, P: m_FNegOrFSubNegZero(X: m_CombineAnd(
747 Ps: m_Instruction(I&: Extract),
748 Ps: m_ExtractElt(Val: m_Value(V&: SrcVec), Idx: m_ConstantInt(V&: ExtIdx))))))
749 return false;
750
751 auto *DstVecTy = cast<FixedVectorType>(Val: DstVec->getType());
752 auto *DstVecScalarTy = DstVecTy->getScalarType();
753 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcVec->getType());
754 if (!SrcVecTy || DstVecScalarTy != SrcVecTy->getScalarType())
755 return false;
756
757 // Ignore if insert/extract index is out of bounds or destination vector has
758 // one element
759 unsigned NumDstElts = DstVecTy->getNumElements();
760 unsigned NumSrcElts = SrcVecTy->getNumElements();
761 if (ExtIdx > NumSrcElts || InsIdx >= NumDstElts || NumDstElts == 1)
762 return false;
763
764 // We are inserting the negated element into the same lane that we extracted
765 // from. This is equivalent to a select-shuffle that chooses all but the
766 // negated element from the destination vector.
767 SmallVector<int> Mask(NumDstElts);
768 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
769 Mask[InsIdx] = (ExtIdx % NumDstElts) + NumDstElts;
770 InstructionCost OldCost =
771 TTI.getArithmeticInstrCost(Opcode: Instruction::FNeg, Ty: DstVecScalarTy, CostKind) +
772 TTI.getVectorInstrCost(I, Val: DstVecTy, CostKind, Index: InsIdx);
773
774 // If the extract has one use, it will be eliminated, so count it in the
775 // original cost. If it has more than one use, ignore the cost because it will
776 // be the same before/after.
777 if (Extract->hasOneUse())
778 OldCost += TTI.getVectorInstrCost(I: *Extract, Val: SrcVecTy, CostKind, Index: ExtIdx);
779
780 InstructionCost NewCost =
781 TTI.getArithmeticInstrCost(Opcode: Instruction::FNeg, Ty: SrcVecTy, CostKind) +
782 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: DstVecTy,
783 SrcTy: DstVecTy, CostKind, Mask);
784
785 bool NeedLenChg = SrcVecTy->getNumElements() != NumDstElts;
786 // If the lengths of the two vectors are not equal,
787 // we need to add a length-change vector. Add this cost.
788 SmallVector<int> SrcMask;
789 if (NeedLenChg) {
790 SrcMask.assign(NumElts: NumDstElts, Elt: PoisonMaskElem);
791 SrcMask[ExtIdx % NumDstElts] = ExtIdx;
792 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
793 DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask: SrcMask);
794 }
795
796 LLVM_DEBUG(dbgs() << "Found an insertion of (extract)fneg : " << I
797 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
798 << "\n");
799 if (NewCost > OldCost)
800 return false;
801
802 Value *NewShuf, *LenChgShuf = nullptr;
803 // insertelt DstVec, (fneg (extractelt SrcVec, Index)), Index
804 Value *VecFNeg = Builder.CreateFNegFMF(V: SrcVec, FMFSource: FNeg);
805 if (NeedLenChg) {
806 // shuffle DstVec, (shuffle (fneg SrcVec), poison, SrcMask), Mask
807 LenChgShuf = Builder.CreateShuffleVector(V: VecFNeg, Mask: SrcMask);
808 NewShuf = Builder.CreateShuffleVector(V1: DstVec, V2: LenChgShuf, Mask);
809 } else {
810 // shuffle DstVec, (fneg SrcVec), Mask
811 NewShuf = Builder.CreateShuffleVector(V1: DstVec, V2: VecFNeg, Mask);
812 }
813
814 replaceValue(Old&: I, New&: *NewShuf);
815 return true;
816}
817
818/// Try to fold insert(binop(x,y),binop(a,b),idx)
819/// --> binop(insert(x,a,idx),insert(y,b,idx))
820bool VectorCombine::foldInsExtBinop(Instruction &I) {
821 BinaryOperator *VecBinOp, *SclBinOp;
822 uint64_t Index;
823 if (!match(V: &I,
824 P: m_InsertElt(Val: m_OneUse(SubPattern: m_BinOp(I&: VecBinOp)),
825 Elt: m_OneUse(SubPattern: m_BinOp(I&: SclBinOp)), Idx: m_ConstantInt(V&: Index))))
826 return false;
827
828 // TODO: Add support for addlike etc.
829 Instruction::BinaryOps BinOpcode = VecBinOp->getOpcode();
830 if (BinOpcode != SclBinOp->getOpcode())
831 return false;
832
833 auto *ResultTy = dyn_cast<FixedVectorType>(Val: I.getType());
834 if (!ResultTy)
835 return false;
836
837 // TODO: Attempt to detect m_ExtractElt for scalar operands and convert to
838 // shuffle?
839
840 InstructionCost OldCost = TTI.getInstructionCost(U: &I, CostKind) +
841 TTI.getInstructionCost(U: VecBinOp, CostKind) +
842 TTI.getInstructionCost(U: SclBinOp, CostKind);
843 InstructionCost NewCost =
844 TTI.getArithmeticInstrCost(Opcode: BinOpcode, Ty: ResultTy, CostKind) +
845 TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: ResultTy, CostKind,
846 Index, Op0: VecBinOp->getOperand(i_nocapture: 0),
847 Op1: SclBinOp->getOperand(i_nocapture: 0)) +
848 TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: ResultTy, CostKind,
849 Index, Op0: VecBinOp->getOperand(i_nocapture: 1),
850 Op1: SclBinOp->getOperand(i_nocapture: 1));
851
852 LLVM_DEBUG(dbgs() << "Found an insertion of two binops: " << I
853 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
854 << "\n");
855 if (NewCost > OldCost)
856 return false;
857
858 Value *NewIns0 = Builder.CreateInsertElement(Vec: VecBinOp->getOperand(i_nocapture: 0),
859 NewElt: SclBinOp->getOperand(i_nocapture: 0), Idx: Index);
860 Value *NewIns1 = Builder.CreateInsertElement(Vec: VecBinOp->getOperand(i_nocapture: 1),
861 NewElt: SclBinOp->getOperand(i_nocapture: 1), Idx: Index);
862 Value *NewBO = Builder.CreateBinOp(Opc: BinOpcode, LHS: NewIns0, RHS: NewIns1);
863
864 // Intersect flags from the old binops.
865 if (auto *NewInst = dyn_cast<Instruction>(Val: NewBO)) {
866 NewInst->copyIRFlags(V: VecBinOp);
867 NewInst->andIRFlags(V: SclBinOp);
868 }
869
870 replaceValue(Old&: I, New&: *NewBO);
871 return true;
872}
873
874/// Match: bitop(castop(x), castop(y)) -> castop(bitop(x, y))
875/// Supports: bitcast, trunc, sext, zext
876bool VectorCombine::foldBitOpOfCastops(Instruction &I) {
877 // Check if this is a bitwise logic operation
878 auto *BinOp = dyn_cast<BinaryOperator>(Val: &I);
879 if (!BinOp || !BinOp->isBitwiseLogicOp())
880 return false;
881
882 // Get the cast instructions
883 auto *LHSCast = dyn_cast<CastInst>(Val: BinOp->getOperand(i_nocapture: 0));
884 auto *RHSCast = dyn_cast<CastInst>(Val: BinOp->getOperand(i_nocapture: 1));
885 if (!LHSCast || !RHSCast) {
886 LLVM_DEBUG(dbgs() << " One or both operands are not cast instructions\n");
887 return false;
888 }
889
890 // Both casts must be the same type
891 Instruction::CastOps CastOpcode = LHSCast->getOpcode();
892 if (CastOpcode != RHSCast->getOpcode())
893 return false;
894
895 // Only handle supported cast operations
896 switch (CastOpcode) {
897 case Instruction::BitCast:
898 case Instruction::Trunc:
899 case Instruction::SExt:
900 case Instruction::ZExt:
901 break;
902 default:
903 return false;
904 }
905
906 Value *LHSSrc = LHSCast->getOperand(i_nocapture: 0);
907 Value *RHSSrc = RHSCast->getOperand(i_nocapture: 0);
908
909 // Source types must match
910 if (LHSSrc->getType() != RHSSrc->getType())
911 return false;
912
913 auto *SrcTy = LHSSrc->getType();
914 auto *DstTy = I.getType();
915 // Bitcasts can handle scalar/vector mixes, such as i16 -> <16 x i1>.
916 // Other casts only handle vector types with integer elements.
917 if (CastOpcode != Instruction::BitCast &&
918 (!isa<FixedVectorType>(Val: SrcTy) || !isa<FixedVectorType>(Val: DstTy)))
919 return false;
920
921 // Only integer scalar/vector values are legal for bitwise logic operations.
922 if (!SrcTy->getScalarType()->isIntegerTy() ||
923 !DstTy->getScalarType()->isIntegerTy())
924 return false;
925
926 // Cost Check :
927 // OldCost = bitlogic + 2*casts
928 // NewCost = bitlogic + cast
929
930 // Calculate specific costs for each cast with instruction context
931 InstructionCost LHSCastCost = TTI.getCastInstrCost(
932 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind, I: LHSCast);
933 InstructionCost RHSCastCost = TTI.getCastInstrCost(
934 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind, I: RHSCast);
935
936 InstructionCost OldCost =
937 TTI.getArithmeticInstrCost(Opcode: BinOp->getOpcode(), Ty: DstTy, CostKind) +
938 LHSCastCost + RHSCastCost;
939
940 // For new cost, we can't provide an instruction (it doesn't exist yet)
941 InstructionCost GenericCastCost = TTI.getCastInstrCost(
942 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
943
944 InstructionCost NewCost =
945 TTI.getArithmeticInstrCost(Opcode: BinOp->getOpcode(), Ty: SrcTy, CostKind) +
946 GenericCastCost;
947
948 // Account for multi-use casts using specific costs
949 if (!LHSCast->hasOneUse())
950 NewCost += LHSCastCost;
951 if (!RHSCast->hasOneUse())
952 NewCost += RHSCastCost;
953
954 LLVM_DEBUG(dbgs() << "foldBitOpOfCastops: OldCost=" << OldCost
955 << " NewCost=" << NewCost << "\n");
956
957 if (NewCost > OldCost)
958 return false;
959
960 // Create the operation on the source type
961 Value *NewOp = Builder.CreateBinOp(Opc: BinOp->getOpcode(), LHS: LHSSrc, RHS: RHSSrc,
962 Name: BinOp->getName() + ".inner");
963 if (auto *NewBinOp = dyn_cast<BinaryOperator>(Val: NewOp))
964 NewBinOp->copyIRFlags(V: BinOp);
965
966 // Create the cast operation directly to ensure we get a new instruction
967 Instruction *NewCast = CastInst::Create(CastOpcode, S: NewOp, Ty: I.getType());
968
969 // Preserve cast instruction flags
970 NewCast->copyIRFlags(V: LHSCast);
971 NewCast->andIRFlags(V: RHSCast);
972
973 // Insert the new instruction
974 Value *Result = Builder.Insert(I: NewCast);
975
976 replaceValue(Old&: I, New&: *Result);
977 return true;
978}
979
980/// Match:
981// bitop(castop(x), C) ->
982// bitop(castop(x), castop(InvC)) ->
983// castop(bitop(x, InvC))
984// Supports: bitcast
985bool VectorCombine::foldBitOpOfCastConstant(Instruction &I) {
986 Instruction *LHS;
987 Constant *C;
988
989 // Check if this is a bitwise logic operation
990 if (!match(V: &I, P: m_c_BitwiseLogic(L: m_Instruction(I&: LHS), R: m_Constant(C))))
991 return false;
992
993 // Get the cast instructions
994 auto *LHSCast = dyn_cast<CastInst>(Val: LHS);
995 if (!LHSCast)
996 return false;
997
998 Instruction::CastOps CastOpcode = LHSCast->getOpcode();
999
1000 // Only handle supported cast operations
1001 switch (CastOpcode) {
1002 case Instruction::BitCast:
1003 case Instruction::ZExt:
1004 case Instruction::SExt:
1005 case Instruction::Trunc:
1006 break;
1007 default:
1008 return false;
1009 }
1010
1011 Value *LHSSrc = LHSCast->getOperand(i_nocapture: 0);
1012
1013 auto *SrcTy = LHSSrc->getType();
1014 auto *DstTy = I.getType();
1015 // Bitcasts can handle scalar/vector mixes, such as i16 -> <16 x i1>.
1016 // Other casts only handle vector types with integer elements.
1017 if (CastOpcode != Instruction::BitCast &&
1018 (!isa<FixedVectorType>(Val: SrcTy) || !isa<FixedVectorType>(Val: DstTy)))
1019 return false;
1020
1021 // Only integer scalar/vector values are legal for bitwise logic operations.
1022 if (!SrcTy->getScalarType()->isIntegerTy() ||
1023 !DstTy->getScalarType()->isIntegerTy())
1024 return false;
1025
1026 // Find the constant InvC, such that castop(InvC) equals to C.
1027 PreservedCastFlags RHSFlags;
1028 Constant *InvC = getLosslessInvCast(C, InvCastTo: SrcTy, CastOp: CastOpcode, DL: *DL, Flags: &RHSFlags);
1029 if (!InvC)
1030 return false;
1031
1032 // Cost Check :
1033 // OldCost = bitlogic + cast
1034 // NewCost = bitlogic + cast
1035
1036 // Calculate specific costs for each cast with instruction context
1037 InstructionCost LHSCastCost = TTI.getCastInstrCost(
1038 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind, I: LHSCast);
1039
1040 InstructionCost OldCost =
1041 TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: DstTy, CostKind) + LHSCastCost;
1042
1043 // For new cost, we can't provide an instruction (it doesn't exist yet)
1044 InstructionCost GenericCastCost = TTI.getCastInstrCost(
1045 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
1046
1047 InstructionCost NewCost =
1048 TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: SrcTy, CostKind) +
1049 GenericCastCost;
1050
1051 // Account for multi-use casts using specific costs
1052 if (!LHSCast->hasOneUse())
1053 NewCost += LHSCastCost;
1054
1055 LLVM_DEBUG(dbgs() << "foldBitOpOfCastConstant: OldCost=" << OldCost
1056 << " NewCost=" << NewCost << "\n");
1057
1058 if (NewCost > OldCost)
1059 return false;
1060
1061 // Create the operation on the source type
1062 Value *NewOp = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)I.getOpcode(),
1063 LHS: LHSSrc, RHS: InvC, Name: I.getName() + ".inner");
1064 if (auto *NewBinOp = dyn_cast<BinaryOperator>(Val: NewOp))
1065 NewBinOp->copyIRFlags(V: &I);
1066
1067 // Create the cast operation directly to ensure we get a new instruction
1068 Instruction *NewCast = CastInst::Create(CastOpcode, S: NewOp, Ty: I.getType());
1069
1070 // Preserve cast instruction flags
1071 if (RHSFlags.NNeg)
1072 NewCast->setNonNeg();
1073 if (RHSFlags.NUW)
1074 NewCast->setHasNoUnsignedWrap();
1075 if (RHSFlags.NSW)
1076 NewCast->setHasNoSignedWrap();
1077
1078 NewCast->andIRFlags(V: LHSCast);
1079
1080 // Insert the new instruction
1081 Value *Result = Builder.Insert(I: NewCast);
1082
1083 replaceValue(Old&: I, New&: *Result);
1084 return true;
1085}
1086
1087/// If this is a bitcast of a shuffle, try to bitcast the source vector to the
1088/// destination type followed by shuffle. This can enable further transforms by
1089/// moving bitcasts or shuffles together.
1090bool VectorCombine::foldBitcastShuffle(Instruction &I) {
1091 Value *V0, *V1;
1092 ArrayRef<int> Mask;
1093 if (!match(V: &I, P: m_BitCast(Op: m_OneUse(
1094 SubPattern: m_Shuffle(v1: m_Value(V&: V0), v2: m_Value(V&: V1), mask: m_Mask(Mask))))))
1095 return false;
1096
1097 // 1) Do not fold bitcast shuffle for scalable type. First, shuffle cost for
1098 // scalable type is unknown; Second, we cannot reason if the narrowed shuffle
1099 // mask for scalable type is a splat or not.
1100 // 2) Disallow non-vector casts.
1101 // TODO: We could allow any shuffle.
1102 auto *DestTy = dyn_cast<FixedVectorType>(Val: I.getType());
1103 auto *SrcTy = dyn_cast<FixedVectorType>(Val: V0->getType());
1104 if (!DestTy || !SrcTy)
1105 return false;
1106
1107 unsigned DestEltSize = DestTy->getScalarSizeInBits();
1108 unsigned SrcEltSize = SrcTy->getScalarSizeInBits();
1109 if (SrcTy->getPrimitiveSizeInBits() % DestEltSize != 0)
1110 return false;
1111
1112 bool IsUnary = isa<UndefValue>(Val: V1);
1113
1114 // For binary shuffles, only fold bitcast(shuffle(X,Y))
1115 // if it won't increase the number of bitcasts.
1116 if (!IsUnary) {
1117 auto *BCTy0 = dyn_cast<FixedVectorType>(Val: peekThroughBitcasts(V: V0)->getType());
1118 auto *BCTy1 = dyn_cast<FixedVectorType>(Val: peekThroughBitcasts(V: V1)->getType());
1119 if (!(BCTy0 && BCTy0->getElementType() == DestTy->getElementType()) &&
1120 !(BCTy1 && BCTy1->getElementType() == DestTy->getElementType()))
1121 return false;
1122 }
1123
1124 SmallVector<int, 16> NewMask;
1125 if (DestEltSize <= SrcEltSize) {
1126 // The bitcast is from wide to narrow/equal elements. The shuffle mask can
1127 // always be expanded to the equivalent form choosing narrower elements.
1128 if (SrcEltSize % DestEltSize != 0)
1129 return false;
1130 unsigned ScaleFactor = SrcEltSize / DestEltSize;
1131 narrowShuffleMaskElts(Scale: ScaleFactor, Mask, ScaledMask&: NewMask);
1132 } else {
1133 // The bitcast is from narrow elements to wide elements. The shuffle mask
1134 // must choose consecutive elements to allow casting first.
1135 if (DestEltSize % SrcEltSize != 0)
1136 return false;
1137 unsigned ScaleFactor = DestEltSize / SrcEltSize;
1138 if (!widenShuffleMaskElts(Scale: ScaleFactor, Mask, ScaledMask&: NewMask))
1139 return false;
1140 }
1141
1142 // Bitcast the shuffle src - keep its original width but using the destination
1143 // scalar type.
1144 unsigned NumSrcElts = SrcTy->getPrimitiveSizeInBits() / DestEltSize;
1145 auto *NewShuffleTy =
1146 FixedVectorType::get(ElementType: DestTy->getScalarType(), NumElts: NumSrcElts);
1147 auto *OldShuffleTy =
1148 FixedVectorType::get(ElementType: SrcTy->getScalarType(), NumElts: Mask.size());
1149 unsigned NumOps = IsUnary ? 1 : 2;
1150
1151 // The new shuffle must not cost more than the old shuffle.
1152 TargetTransformInfo::ShuffleKind SK =
1153 IsUnary ? TargetTransformInfo::SK_PermuteSingleSrc
1154 : TargetTransformInfo::SK_PermuteTwoSrc;
1155
1156 InstructionCost NewCost =
1157 TTI.getShuffleCost(Kind: SK, DstTy: DestTy, SrcTy: NewShuffleTy, CostKind, Mask: NewMask) +
1158 (NumOps * TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: NewShuffleTy, Src: SrcTy,
1159 CCH: TargetTransformInfo::CastContextHint::None,
1160 CostKind));
1161 InstructionCost OldCost =
1162 TTI.getShuffleCost(Kind: SK, DstTy: OldShuffleTy, SrcTy, CostKind, Mask) +
1163 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: DestTy, Src: OldShuffleTy,
1164 CCH: TargetTransformInfo::CastContextHint::None,
1165 CostKind);
1166
1167 LLVM_DEBUG(dbgs() << "Found a bitcasted shuffle: " << I << "\n OldCost: "
1168 << OldCost << " vs NewCost: " << NewCost << "\n");
1169
1170 if (NewCost > OldCost || !NewCost.isValid())
1171 return false;
1172
1173 // bitcast (shuf V0, V1, MaskC) --> shuf (bitcast V0), (bitcast V1), MaskC'
1174 ++NumShufOfBitcast;
1175 Value *CastV0 = Builder.CreateBitCast(V: peekThroughBitcasts(V: V0), DestTy: NewShuffleTy);
1176 Value *CastV1 = Builder.CreateBitCast(V: peekThroughBitcasts(V: V1), DestTy: NewShuffleTy);
1177 Value *Shuf = Builder.CreateShuffleVector(V1: CastV0, V2: CastV1, Mask: NewMask);
1178 replaceValue(Old&: I, New&: *Shuf);
1179 return true;
1180}
1181
1182/// Match a vector op/compare/intrinsic with at least one
1183/// inserted scalar operand and convert to scalar op/cmp/intrinsic followed
1184/// by insertelement.
1185bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
1186 auto *UO = dyn_cast<UnaryOperator>(Val: &I);
1187 auto *BO = dyn_cast<BinaryOperator>(Val: &I);
1188 auto *CI = dyn_cast<CmpInst>(Val: &I);
1189 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
1190 if (!UO && !BO && !CI && !II)
1191 return false;
1192
1193 // TODO: Allow intrinsics with different argument types
1194 if (II) {
1195 if (!isTriviallyVectorizable(ID: II->getIntrinsicID()))
1196 return false;
1197 for (auto [Idx, Arg] : enumerate(First: II->args()))
1198 if (Arg->getType() != II->getType() &&
1199 !isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(), ScalarOpdIdx: Idx, TTI: &TTI))
1200 return false;
1201 }
1202
1203 // Do not convert the vector condition of a vector select into a scalar
1204 // condition. That may cause problems for codegen because of differences in
1205 // boolean formats and register-file transfers.
1206 // TODO: Can we account for that in the cost model?
1207 if (CI)
1208 for (User *U : I.users())
1209 if (match(V: U, P: m_Select(C: m_Specific(V: &I), L: m_Value(), R: m_Value())))
1210 return false;
1211
1212 // Match constant vectors or scalars being inserted into constant vectors:
1213 // vec_op [VecC0 | (inselt VecC0, V0, Index)], ...
1214 SmallVector<Value *> VecCs, ScalarOps;
1215 std::optional<uint64_t> Index;
1216
1217 auto Ops = II ? II->args() : I.operands();
1218 for (auto [OpNum, Op] : enumerate(First&: Ops)) {
1219 Constant *VecC;
1220 Value *V;
1221 uint64_t InsIdx = 0;
1222 if (match(V: Op.get(), P: m_InsertElt(Val: m_Constant(C&: VecC), Elt: m_Value(V),
1223 Idx: m_ConstantInt(V&: InsIdx)))) {
1224 // Bail if any inserts are out of bounds.
1225 VectorType *OpTy = cast<VectorType>(Val: Op->getType());
1226 if (OpTy->getElementCount().getKnownMinValue() <= InsIdx)
1227 return false;
1228 // All inserts must have the same index.
1229 // TODO: Deal with mismatched index constants and variable indexes?
1230 if (!Index)
1231 Index = InsIdx;
1232 else if (InsIdx != *Index)
1233 return false;
1234 VecCs.push_back(Elt: VecC);
1235 ScalarOps.push_back(Elt: V);
1236 } else if (II && isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(),
1237 ScalarOpdIdx: OpNum, TTI: &TTI)) {
1238 VecCs.push_back(Elt: Op.get());
1239 ScalarOps.push_back(Elt: Op.get());
1240 } else if (match(V: Op.get(), P: m_Constant(C&: VecC))) {
1241 VecCs.push_back(Elt: VecC);
1242 ScalarOps.push_back(Elt: nullptr);
1243 } else {
1244 return false;
1245 }
1246 }
1247
1248 // Bail if all operands are constant.
1249 if (!Index.has_value())
1250 return false;
1251
1252 VectorType *VecTy = cast<VectorType>(Val: I.getType());
1253 Type *ScalarTy = VecTy->getScalarType();
1254 assert(VecTy->isVectorTy() &&
1255 (ScalarTy->isIntegerTy() || ScalarTy->isFloatingPointTy() ||
1256 ScalarTy->isPointerTy()) &&
1257 "Unexpected types for insert element into binop or cmp");
1258
1259 unsigned Opcode = I.getOpcode();
1260 InstructionCost ScalarOpCost, VectorOpCost;
1261 if (CI) {
1262 CmpInst::Predicate Pred = CI->getPredicate();
1263 ScalarOpCost = TTI.getCmpSelInstrCost(
1264 Opcode, ValTy: ScalarTy, CondTy: CmpInst::makeCmpResultType(opnd_type: ScalarTy), VecPred: Pred, CostKind);
1265 VectorOpCost = TTI.getCmpSelInstrCost(
1266 Opcode, ValTy: VecTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VecTy), VecPred: Pred, CostKind);
1267 } else if (UO || BO) {
1268 ScalarOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: ScalarTy, CostKind);
1269 VectorOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1270 } else {
1271 IntrinsicCostAttributes ScalarICA(
1272 II->getIntrinsicID(), ScalarTy,
1273 SmallVector<Type *>(II->arg_size(), ScalarTy));
1274 ScalarOpCost = TTI.getIntrinsicInstrCost(ICA: ScalarICA, CostKind);
1275 IntrinsicCostAttributes VectorICA(
1276 II->getIntrinsicID(), VecTy,
1277 SmallVector<Type *>(II->arg_size(), VecTy));
1278 VectorOpCost = TTI.getIntrinsicInstrCost(ICA: VectorICA, CostKind);
1279 }
1280
1281 // Fold the vector constants in the original vectors into a new base vector to
1282 // get more accurate cost modelling.
1283 Value *NewVecC = nullptr;
1284 if (CI)
1285 NewVecC = simplifyCmpInst(Predicate: CI->getPredicate(), LHS: VecCs[0], RHS: VecCs[1], Q: SQ);
1286 else if (UO)
1287 NewVecC =
1288 simplifyUnOp(Opcode: UO->getOpcode(), Op: VecCs[0], FMF: UO->getFastMathFlags(), Q: SQ);
1289 else if (BO)
1290 NewVecC = simplifyBinOp(Opcode: BO->getOpcode(), LHS: VecCs[0], RHS: VecCs[1], Q: SQ);
1291 else if (II)
1292 NewVecC = simplifyCall(Call: II, Callee: II->getCalledOperand(), Args: VecCs, Q: SQ);
1293
1294 if (!NewVecC)
1295 return false;
1296
1297 // Get cost estimate for the insert element. This cost will factor into
1298 // both sequences.
1299 InstructionCost OldCost = VectorOpCost;
1300 InstructionCost NewCost =
1301 ScalarOpCost + TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: VecTy,
1302 CostKind, Index: *Index, Op0: NewVecC);
1303
1304 for (auto [Idx, Op, VecC, Scalar] : enumerate(First&: Ops, Rest&: VecCs, Rest&: ScalarOps)) {
1305 if (!Scalar || (II && isVectorIntrinsicWithScalarOpAtArg(
1306 ID: II->getIntrinsicID(), ScalarOpdIdx: Idx, TTI: &TTI)))
1307 continue;
1308 InstructionCost InsertCost = TTI.getVectorInstrCost(
1309 Opcode: Instruction::InsertElement, Val: VecTy, CostKind, Index: *Index, Op0: VecC, Op1: Scalar);
1310 OldCost += InsertCost;
1311 NewCost += !Op->hasOneUse() * InsertCost;
1312 }
1313
1314 // We want to scalarize unless the vector variant actually has lower cost.
1315 if (OldCost < NewCost || !NewCost.isValid())
1316 return false;
1317
1318 // vec_op (inselt VecC0, V0, Index), (inselt VecC1, V1, Index) -->
1319 // inselt NewVecC, (scalar_op V0, V1), Index
1320 if (CI)
1321 ++NumScalarCmp;
1322 else if (UO || BO)
1323 ++NumScalarOps;
1324 else
1325 ++NumScalarIntrinsic;
1326
1327 // For constant cases, extract the scalar element, this should constant fold.
1328 for (auto [OpIdx, Scalar, VecC] : enumerate(First&: ScalarOps, Rest&: VecCs))
1329 if (!Scalar)
1330 ScalarOps[OpIdx] = ConstantExpr::getExtractElement(
1331 Vec: cast<Constant>(Val: VecC), Idx: Builder.getInt64(C: *Index));
1332
1333 Value *Scalar;
1334 // We need to pass the flags during the creation of instructions. Constant
1335 // folding might remove the instructions, so post setting the flags might
1336 // pollute the later instructions.
1337 if (CI) {
1338 if (FPMathOperator *FPMO = dyn_cast<FPMathOperator>(Val: &I)) {
1339 Scalar = Builder.CreateFCmpFMF(P: CI->getPredicate(), LHS: ScalarOps[0],
1340 RHS: ScalarOps[1], FMFSource: FPMO->getFastMathFlags(),
1341 Name: CI->getName() + ".scalar");
1342 } else {
1343 Scalar = Builder.CreateICmp(P: CI->getPredicate(), LHS: ScalarOps[0],
1344 RHS: ScalarOps[1], Name: CI->getName() + ".scalar");
1345 }
1346 } else if (UO) {
1347 Scalar = Builder.CreateUnOpFMF(Opc: UO->getOpcode(), V: ScalarOps[0], FMFSource: UO,
1348 Name: UO->getName() + ".scalar");
1349 } else if (BO) {
1350 if (OverflowingBinaryOperator *OBO =
1351 dyn_cast<OverflowingBinaryOperator>(Val: &I)) {
1352 Scalar = Builder.CreateNoWrapBinOp(
1353 Opc: BO->getOpcode(), LHS: ScalarOps[0], RHS: ScalarOps[1], IsNUW: OBO->hasNoUnsignedWrap(),
1354 IsNSW: OBO->hasNoSignedWrap(), Name: BO->getName() + ".scalar");
1355 } else if (PossiblyDisjointInst *PDI = dyn_cast<PossiblyDisjointInst>(Val: &I)) {
1356 Scalar = Builder.CreateOr(LHS: ScalarOps[0], RHS: ScalarOps[1],
1357 Name: BO->getName() + ".scalar", IsDisjoint: PDI->isDisjoint());
1358 } else if (PossiblyExactOperator *PEO =
1359 dyn_cast<PossiblyExactOperator>(Val: &I)) {
1360 Scalar =
1361 Builder.CreateExactBinOp(Opc: BO->getOpcode(), LHS: ScalarOps[0], RHS: ScalarOps[1],
1362 IsExact: PEO->isExact(), Name: BO->getName() + ".scalar");
1363 } else if (FPMathOperator *FPMO = dyn_cast<FPMathOperator>(Val: &I)) {
1364 Scalar = Builder.CreateBinOpFMF(Opc: BO->getOpcode(), LHS: ScalarOps[0],
1365 RHS: ScalarOps[1], FMFSource: FPMO->getFastMathFlags(),
1366 Name: BO->getName() + ".scalar");
1367 } else {
1368 Scalar = Builder.CreateBinOp(Opc: BO->getOpcode(), LHS: ScalarOps[0], RHS: ScalarOps[1],
1369 Name: BO->getName() + ".scalar");
1370 }
1371 } else {
1372 FastMathFlags FMF;
1373 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &I))
1374 FMF = FPMO->getFastMathFlags();
1375 Scalar = Builder.CreateIntrinsic(RetTy: ScalarTy, ID: II->getIntrinsicID(), Args: ScalarOps,
1376 FMFSource: FMF, Name: II->getName() + ".scalar");
1377 }
1378
1379 Value *Insert = Builder.CreateInsertElement(Vec: NewVecC, NewElt: Scalar, Idx: *Index);
1380 replaceValue(Old&: I, New&: *Insert);
1381 return true;
1382}
1383
1384/// Try to combine a scalar binop + 2 scalar compares of extracted elements of
1385/// a vector into vector operations followed by extract. Note: The SLP pass
1386/// may miss this pattern because of implementation problems.
1387bool VectorCombine::foldExtractedCmps(Instruction &I) {
1388 auto *BI = dyn_cast<BinaryOperator>(Val: &I);
1389
1390 // We are looking for a scalar binop of booleans.
1391 // binop i1 (cmp Pred I0, C0), (cmp Pred I1, C1)
1392 if (!BI || !I.getType()->isIntegerTy(BitWidth: 1))
1393 return false;
1394
1395 // The compare predicates should match, and each compare should have a
1396 // constant operand.
1397 Value *B0 = I.getOperand(i: 0), *B1 = I.getOperand(i: 1);
1398 Instruction *I0, *I1;
1399 Constant *C0, *C1;
1400 CmpPredicate P0, P1;
1401 if (!match(V: B0, P: m_Cmp(Pred&: P0, L: m_Instruction(I&: I0), R: m_Constant(C&: C0))) ||
1402 !match(V: B1, P: m_Cmp(Pred&: P1, L: m_Instruction(I&: I1), R: m_Constant(C&: C1))))
1403 return false;
1404
1405 auto MatchingPred = CmpPredicate::getMatching(A: P0, B: P1);
1406 if (!MatchingPred)
1407 return false;
1408
1409 // The compare operands must be extracts of the same vector with constant
1410 // extract indexes.
1411 Value *X;
1412 uint64_t Index0, Index1;
1413 if (!match(V: I0, P: m_ExtractElt(Val: m_Value(V&: X), Idx: m_ConstantInt(V&: Index0))) ||
1414 !match(V: I1, P: m_ExtractElt(Val: m_Specific(V: X), Idx: m_ConstantInt(V&: Index1))))
1415 return false;
1416
1417 auto *Ext0 = cast<ExtractElementInst>(Val: I0);
1418 auto *Ext1 = cast<ExtractElementInst>(Val: I1);
1419 ExtractElementInst *ConvertToShuf = getShuffleExtract(Ext0, Ext1, PreferredExtractIndex: CostKind);
1420 if (!ConvertToShuf)
1421 return false;
1422 assert((ConvertToShuf == Ext0 || ConvertToShuf == Ext1) &&
1423 "Unknown ExtractElementInst");
1424
1425 // The original scalar pattern is:
1426 // binop i1 (cmp Pred (ext X, Index0), C0), (cmp Pred (ext X, Index1), C1)
1427 CmpInst::Predicate Pred = *MatchingPred;
1428 unsigned CmpOpcode =
1429 CmpInst::isFPPredicate(P: Pred) ? Instruction::FCmp : Instruction::ICmp;
1430 auto *VecTy = dyn_cast<FixedVectorType>(Val: X->getType());
1431 if (!VecTy)
1432 return false;
1433
1434 if (Index0 >= VecTy->getNumElements() || Index1 >= VecTy->getNumElements())
1435 return false;
1436
1437 InstructionCost Ext0Cost =
1438 TTI.getVectorInstrCost(I: *Ext0, Val: VecTy, CostKind, Index: Index0);
1439 InstructionCost Ext1Cost =
1440 TTI.getVectorInstrCost(I: *Ext1, Val: VecTy, CostKind, Index: Index1);
1441 InstructionCost CmpCost = TTI.getCmpSelInstrCost(
1442 Opcode: CmpOpcode, ValTy: I0->getType(), CondTy: CmpInst::makeCmpResultType(opnd_type: I0->getType()), VecPred: Pred,
1443 CostKind);
1444
1445 InstructionCost OldCost =
1446 Ext0Cost + Ext1Cost + CmpCost * 2 +
1447 TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: I.getType(), CostKind);
1448
1449 // The proposed vector pattern is:
1450 // vcmp = cmp Pred X, VecC
1451 // ext (binop vNi1 vcmp, (shuffle vcmp, Index1)), Index0
1452 int CheapIndex = ConvertToShuf == Ext0 ? Index1 : Index0;
1453 int ExpensiveIndex = ConvertToShuf == Ext0 ? Index0 : Index1;
1454 auto *CmpTy = cast<FixedVectorType>(Val: CmpInst::makeCmpResultType(opnd_type: VecTy));
1455 InstructionCost NewCost = TTI.getCmpSelInstrCost(
1456 Opcode: CmpOpcode, ValTy: VecTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VecTy), VecPred: Pred, CostKind);
1457 SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
1458 ShufMask[CheapIndex] = ExpensiveIndex;
1459 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc, DstTy: CmpTy,
1460 SrcTy: CmpTy, CostKind, Mask: ShufMask);
1461 NewCost += TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: CmpTy, CostKind);
1462 NewCost += TTI.getVectorInstrCost(I: *Ext0, Val: CmpTy, CostKind, Index: CheapIndex);
1463 NewCost += Ext0->hasOneUse() ? 0 : Ext0Cost;
1464 NewCost += Ext1->hasOneUse() ? 0 : Ext1Cost;
1465
1466 // Aggressively form vector ops if the cost is equal because the transform
1467 // may enable further optimization.
1468 // Codegen can reverse this transform (scalarize) if it was not profitable.
1469 if (OldCost < NewCost || !NewCost.isValid())
1470 return false;
1471
1472 // Create a vector constant from the 2 scalar constants.
1473 SmallVector<Constant *, 32> CmpC(VecTy->getNumElements(),
1474 PoisonValue::get(T: VecTy->getElementType()));
1475 CmpC[Index0] = C0;
1476 CmpC[Index1] = C1;
1477 Value *VCmp = Builder.CreateCmp(Pred, LHS: X, RHS: ConstantVector::get(V: CmpC));
1478 Value *Shuf = createShiftShuffle(Vec: VCmp, OldIndex: ExpensiveIndex, NewIndex: CheapIndex, Builder);
1479 Value *LHS = ConvertToShuf == Ext0 ? Shuf : VCmp;
1480 Value *RHS = ConvertToShuf == Ext0 ? VCmp : Shuf;
1481 Value *VecLogic = Builder.CreateBinOp(Opc: BI->getOpcode(), LHS, RHS);
1482 Value *NewExt = Builder.CreateExtractElement(Vec: VecLogic, Idx: CheapIndex);
1483 replaceValue(Old&: I, New&: *NewExt);
1484 ++NumVecCmpBO;
1485 return true;
1486}
1487
1488/// Try to fold scalar selects that select between extracted elements and zero
1489/// into extracting from a vector select. This is rooted at the bitcast.
1490///
1491/// This pattern arises when a vector is bitcast to a smaller element type,
1492/// elements are extracted, and then conditionally selected with zero:
1493///
1494/// %bc = bitcast <4 x i32> %src to <16 x i8>
1495/// %e0 = extractelement <16 x i8> %bc, i32 0
1496/// %s0 = select i1 %cond, i8 %e0, i8 0
1497/// %e1 = extractelement <16 x i8> %bc, i32 1
1498/// %s1 = select i1 %cond, i8 %e1, i8 0
1499/// ...
1500///
1501/// Transforms to:
1502/// %sel = select i1 %cond, <4 x i32> %src, <4 x i32> zeroinitializer
1503/// %bc = bitcast <4 x i32> %sel to <16 x i8>
1504/// %e0 = extractelement <16 x i8> %bc, i32 0
1505/// %e1 = extractelement <16 x i8> %bc, i32 1
1506/// ...
1507///
1508/// This is profitable because vector select on wider types produces fewer
1509/// select/cndmask instructions than scalar selects on each element.
1510bool VectorCombine::foldSelectsFromBitcast(Instruction &I) {
1511 auto *BC = dyn_cast<BitCastInst>(Val: &I);
1512 if (!BC)
1513 return false;
1514
1515 FixedVectorType *SrcVecTy = dyn_cast<FixedVectorType>(Val: BC->getSrcTy());
1516 FixedVectorType *DstVecTy = dyn_cast<FixedVectorType>(Val: BC->getDestTy());
1517 if (!SrcVecTy || !DstVecTy)
1518 return false;
1519
1520 // Source must be 32-bit or 64-bit elements, destination must be smaller
1521 // integer elements. Zero in all these types is all-bits-zero.
1522 Type *SrcEltTy = SrcVecTy->getElementType();
1523 Type *DstEltTy = DstVecTy->getElementType();
1524 unsigned SrcEltBits = SrcEltTy->getPrimitiveSizeInBits();
1525 unsigned DstEltBits = DstEltTy->getPrimitiveSizeInBits();
1526
1527 if (SrcEltBits != 32 && SrcEltBits != 64)
1528 return false;
1529
1530 if (!DstEltTy->isIntegerTy() || DstEltBits >= SrcEltBits)
1531 return false;
1532
1533 // Check profitability using TTI before collecting users.
1534 Type *CondTy = CmpInst::makeCmpResultType(opnd_type: DstEltTy);
1535 Type *VecCondTy = CmpInst::makeCmpResultType(opnd_type: SrcVecTy);
1536
1537 InstructionCost ScalarSelCost =
1538 TTI.getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: DstEltTy, CondTy,
1539 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1540 InstructionCost VecSelCost =
1541 TTI.getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: SrcVecTy, CondTy: VecCondTy,
1542 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1543
1544 // We need at least this many selects for vectorization to be profitable.
1545 // VecSelCost < ScalarSelCost * NumSelects => NumSelects > VecSelCost /
1546 // ScalarSelCost
1547 if (!ScalarSelCost.isValid() || ScalarSelCost == 0)
1548 return false;
1549
1550 unsigned MinSelects = (VecSelCost.getValue() / ScalarSelCost.getValue()) + 1;
1551
1552 // Quick check: if bitcast doesn't have enough users, bail early.
1553 if (!BC->hasNUsesOrMore(N: MinSelects))
1554 return false;
1555
1556 // Collect all select users that match the pattern, grouped by condition.
1557 // Pattern: select i1 %cond, (extractelement %bc, idx), 0
1558 DenseMap<Value *, SmallVector<SelectInst *, 8>> CondToSelects;
1559
1560 for (User *U : BC->users()) {
1561 auto *Ext = dyn_cast<ExtractElementInst>(Val: U);
1562 if (!Ext)
1563 continue;
1564
1565 for (User *ExtUser : Ext->users()) {
1566 Value *Cond;
1567 // Match: select i1 %cond, %ext, 0
1568 if (match(V: ExtUser, P: m_Select(C: m_Value(V&: Cond), L: m_Specific(V: Ext), R: m_Zero())) &&
1569 Cond->getType()->isIntegerTy(BitWidth: 1))
1570 CondToSelects[Cond].push_back(Elt: cast<SelectInst>(Val: ExtUser));
1571 }
1572 }
1573
1574 if (CondToSelects.empty())
1575 return false;
1576
1577 bool MadeChange = false;
1578 Value *SrcVec = BC->getOperand(i_nocapture: 0);
1579
1580 // Process each group of selects with the same condition.
1581 for (auto [Cond, Selects] : CondToSelects) {
1582 // Only profitable if vector select cost < total scalar select cost.
1583 if (Selects.size() < MinSelects) {
1584 LLVM_DEBUG(dbgs() << "VectorCombine: foldSelectsFromBitcast not "
1585 << "profitable (VecCost=" << VecSelCost
1586 << ", ScalarCost=" << ScalarSelCost
1587 << ", NumSelects=" << Selects.size() << ")\n");
1588 continue;
1589 }
1590
1591 // Create the vector select and bitcast once for this condition.
1592 auto InsertPt = std::next(x: BC->getIterator());
1593
1594 if (auto *CondInst = dyn_cast<Instruction>(Val: Cond))
1595 if (DT.dominates(Def: BC, User: CondInst))
1596 InsertPt = std::next(x: CondInst->getIterator());
1597
1598 Builder.SetInsertPoint(InsertPt);
1599 Value *VecSel =
1600 Builder.CreateSelect(C: Cond, True: SrcVec, False: Constant::getNullValue(Ty: SrcVecTy));
1601 Value *NewBC = Builder.CreateBitCast(V: VecSel, DestTy: DstVecTy);
1602
1603 // Replace each scalar select with an extract from the new bitcast.
1604 for (SelectInst *Sel : Selects) {
1605 auto *Ext = cast<ExtractElementInst>(Val: Sel->getTrueValue());
1606 Value *Idx = Ext->getIndexOperand();
1607
1608 Builder.SetInsertPoint(Sel);
1609 Value *NewExt = Builder.CreateExtractElement(Vec: NewBC, Idx);
1610 replaceValue(Old&: *Sel, New&: *NewExt);
1611 MadeChange = true;
1612 }
1613
1614 LLVM_DEBUG(dbgs() << "VectorCombine: folded " << Selects.size()
1615 << " selects into vector select\n");
1616 }
1617
1618 return MadeChange;
1619}
1620
1621static void analyzeCostOfVecReduction(const IntrinsicInst &II,
1622 TTI::TargetCostKind CostKind,
1623 const TargetTransformInfo &TTI,
1624 InstructionCost &CostBeforeReduction,
1625 InstructionCost &CostAfterReduction) {
1626 Instruction *Op0, *Op1;
1627 auto *RedOp = dyn_cast<Instruction>(Val: II.getOperand(i_nocapture: 0));
1628 auto *VecRedTy = cast<VectorType>(Val: II.getOperand(i_nocapture: 0)->getType());
1629 unsigned ReductionOpc =
1630 getArithmeticReductionInstruction(RdxID: II.getIntrinsicID());
1631 if (RedOp && match(V: RedOp, P: m_ZExtOrSExt(Op: m_Value()))) {
1632 bool IsUnsigned = isa<ZExtInst>(Val: RedOp);
1633 auto *ExtType = cast<VectorType>(Val: RedOp->getOperand(i: 0)->getType());
1634
1635 CostBeforeReduction =
1636 TTI.getCastInstrCost(Opcode: RedOp->getOpcode(), Dst: VecRedTy, Src: ExtType,
1637 CCH: TTI::CastContextHint::None, CostKind, I: RedOp);
1638 CostAfterReduction =
1639 TTI.getExtendedReductionCost(Opcode: ReductionOpc, IsUnsigned, ResTy: II.getType(),
1640 Ty: ExtType, FMF: FastMathFlags(), CostKind);
1641 return;
1642 }
1643 if (RedOp && II.getIntrinsicID() == Intrinsic::vector_reduce_add &&
1644 match(V: RedOp,
1645 P: m_ZExtOrSExt(Op: m_Mul(L: m_Instruction(I&: Op0), R: m_Instruction(I&: Op1)))) &&
1646 match(V: Op0, P: m_ZExtOrSExt(Op: m_Value())) &&
1647 Op0->getOpcode() == Op1->getOpcode() &&
1648 Op0->getOperand(i: 0)->getType() == Op1->getOperand(i: 0)->getType() &&
1649 (Op0->getOpcode() == RedOp->getOpcode() || Op0 == Op1)) {
1650 // Matched reduce.add(ext(mul(ext(A), ext(B)))
1651 bool IsUnsigned = isa<ZExtInst>(Val: Op0);
1652 auto *ExtType = cast<VectorType>(Val: Op0->getOperand(i: 0)->getType());
1653 VectorType *MulType = VectorType::get(ElementType: Op0->getType(), Other: VecRedTy);
1654
1655 InstructionCost ExtCost =
1656 TTI.getCastInstrCost(Opcode: Op0->getOpcode(), Dst: MulType, Src: ExtType,
1657 CCH: TTI::CastContextHint::None, CostKind, I: Op0);
1658 InstructionCost MulCost =
1659 TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: MulType, CostKind);
1660 InstructionCost Ext2Cost =
1661 TTI.getCastInstrCost(Opcode: RedOp->getOpcode(), Dst: VecRedTy, Src: MulType,
1662 CCH: TTI::CastContextHint::None, CostKind, I: RedOp);
1663
1664 CostBeforeReduction = ExtCost * 2 + MulCost + Ext2Cost;
1665 CostAfterReduction = TTI.getMulAccReductionCost(
1666 IsUnsigned, RedOpcode: ReductionOpc, ResTy: II.getType(), Ty: ExtType, CostKind);
1667 return;
1668 }
1669 CostAfterReduction = TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: VecRedTy,
1670 FMF: std::nullopt, CostKind);
1671}
1672
1673bool VectorCombine::foldBinopOfReductions(Instruction &I) {
1674 Instruction::BinaryOps BinOpOpc = cast<BinaryOperator>(Val: &I)->getOpcode();
1675 Intrinsic::ID ReductionIID = getReductionForBinop(Opc: BinOpOpc);
1676 if (BinOpOpc == Instruction::Sub)
1677 ReductionIID = Intrinsic::vector_reduce_add;
1678 if (ReductionIID == Intrinsic::not_intrinsic)
1679 return false;
1680 // FP reductions have a start-value operand that this fold doesn't handle.
1681 if (ReductionIID == Intrinsic::vector_reduce_fadd ||
1682 ReductionIID == Intrinsic::vector_reduce_fmul)
1683 return false;
1684
1685 auto checkIntrinsicAndGetItsArgument = [](Value *V,
1686 Intrinsic::ID IID) -> Value * {
1687 auto *II = dyn_cast<IntrinsicInst>(Val: V);
1688 if (!II)
1689 return nullptr;
1690 if (II->getIntrinsicID() == IID && II->hasOneUse())
1691 return II->getArgOperand(i: 0);
1692 return nullptr;
1693 };
1694
1695 Value *V0 = checkIntrinsicAndGetItsArgument(I.getOperand(i: 0), ReductionIID);
1696 if (!V0)
1697 return false;
1698 Value *V1 = checkIntrinsicAndGetItsArgument(I.getOperand(i: 1), ReductionIID);
1699 if (!V1)
1700 return false;
1701
1702 auto *VTy = cast<VectorType>(Val: V0->getType());
1703 if (V1->getType() != VTy)
1704 return false;
1705 const auto &II0 = *cast<IntrinsicInst>(Val: I.getOperand(i: 0));
1706 const auto &II1 = *cast<IntrinsicInst>(Val: I.getOperand(i: 1));
1707 unsigned ReductionOpc =
1708 getArithmeticReductionInstruction(RdxID: II0.getIntrinsicID());
1709
1710 InstructionCost OldCost = 0;
1711 InstructionCost NewCost = 0;
1712 InstructionCost CostOfRedOperand0 = 0;
1713 InstructionCost CostOfRed0 = 0;
1714 InstructionCost CostOfRedOperand1 = 0;
1715 InstructionCost CostOfRed1 = 0;
1716 analyzeCostOfVecReduction(II: II0, CostKind, TTI, CostBeforeReduction&: CostOfRedOperand0, CostAfterReduction&: CostOfRed0);
1717 analyzeCostOfVecReduction(II: II1, CostKind, TTI, CostBeforeReduction&: CostOfRedOperand1, CostAfterReduction&: CostOfRed1);
1718 OldCost = CostOfRed0 + CostOfRed1 + TTI.getInstructionCost(U: &I, CostKind);
1719 NewCost =
1720 CostOfRedOperand0 + CostOfRedOperand1 +
1721 TTI.getArithmeticInstrCost(Opcode: BinOpOpc, Ty: VTy, CostKind) +
1722 TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: VTy, FMF: std::nullopt, CostKind);
1723 if (NewCost >= OldCost || !NewCost.isValid())
1724 return false;
1725
1726 LLVM_DEBUG(dbgs() << "Found two mergeable reductions: " << I
1727 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
1728 << "\n");
1729 Value *VectorBO;
1730 if (BinOpOpc == Instruction::Or)
1731 VectorBO = Builder.CreateOr(LHS: V0, RHS: V1, Name: "",
1732 IsDisjoint: cast<PossiblyDisjointInst>(Val&: I).isDisjoint());
1733 else
1734 VectorBO = Builder.CreateBinOp(Opc: BinOpOpc, LHS: V0, RHS: V1);
1735
1736 Value *Rdx = Builder.CreateIntrinsic(ID: ReductionIID, OverloadTypes: {VTy}, Args: {VectorBO});
1737 replaceValue(Old&: I, New&: *Rdx);
1738 return true;
1739}
1740
1741// Check if memory is modified, freed, or synchronized between two instrs in
1742// the same BB.
1743static bool isMemModifiedBetween(const VectorizeOptions &Opts,
1744 BasicBlock::iterator Begin,
1745 BasicBlock::iterator End,
1746 const MemoryLocation &Loc, AAResults &AA) {
1747 unsigned NumScanned = 0;
1748 if (std::any_of(first: Begin, last: End, pred: [&](const Instruction &Instr) {
1749 return isModSet(MRI: AA.getModRefInfo(I: &Instr, OptLoc: Loc)) ||
1750 ++NumScanned > Opts.vector_combine_max_scan_instrs;
1751 }))
1752 return true;
1753
1754 // willNotFreeBetween expects instructions rather than iterators. An empty
1755 // range cannot free or synchronize, so avoid dereferencing its end.
1756 return Begin != End && !willNotFreeBetween(Assume: &*Begin, CtxI: &*End);
1757}
1758
1759namespace {
1760/// Helper class to indicate whether a vector index can be safely scalarized and
1761/// if a freeze needs to be inserted.
1762class ScalarizationResult {
1763 enum class StatusTy { Unsafe, Safe, SafeWithFreeze };
1764
1765 StatusTy Status;
1766 Value *ToFreeze;
1767
1768 ScalarizationResult(StatusTy Status, Value *ToFreeze = nullptr)
1769 : Status(Status), ToFreeze(ToFreeze) {}
1770
1771public:
1772 ScalarizationResult(const ScalarizationResult &Other) = default;
1773 ~ScalarizationResult() {
1774 assert(!ToFreeze && "freeze() not called with ToFreeze being set");
1775 }
1776
1777 static ScalarizationResult unsafe() { return {StatusTy::Unsafe}; }
1778 static ScalarizationResult safe() { return {StatusTy::Safe}; }
1779 static ScalarizationResult safeWithFreeze(Value *ToFreeze) {
1780 return {StatusTy::SafeWithFreeze, ToFreeze};
1781 }
1782
1783 /// Returns true if the index can be scalarize without requiring a freeze.
1784 bool isSafe() const { return Status == StatusTy::Safe; }
1785 /// Returns true if the index cannot be scalarized.
1786 bool isUnsafe() const { return Status == StatusTy::Unsafe; }
1787 /// Returns true if the index can be scalarize, but requires inserting a
1788 /// freeze.
1789 bool isSafeWithFreeze() const { return Status == StatusTy::SafeWithFreeze; }
1790
1791 /// Reset the state of Unsafe and clear ToFreze if set.
1792 void discard() {
1793 ToFreeze = nullptr;
1794 Status = StatusTy::Unsafe;
1795 }
1796
1797 /// Freeze the ToFreeze and update the use in \p User to use it.
1798 void freeze(IRBuilderBase &Builder, Instruction &UserI) {
1799 assert(isSafeWithFreeze() &&
1800 "should only be used when freezing is required");
1801 assert(is_contained(ToFreeze->users(), &UserI) &&
1802 "UserI must be a user of ToFreeze");
1803 IRBuilder<>::InsertPointGuard Guard(Builder);
1804 Builder.SetInsertPoint(cast<Instruction>(Val: &UserI));
1805 Value *Frozen =
1806 Builder.CreateFreeze(V: ToFreeze, Name: ToFreeze->getName() + ".frozen");
1807 for (Use &U : make_early_inc_range(Range: (UserI.operands())))
1808 if (U.get() == ToFreeze)
1809 U.set(Frozen);
1810
1811 ToFreeze = nullptr;
1812 }
1813};
1814} // namespace
1815
1816/// Check if it is legal to scalarize a memory access to \p VecTy at index \p
1817/// Idx. \p Idx must access a valid vector element.
1818static ScalarizationResult canScalarizeAccess(VectorType *VecTy, Value *Idx,
1819 const SimplifyQuery &SQ) {
1820 // We do checks for both fixed vector types and scalable vector types.
1821 // This is the number of elements of fixed vector types,
1822 // or the minimum number of elements of scalable vector types.
1823 uint64_t NumElements = VecTy->getElementCount().getKnownMinValue();
1824 unsigned IntWidth = Idx->getType()->getScalarSizeInBits();
1825
1826 if (auto *C = dyn_cast<ConstantInt>(Val: Idx)) {
1827 if (C->getValue().ult(RHS: NumElements))
1828 return ScalarizationResult::safe();
1829 return ScalarizationResult::unsafe();
1830 }
1831
1832 // Always unsafe if the index type can't handle all inbound values.
1833 if (!llvm::isUIntN(N: IntWidth, x: NumElements))
1834 return ScalarizationResult::unsafe();
1835
1836 APInt Zero(IntWidth, 0);
1837 APInt MaxElts(IntWidth, NumElements);
1838 ConstantRange ValidIndices(Zero, MaxElts);
1839 ConstantRange IdxRange(IntWidth, true);
1840
1841 if (isGuaranteedNotToBePoison(V: Idx, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)) {
1842 if (ValidIndices.contains(
1843 CR: computeConstantRange(V: Idx, /*ForSigned=*/false, SQ)))
1844 return ScalarizationResult::safe();
1845 return ScalarizationResult::unsafe();
1846 }
1847
1848 // If the index may be poison, check if we can insert a freeze before the
1849 // range of the index is restricted.
1850 Value *IdxBase;
1851 ConstantInt *CI;
1852 if (match(V: Idx, P: m_And(L: m_Value(V&: IdxBase), R: m_ConstantInt(CI)))) {
1853 IdxRange = IdxRange.binaryAnd(Other: CI->getValue());
1854 } else if (match(V: Idx, P: m_URem(L: m_Value(V&: IdxBase), R: m_ConstantInt(CI)))) {
1855 IdxRange = IdxRange.urem(Other: CI->getValue());
1856 }
1857
1858 if (ValidIndices.contains(CR: IdxRange))
1859 return ScalarizationResult::safeWithFreeze(ToFreeze: IdxBase);
1860 return ScalarizationResult::unsafe();
1861}
1862
1863/// Return the GEP index type if the unsigned vector index \p Idx can be
1864/// represented by an inbounds GEP. A null result means that the maximum byte
1865/// offset cannot be represented by the pointer's signed GEP index type.
1866///
1867/// unsigned lane range
1868/// |
1869/// v
1870/// MaxByteOffset = MaxLane * element store size
1871/// |
1872/// +-- unavailable or outside signed GEP range --> reject
1873/// |
1874/// v
1875/// valid range --> use the pointer's GEP index type
1876static IntegerType *getScalarizedGEPIndexInfo(VectorType *VecTy, Value *Idx,
1877 Type *PtrTy,
1878 const DataLayout &DL) {
1879 auto *GEPIndexTy = cast<IntegerType>(Val: DL.getIndexType(PtrTy));
1880 unsigned GEPBits = GEPIndexTy->getBitWidth();
1881 uint64_t NumElements = VecTy->getElementCount().getKnownMinValue();
1882
1883 uint64_t MaxLane = NumElements - 1;
1884 if (auto *C = dyn_cast<ConstantInt>(Val: Idx)) {
1885 if (C->getValue().uge(RHS: NumElements))
1886 return nullptr;
1887 MaxLane = C->getZExtValue();
1888 }
1889
1890 Type *ElemTy = VecTy->getElementType();
1891 if (!DL.typeSizeEqualsStoreSize(Ty: ElemTy))
1892 return nullptr;
1893
1894 TypeSize ElemStride = DL.getTypeStoreSize(Ty: ElemTy);
1895 if (ElemStride.isScalable())
1896 return nullptr;
1897
1898 // Compare both values in a common width:
1899 //
1900 // MaxLane (uint64_t) * ElemStride (uint64_t) signed_max(GEPBits)
1901 // | |
1902 // v v
1903 // ByteOffset (up to 128 bits) sext to WideBits
1904 // \ /
1905 // +------------ ugt ------------+
1906 // |
1907 // greater -> reject
1908 //
1909 // WideBits = max(GEPBits, 128) prevents the multiplication from wrapping
1910 // and preserves the GEP limit during the comparison.
1911 unsigned WideBits = std::max(a: GEPBits, b: 128u);
1912 APInt MaxLaneValue(WideBits, MaxLane);
1913 APInt ByteOffset = MaxLaneValue;
1914 ByteOffset *= APInt(WideBits, ElemStride.getFixedValue());
1915 APInt MaxGEPOffset = APInt::getSignedMaxValue(numBits: GEPBits).sext(width: WideBits);
1916 // Reject offsets outside the GEP's positive signed range. Compare as
1917 // unsigned because the full 128-bit product may set its sign bit.
1918 if (ByteOffset.ugt(RHS: MaxGEPOffset))
1919 return nullptr;
1920
1921 return GEPIndexTy;
1922}
1923
1924/// Materialize an index for a scalarized GEP after profitability is known.
1925/// Vector element indices are unsigned, but GEP sign-extends narrow integer
1926/// indices. Widen a narrow index explicitly so its unsigned value is retained.
1927static Value *materializeScalarizedGEPIndex(Value *Idx, IntegerType *GEPIndexTy,
1928 IRBuilderBase &Builder) {
1929 unsigned SrcBits = Idx->getType()->getIntegerBitWidth();
1930 unsigned DstBits = GEPIndexTy->getBitWidth();
1931 if (SrcBits >= DstBits)
1932 return Idx;
1933
1934 return Builder.CreateZExt(V: Idx, DestTy: GEPIndexTy, Name: Idx->getName() + ".gepidx");
1935}
1936
1937/// The memory operation on a vector of \p ScalarType had alignment of
1938/// \p VectorAlignment. Compute the maximal, but conservatively correct,
1939/// alignment that will be valid for the memory operation on a single scalar
1940/// element of the same type with index \p Idx.
1941static Align computeAlignmentAfterScalarization(Align VectorAlignment,
1942 Type *ScalarType, Value *Idx,
1943 const DataLayout &DL) {
1944 if (auto *C = dyn_cast<ConstantInt>(Val: Idx))
1945 return commonAlignment(A: VectorAlignment,
1946 Offset: C->getZExtValue() * DL.getTypeStoreSize(Ty: ScalarType));
1947 return commonAlignment(A: VectorAlignment, Offset: DL.getTypeStoreSize(Ty: ScalarType));
1948}
1949
1950/// Fold a vector store fed by a single-use insertelement chain into scalar
1951/// stores.
1952///
1953/// Before:
1954///
1955/// %p --> vector load --> insert %x, lane 1 --> insert %y, lane 3
1956/// |
1957/// v
1958/// vector store to %p
1959///
1960/// Vector lanes: [ 0 ] [ 1 ] [ 2 ] [ 3 ]
1961/// Stored value: [ old | x | old | y ] (one vector store)
1962///
1963/// After:
1964///
1965/// +--> GEP(%p, lane 1) --> store %x
1966/// %p -------------+
1967/// +--> GEP(%p, lane 3) --> store %y
1968///
1969/// Vector lanes: [ 0 ] [ 1 ] [ 2 ] [ 3 ]
1970/// Scalar stores: x y
1971/// store@1 store@3
1972///
1973/// Step 1. Gate:
1974/// target supports vector-element GEP addressing
1975///
1976/// Step 2. Trace:
1977/// vector store <-- insertelement <-- ... <-- insertelement <-- load
1978///
1979/// Steps 3-5. Validate:
1980/// reject unprofitable full overwrites; require simple accesses, a
1981/// common address/block, no memory write in between, and scalarizable
1982/// indices.
1983bool VectorCombine::foldInsertElementsToStores(Instruction &I) {
1984 // Step 1: The target must support addressing a vector element with a GEP.
1985 if (!TTI.allowVectorElementIndexingUsingGEP())
1986 return false;
1987
1988 auto *SI = cast<StoreInst>(Val: &I);
1989 if (!SI->isSimple() || !isa<VectorType>(Val: SI->getValueOperand()->getType()))
1990 return false;
1991
1992 // Step 2: Collect a single-use insertelement chain, starting at the vector
1993 // store and walking back to the candidate load.
1994 Value *Source = SI->getValueOperand();
1995 SmallVector<std::pair<Value *, Value *>, 4> InsertElements;
1996 Value *Base = Source;
1997 while (auto *Insert = dyn_cast<InsertElementInst>(Val: Base)) {
1998 if (!Insert->hasOneUse())
1999 break;
2000 Value *InsertVal = Insert->getOperand(i_nocapture: 1);
2001 Value *Idx = Insert->getOperand(i_nocapture: 2);
2002 InsertElements.push_back(Elt: {InsertVal, Idx});
2003 Base = Insert->getOperand(i_nocapture: 0);
2004 }
2005
2006 if (InsertElements.empty())
2007 return false;
2008
2009 // The backwards walk collected the inserts in reverse program order. Restore
2010 // it now so later scalar stores preserve writes to duplicate/equal indices.
2011 std::reverse(first: InsertElements.begin(), last: InsertElements.end());
2012 auto *Load = dyn_cast<LoadInst>(Val: Base);
2013 if (!Load)
2014 return false;
2015 auto *VecTy = cast<VectorType>(Val: SI->getValueOperand()->getType());
2016
2017 // Step 3: Avoid replacing a complete overwrite with scalar stores when every
2018 // lane receives the same value; keeping the vector operation is preferable.
2019 if (auto *FVT = dyn_cast<FixedVectorType>(Val: VecTy)) {
2020 if (InsertElements.size() == FVT->getNumElements()) {
2021 Value *FirstVal = InsertElements.front().first;
2022 if (all_of(Range&: InsertElements,
2023 P: [FirstVal](const auto &Elt) { return Elt.first == FirstVal; }))
2024 return false;
2025 }
2026 }
2027 Value *SrcAddr = Load->getPointerOperand()->stripPointerCasts();
2028 // Step 4: Establish the load/store update is legal: both accesses are simple,
2029 // have the same base address and block, have scalar elements whose type size
2030 // equals their store size, and no intervening operation modifies the updated
2031 // memory.
2032 if (!Load->isSimple() || Load->getParent() != SI->getParent() ||
2033 !DL->typeSizeEqualsStoreSize(Ty: Load->getType()->getScalarType()) ||
2034 SrcAddr != SI->getPointerOperand()->stripPointerCasts())
2035 return false;
2036
2037 if (isMemModifiedBetween(Opts, Begin: Load->getIterator(), End: SI->getIterator(),
2038 Loc: MemoryLocation::get(SI), AA))
2039 return false;
2040
2041 // Step 5: Validate every index before changing IR. A safe-with-freeze result
2042 // is recorded by ScalarizationResult, so discard it until profitability is
2043 // known; otherwise a rejected candidate could leave a freeze behind.
2044 for (auto [InsertVal, Idx] : InsertElements) {
2045 auto ScalarizableIdx =
2046 canScalarizeAccess(VecTy, Idx, SQ: SQ.getWithInstruction(I: &I));
2047 if (ScalarizableIdx.isUnsafe())
2048 return false;
2049
2050 auto GEPIndex =
2051 getScalarizedGEPIndexInfo(VecTy, Idx, PtrTy: SI->getPointerOperandType(), DL: *DL);
2052 if (!GEPIndex) {
2053 ScalarizableIdx.discard();
2054 return false;
2055 }
2056
2057 // We are only checking legality here. Do not mutate IR before the
2058 // profitability check, but also do not leave a pending ToFreeze behind.
2059 ScalarizableIdx.discard();
2060 }
2061
2062 InstructionCost OldCost = TTI.getMemoryOpCost(
2063 Opcode: Instruction::Store, Src: SI->getValueOperand()->getType(), Alignment: SI->getAlign(),
2064 AddressSpace: SI->getPointerAddressSpace(), CostKind);
2065
2066 if (Load->hasOneUse())
2067 OldCost += TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: Load->getType(),
2068 Alignment: Load->getAlign(),
2069 AddressSpace: Load->getPointerAddressSpace(), CostKind);
2070
2071 for (auto [InsertVal, Idx] : InsertElements) {
2072 int Index = -1;
2073 if (auto *CIdx = dyn_cast<ConstantInt>(Val: Idx))
2074 Index = CIdx->getZExtValue();
2075
2076 OldCost += TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: VecTy,
2077 CostKind, Index);
2078 }
2079
2080 InstructionCost NewCost = 0;
2081 // This transform replaces insertelement operations on a single vector with
2082 // GEPs and scalar stores, so assume constant-index GEP offsets stay within
2083 // addressing-mode ranges that getGEPCost considers TCC_Free. Cost only GEPs
2084 // with dynamic indices.
2085 for (auto [InsertVal, Idx] : InsertElements) {
2086 if (isa<ConstantInt>(Val: Idx))
2087 continue;
2088 const Value *GEPIndices[] = {ConstantInt::get(Ty: Idx->getType(), V: 0), Idx};
2089 NewCost += TTI.getGEPCost(PointeeType: VecTy, Ptr: SI->getPointerOperand(), Operands: GEPIndices,
2090 CostKind, AccessType: InsertVal->getType());
2091 }
2092
2093 for (auto [InsertVal, Idx] : InsertElements) {
2094 Align ScalarOpAlignment = computeAlignmentAfterScalarization(
2095 VectorAlignment: std::max(a: SI->getAlign(), b: Load->getAlign()), ScalarType: InsertVal->getType(), Idx,
2096 DL: *DL);
2097
2098 NewCost += TTI.getMemoryOpCost(Opcode: Instruction::Store, Src: InsertVal->getType(),
2099 Alignment: ScalarOpAlignment,
2100 AddressSpace: SI->getPointerAddressSpace(), CostKind);
2101 }
2102
2103 LLVM_DEBUG(dbgs() << "Found an insert-elements vector store scalarization "
2104 "candidate: "
2105 << I << "\n"
2106 << " NumInserts: " << InsertElements.size() << "\n"
2107 << " OldCost: " << OldCost << " vs NewCost: " << NewCost
2108 << "\n");
2109
2110 if (OldCost <= NewCost)
2111 return false;
2112
2113 for (auto [InsertVal, Idx] : InsertElements) {
2114 auto ScalarizableIdx =
2115 canScalarizeAccess(VecTy, Idx, SQ: SQ.getWithInstruction(I: &I));
2116 assert(!ScalarizableIdx.isUnsafe() && "already checked above");
2117
2118 if (ScalarizableIdx.isSafeWithFreeze())
2119 ScalarizableIdx.freeze(Builder, UserI&: *cast<Instruction>(Val: Idx));
2120 }
2121
2122 Worklist.push(I: Load);
2123 StoreInst *LastStore = nullptr;
2124 for (auto [InsertVal, Idx] : InsertElements) {
2125 auto ScalarizableIdx =
2126 canScalarizeAccess(VecTy, Idx, SQ: SQ.getWithInstruction(I: &I));
2127 if (ScalarizableIdx.isUnsafe())
2128 return false;
2129
2130 IntegerType *GEPIndexTy =
2131 getScalarizedGEPIndexInfo(VecTy, Idx, PtrTy: SI->getPointerOperandType(), DL: *DL);
2132
2133 Value *GEPIdx = materializeScalarizedGEPIndex(Idx, GEPIndexTy, Builder);
2134 Value *GEP = Builder.CreateInBoundsGEP(
2135 Ty: SI->getValueOperand()->getType(), Ptr: SI->getPointerOperand(),
2136 IdxList: {ConstantInt::get(Ty: GEPIdx->getType(), V: 0), GEPIdx});
2137
2138 LastStore = Builder.CreateStore(Val: InsertVal, Ptr: GEP);
2139 LastStore->copyMetadata(SrcInst: *SI);
2140
2141 // The new GEP may change the pointer operand, so !invariant.group cannot
2142 // be transferred to the scalar store.
2143 LastStore->setMetadata(KindID: LLVMContext::MD_invariant_group, Node: nullptr);
2144 Align ScalarOpAlignment = computeAlignmentAfterScalarization(
2145 VectorAlignment: std::max(a: SI->getAlign(), b: Load->getAlign()), ScalarType: InsertVal->getType(), Idx,
2146 DL: *DL);
2147 LastStore->setAlignment(ScalarOpAlignment);
2148 }
2149
2150 replaceValue(Old&: I, New&: *LastStore);
2151 eraseInstruction(I);
2152 return true;
2153}
2154
2155/// Try to scalarize vector loads feeding extractelement or bitcast
2156/// instructions.
2157bool VectorCombine::scalarizeLoad(Instruction &I) {
2158 Value *Ptr;
2159 if (!match(V: &I, P: m_Load(Op: m_Value(V&: Ptr))))
2160 return false;
2161
2162 auto *LI = cast<LoadInst>(Val: &I);
2163 auto *VecTy = cast<VectorType>(Val: LI->getType());
2164
2165 // The isSimple() check could be isUnordered(), but for now we cowardly
2166 // refuse to handle even unordered atomics.
2167 if (!LI->isSimple() || !DL->typeSizeEqualsStoreSize(Ty: VecTy->getScalarType()))
2168 return false;
2169
2170 bool AllExtracts = true;
2171 bool AllBitcasts = true;
2172 Instruction *LastCheckedInst = LI;
2173 unsigned NumInstChecked = 0;
2174
2175 // Check what type of users we have (must either all be extracts or
2176 // bitcasts) and ensure no memory modifications between the load and
2177 // its users.
2178 for (User *U : LI->users()) {
2179 auto *UI = dyn_cast<Instruction>(Val: U);
2180 if (!UI || UI->getParent() != LI->getParent())
2181 return false;
2182
2183 // If any user is waiting to be erased, then bail out as this will
2184 // distort the cost calculation and possibly lead to infinite loops.
2185 if (UI->use_empty())
2186 return false;
2187
2188 if (!isa<ExtractElementInst>(Val: UI))
2189 AllExtracts = false;
2190 if (!isa<BitCastInst>(Val: UI))
2191 AllBitcasts = false;
2192
2193 // Check if any instruction between the load and the user may modify memory.
2194 if (LastCheckedInst->comesBefore(Other: UI)) {
2195 for (Instruction &I :
2196 make_range(x: std::next(x: LI->getIterator()), y: UI->getIterator())) {
2197 // Bail out if we reached the check limit or the instruction may write
2198 // to memory.
2199 if (NumInstChecked == Opts.vector_combine_max_scan_instrs ||
2200 I.mayWriteToMemory())
2201 return false;
2202 NumInstChecked++;
2203 }
2204 LastCheckedInst = UI;
2205 }
2206 }
2207
2208 if (AllExtracts)
2209 return scalarizeLoadExtract(LI, VecTy, Ptr);
2210 if (AllBitcasts)
2211 return scalarizeLoadBitcast(LI, VecTy, Ptr);
2212 return false;
2213}
2214
2215/// Try to scalarize vector loads feeding extractelement instructions.
2216bool VectorCombine::scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy,
2217 Value *Ptr) {
2218 if (!TTI.allowVectorElementIndexingUsingGEP())
2219 return false;
2220
2221 DenseMap<ExtractElementInst *, ScalarizationResult> NeedFreeze;
2222 DenseMap<ExtractElementInst *, IntegerType *> GEPIndexInfos;
2223 llvm::scope_exit FailureGuard([&]() {
2224 // If the transform is aborted, discard the ScalarizationResults.
2225 for (auto &Pair : NeedFreeze)
2226 Pair.second.discard();
2227 });
2228
2229 InstructionCost OriginalCost =
2230 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: VecTy, Alignment: LI->getAlign(),
2231 AddressSpace: LI->getPointerAddressSpace(), CostKind);
2232 InstructionCost ScalarizedCost = 0;
2233
2234 for (User *U : LI->users()) {
2235 auto *UI = cast<ExtractElementInst>(Val: U);
2236
2237 auto ScalarIdx = canScalarizeAccess(VecTy, Idx: UI->getIndexOperand(),
2238 SQ: SQ.getWithInstruction(I: LI));
2239 if (ScalarIdx.isUnsafe())
2240 return false;
2241
2242 IntegerType *GEPIndex = getScalarizedGEPIndexInfo(
2243 VecTy, Idx: UI->getIndexOperand(), PtrTy: LI->getPointerOperandType(), DL: *DL);
2244 if (!GEPIndex) {
2245 ScalarIdx.discard();
2246 return false;
2247 }
2248
2249 GEPIndexInfos.try_emplace(Key: UI, Args&: GEPIndex);
2250
2251 if (ScalarIdx.isSafeWithFreeze()) {
2252 NeedFreeze.try_emplace(Key: UI, Args&: ScalarIdx);
2253 ScalarIdx.discard();
2254 }
2255
2256 auto *Index = dyn_cast<ConstantInt>(Val: UI->getIndexOperand());
2257 OriginalCost +=
2258 TTI.getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: VecTy, CostKind,
2259 Index: Index ? Index->getZExtValue() : -1);
2260 ScalarizedCost +=
2261 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: VecTy->getElementType(),
2262 Alignment: Align(1), AddressSpace: LI->getPointerAddressSpace(), CostKind);
2263 ScalarizedCost += TTI.getAddressComputationCost(PtrTy: LI->getPointerOperandType(),
2264 SE: nullptr, Ptr: nullptr, CostKind);
2265 if (!Index && UI->getIndexOperand()->getType()->getIntegerBitWidth() <
2266 GEPIndex->getBitWidth())
2267 ScalarizedCost += TTI.getCastInstrCost(
2268 Opcode: Instruction::ZExt, Dst: GEPIndex, Src: UI->getIndexOperand()->getType(),
2269 CCH: TTI::CastContextHint::None, CostKind);
2270 }
2271
2272 LLVM_DEBUG(dbgs() << "Found all extractions of a vector load: " << *LI
2273 << "\n LoadExtractCost: " << OriginalCost
2274 << " vs ScalarizedCost: " << ScalarizedCost << "\n");
2275
2276 if (ScalarizedCost > OriginalCost)
2277 return false;
2278 if (ScalarizedCost == OriginalCost && !LI->hasOneUse())
2279 return false;
2280
2281 // Ensure we add the load back to the worklist BEFORE its users so they can
2282 // erased in the correct order.
2283 Worklist.push(I: LI);
2284
2285 Type *ElemType = VecTy->getElementType();
2286
2287 // Replace extracts with narrow scalar loads.
2288 for (User *U : LI->users()) {
2289 auto *EI = cast<ExtractElementInst>(Val: U);
2290 Value *Idx = EI->getIndexOperand();
2291
2292 // Insert 'freeze' for poison indexes.
2293 if (auto It = NeedFreeze.find(Val: EI); It != NeedFreeze.end())
2294 It->second.freeze(Builder, UserI&: *cast<Instruction>(Val: Idx));
2295
2296 Builder.SetInsertPoint(EI);
2297 auto It = GEPIndexInfos.find(Val: EI);
2298 assert(It != GEPIndexInfos.end() &&
2299 "Missing scalarized GEP index information");
2300 Value *GEPIdx = materializeScalarizedGEPIndex(Idx, GEPIndexTy: It->second, Builder);
2301 Value *GEP = Builder.CreateInBoundsGEP(
2302 Ty: VecTy, Ptr, IdxList: {ConstantInt::get(Ty: GEPIdx->getType(), V: 0), GEPIdx});
2303 auto *NewLoad = cast<LoadInst>(
2304 Val: Builder.CreateLoad(Ty: ElemType, Ptr: GEP, Name: EI->getName() + ".scalar"));
2305
2306 Align ScalarOpAlignment =
2307 computeAlignmentAfterScalarization(VectorAlignment: LI->getAlign(), ScalarType: ElemType, Idx, DL: *DL);
2308 NewLoad->setAlignment(ScalarOpAlignment);
2309
2310 if (auto *ConstIdx = dyn_cast<ConstantInt>(Val: Idx)) {
2311 size_t Offset = ConstIdx->getZExtValue() * DL->getTypeStoreSize(Ty: ElemType);
2312 AAMDNodes OldAAMD = LI->getAAMetadata();
2313 NewLoad->setAAMetadata(OldAAMD.adjustForAccess(Offset, AccessTy: ElemType, DL: *DL));
2314 }
2315
2316 replaceValue(Old&: *EI, New&: *NewLoad, Erase: false);
2317 }
2318
2319 FailureGuard.release();
2320 return true;
2321}
2322
2323/// Try to scalarize vector loads feeding bitcast instructions.
2324bool VectorCombine::scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy,
2325 Value *Ptr) {
2326 InstructionCost OriginalCost =
2327 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: VecTy, Alignment: LI->getAlign(),
2328 AddressSpace: LI->getPointerAddressSpace(), CostKind);
2329
2330 if (!isa<FixedVectorType>(Val: VecTy))
2331 return false;
2332
2333 Type *TargetScalarType = nullptr;
2334 unsigned VecBitWidth = DL->getTypeSizeInBits(Ty: VecTy);
2335
2336 for (User *U : LI->users()) {
2337 auto *BC = cast<BitCastInst>(Val: U);
2338
2339 Type *DestTy = BC->getDestTy();
2340 if (!DestTy->isIntegerTy() && !DestTy->isFloatingPointTy())
2341 return false;
2342
2343 unsigned DestBitWidth = DL->getTypeSizeInBits(Ty: DestTy);
2344 if (DestBitWidth != VecBitWidth)
2345 return false;
2346
2347 // All bitcasts must target the same scalar type.
2348 if (!TargetScalarType)
2349 TargetScalarType = DestTy;
2350 else if (TargetScalarType != DestTy)
2351 return false;
2352
2353 OriginalCost +=
2354 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: TargetScalarType, Src: VecTy,
2355 CCH: TTI.getCastContextHint(I: BC), CostKind, I: BC);
2356 }
2357
2358 if (!TargetScalarType)
2359 return false;
2360
2361 assert(!LI->user_empty() && "Unexpected load without bitcast users");
2362 InstructionCost ScalarizedCost =
2363 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: TargetScalarType, Alignment: LI->getAlign(),
2364 AddressSpace: LI->getPointerAddressSpace(), CostKind);
2365
2366 LLVM_DEBUG(dbgs() << "Found vector load feeding only bitcasts: " << *LI
2367 << "\n OriginalCost: " << OriginalCost
2368 << " vs ScalarizedCost: " << ScalarizedCost << "\n");
2369
2370 if (ScalarizedCost >= OriginalCost)
2371 return false;
2372
2373 // Ensure we add the load back to the worklist BEFORE its users so they can
2374 // erased in the correct order.
2375 Worklist.push(I: LI);
2376
2377 Builder.SetInsertPoint(LI);
2378 auto *ScalarLoad =
2379 Builder.CreateLoad(Ty: TargetScalarType, Ptr, Name: LI->getName() + ".scalar");
2380 ScalarLoad->setAlignment(LI->getAlign());
2381 ScalarLoad->copyMetadata(SrcInst: *LI);
2382
2383 // Replace all bitcast users with the scalar load.
2384 for (User *U : LI->users()) {
2385 auto *BC = cast<BitCastInst>(Val: U);
2386 replaceValue(Old&: *BC, New&: *ScalarLoad, Erase: false);
2387 }
2388
2389 return true;
2390}
2391
2392bool VectorCombine::scalarizeExtExtract(Instruction &I) {
2393 if (!TTI.allowVectorElementIndexingUsingGEP())
2394 return false;
2395 auto *Ext = dyn_cast<ZExtInst>(Val: &I);
2396 if (!Ext)
2397 return false;
2398
2399 // Try to convert a vector zext feeding only extracts to a set of scalar
2400 // (Src << ExtIdx *Size) & (Size -1)
2401 // if profitable .
2402 auto *SrcTy = dyn_cast<FixedVectorType>(Val: Ext->getOperand(i_nocapture: 0)->getType());
2403 if (!SrcTy)
2404 return false;
2405 auto *DstTy = cast<FixedVectorType>(Val: Ext->getType());
2406
2407 Type *ScalarDstTy = DstTy->getElementType();
2408 if (DL->getTypeSizeInBits(Ty: SrcTy) != DL->getTypeSizeInBits(Ty: ScalarDstTy))
2409 return false;
2410
2411 InstructionCost VectorCost =
2412 TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: DstTy, Src: SrcTy,
2413 CCH: TTI::CastContextHint::None, CostKind, I: Ext);
2414 unsigned ExtCnt = 0;
2415 bool ExtLane0 = false;
2416 for (User *U : Ext->users()) {
2417 uint64_t Idx;
2418 if (!match(V: U, P: m_ExtractElt(Val: m_Value(), Idx: m_ConstantInt(V&: Idx))))
2419 return false;
2420 // An out-of-bounds extractelement produces poison; bail out rather
2421 // than computing a shift amount that overflows the packed type.
2422 if (Idx >= SrcTy->getNumElements())
2423 return false;
2424 if (cast<Instruction>(Val: U)->use_empty())
2425 continue;
2426 ExtCnt += 1;
2427 ExtLane0 |= !Idx;
2428 VectorCost += TTI.getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: DstTy,
2429 CostKind, Index: Idx, Op0: U);
2430 }
2431
2432 InstructionCost ScalarCost =
2433 ExtCnt * TTI.getArithmeticInstrCost(
2434 Opcode: Instruction::And, Ty: ScalarDstTy, CostKind,
2435 Opd1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
2436 Opd2Info: {.Kind: TTI::OK_NonUniformConstantValue, .Properties: TTI::OP_None}) +
2437 (ExtCnt - ExtLane0) *
2438 TTI.getArithmeticInstrCost(
2439 Opcode: Instruction::LShr, Ty: ScalarDstTy, CostKind,
2440 Opd1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
2441 Opd2Info: {.Kind: TTI::OK_NonUniformConstantValue, .Properties: TTI::OP_None});
2442 if (ScalarCost > VectorCost)
2443 return false;
2444
2445 Value *ScalarV = Ext->getOperand(i_nocapture: 0);
2446 if (!isGuaranteedNotToBePoison(V: ScalarV, AC: SQ.AC, CtxI: dyn_cast<Instruction>(Val: ScalarV),
2447 DT: SQ.DT)) {
2448 // Check wether all lanes are extracted, all extracts trigger UB
2449 // on poison, and the last extract (and hence all previous ones)
2450 // are guaranteed to execute if Ext executes. If so, we do not
2451 // need to insert a freeze.
2452 SmallDenseSet<ConstantInt *, 8> ExtractedLanes;
2453 bool AllExtractsTriggerUB = true;
2454 ExtractElementInst *LastExtract = nullptr;
2455 BasicBlock *ExtBB = Ext->getParent();
2456 for (User *U : Ext->users()) {
2457 auto *Extract = cast<ExtractElementInst>(Val: U);
2458 if (Extract->getParent() != ExtBB || !programUndefinedIfPoison(Inst: Extract)) {
2459 AllExtractsTriggerUB = false;
2460 break;
2461 }
2462 ExtractedLanes.insert(V: cast<ConstantInt>(Val: Extract->getIndexOperand()));
2463 if (!LastExtract || LastExtract->comesBefore(Other: Extract))
2464 LastExtract = Extract;
2465 }
2466 if (ExtractedLanes.size() != DstTy->getNumElements() ||
2467 !AllExtractsTriggerUB ||
2468 !isGuaranteedToTransferExecutionToSuccessor(Begin: Ext->getIterator(),
2469 End: LastExtract->getIterator()))
2470 ScalarV = Builder.CreateFreeze(V: ScalarV);
2471 }
2472 ScalarV = Builder.CreateBitCast(
2473 V: ScalarV,
2474 DestTy: IntegerType::get(C&: SrcTy->getContext(), NumBits: DL->getTypeSizeInBits(Ty: SrcTy)));
2475 uint64_t SrcEltSizeInBits = DL->getTypeSizeInBits(Ty: SrcTy->getElementType());
2476 uint64_t TotalBits = DL->getTypeSizeInBits(Ty: SrcTy);
2477 APInt EltBitMask = APInt::getLowBitsSet(numBits: TotalBits, loBitsSet: SrcEltSizeInBits);
2478 Type *PackedTy = IntegerType::get(C&: SrcTy->getContext(), NumBits: TotalBits);
2479 Value *Mask = ConstantInt::get(Ty: PackedTy, V: EltBitMask);
2480 for (User *U : Ext->users()) {
2481 auto *Extract = cast<ExtractElementInst>(Val: U);
2482 uint64_t Idx =
2483 cast<ConstantInt>(Val: Extract->getIndexOperand())->getZExtValue();
2484 uint64_t ShiftAmt =
2485 DL->isBigEndian()
2486 ? (TotalBits - SrcEltSizeInBits - Idx * SrcEltSizeInBits)
2487 : (Idx * SrcEltSizeInBits);
2488 Value *LShr = Builder.CreateLShr(LHS: ScalarV, RHS: ShiftAmt);
2489 Value *And = Builder.CreateAnd(LHS: LShr, RHS: Mask);
2490 U->replaceAllUsesWith(V: And);
2491 }
2492 return true;
2493}
2494
2495/// Try to fold "(or (zext (bitcast X)), (shl (zext (bitcast Y)), C))"
2496/// to "(bitcast (concat X, Y))"
2497/// where X/Y are bitcasted from i1 mask vectors.
2498bool VectorCombine::foldConcatOfBoolMasks(Instruction &I) {
2499 Type *Ty = I.getType();
2500 if (!Ty->isIntegerTy())
2501 return false;
2502
2503 // TODO: Add big endian test coverage
2504 if (DL->isBigEndian())
2505 return false;
2506
2507 // Restrict to disjoint cases so the mask vectors aren't overlapping.
2508 Instruction *X, *Y;
2509 if (!match(V: &I, P: m_DisjointOr(L: m_Instruction(I&: X), R: m_Instruction(I&: Y))))
2510 return false;
2511
2512 // Allow both sources to contain shl, to handle more generic pattern:
2513 // "(or (shl (zext (bitcast X)), C1), (shl (zext (bitcast Y)), C2))"
2514 Value *SrcX;
2515 uint64_t ShAmtX = 0;
2516 if (!match(V: X, P: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcX)))))) &&
2517 !match(V: X, P: m_OneUse(
2518 SubPattern: m_Shl(L: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcX))))),
2519 R: m_ConstantInt(V&: ShAmtX)))))
2520 return false;
2521
2522 Value *SrcY;
2523 uint64_t ShAmtY = 0;
2524 if (!match(V: Y, P: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcY)))))) &&
2525 !match(V: Y, P: m_OneUse(
2526 SubPattern: m_Shl(L: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcY))))),
2527 R: m_ConstantInt(V&: ShAmtY)))))
2528 return false;
2529
2530 // Canonicalize larger shift to the RHS.
2531 if (ShAmtX > ShAmtY) {
2532 std::swap(a&: X, b&: Y);
2533 std::swap(a&: SrcX, b&: SrcY);
2534 std::swap(a&: ShAmtX, b&: ShAmtY);
2535 }
2536
2537 // Ensure both sources are matching vXi1 bool mask types, and that the shift
2538 // difference is the mask width so they can be easily concatenated together.
2539 uint64_t ShAmtDiff = ShAmtY - ShAmtX;
2540 unsigned NumSHL = (ShAmtX > 0) + (ShAmtY > 0);
2541 unsigned BitWidth = Ty->getPrimitiveSizeInBits();
2542 auto *MaskTy = dyn_cast<FixedVectorType>(Val: SrcX->getType());
2543 if (!MaskTy || SrcX->getType() != SrcY->getType() ||
2544 !MaskTy->getElementType()->isIntegerTy(BitWidth: 1) ||
2545 MaskTy->getNumElements() != ShAmtDiff ||
2546 MaskTy->getNumElements() > (BitWidth / 2))
2547 return false;
2548
2549 auto *ConcatTy = FixedVectorType::getDoubleElementsVectorType(VTy: MaskTy);
2550 auto *ConcatIntTy =
2551 Type::getIntNTy(C&: Ty->getContext(), N: ConcatTy->getNumElements());
2552 auto *MaskIntTy = Type::getIntNTy(C&: Ty->getContext(), N: ShAmtDiff);
2553
2554 SmallVector<int, 32> ConcatMask(ConcatTy->getNumElements());
2555 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
2556
2557 // TODO: Is it worth supporting multi use cases?
2558 InstructionCost OldCost = 0;
2559 OldCost += TTI.getArithmeticInstrCost(Opcode: Instruction::Or, Ty, CostKind);
2560 OldCost +=
2561 NumSHL * TTI.getArithmeticInstrCost(Opcode: Instruction::Shl, Ty, CostKind);
2562 OldCost += 2 * TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: Ty, Src: MaskIntTy,
2563 CCH: TTI::CastContextHint::None, CostKind);
2564 OldCost += 2 * TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: MaskIntTy, Src: MaskTy,
2565 CCH: TTI::CastContextHint::None, CostKind);
2566
2567 InstructionCost NewCost = 0;
2568 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ConcatTy,
2569 SrcTy: MaskTy, CostKind, Mask: ConcatMask);
2570 NewCost += TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: ConcatIntTy, Src: ConcatTy,
2571 CCH: TTI::CastContextHint::None, CostKind);
2572 if (Ty != ConcatIntTy)
2573 NewCost += TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: Ty, Src: ConcatIntTy,
2574 CCH: TTI::CastContextHint::None, CostKind);
2575 if (ShAmtX > 0)
2576 NewCost += TTI.getArithmeticInstrCost(Opcode: Instruction::Shl, Ty, CostKind);
2577
2578 LLVM_DEBUG(dbgs() << "Found a concatenation of bitcasted bool masks: " << I
2579 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2580 << "\n");
2581
2582 if (NewCost > OldCost)
2583 return false;
2584
2585 // Build bool mask concatenation, bitcast back to scalar integer, and perform
2586 // any residual zero-extension or shifting.
2587 Value *Concat = Builder.CreateShuffleVector(V1: SrcX, V2: SrcY, Mask: ConcatMask);
2588
2589 Value *Result = Builder.CreateBitCast(V: Concat, DestTy: ConcatIntTy);
2590
2591 if (Ty != ConcatIntTy) {
2592 Result = Builder.CreateZExt(V: Result, DestTy: Ty);
2593 }
2594
2595 if (ShAmtX > 0) {
2596 Result = Builder.CreateShl(LHS: Result, RHS: ShAmtX);
2597 }
2598
2599 replaceValue(Old&: I, New&: *Result);
2600 return true;
2601}
2602
2603/// Try to convert "shuffle (binop (shuffle, shuffle)), undef"
2604/// --> "binop (shuffle), (shuffle)".
2605bool VectorCombine::foldPermuteOfBinops(Instruction &I) {
2606 BinaryOperator *BinOp;
2607 ArrayRef<int> OuterMask;
2608 if (!match(V: &I, P: m_Shuffle(v1: m_BinOp(I&: BinOp), v2: m_Undef(), mask: m_Mask(OuterMask))))
2609 return false;
2610
2611 // Don't introduce poison into div/rem.
2612 if (BinOp->isIntDivRem() && llvm::is_contained(Range&: OuterMask, Element: PoisonMaskElem))
2613 return false;
2614
2615 Value *Op00, *Op01, *Op10, *Op11;
2616 ArrayRef<int> Mask0, Mask1;
2617 bool Match0 = match(V: BinOp->getOperand(i_nocapture: 0),
2618 P: m_Shuffle(v1: m_Value(V&: Op00), v2: m_Value(V&: Op01), mask: m_Mask(Mask0)));
2619 bool Match1 = match(V: BinOp->getOperand(i_nocapture: 1),
2620 P: m_Shuffle(v1: m_Value(V&: Op10), v2: m_Value(V&: Op11), mask: m_Mask(Mask1)));
2621 if (!Match0 && !Match1)
2622 return false;
2623
2624 Op00 = Match0 ? Op00 : BinOp->getOperand(i_nocapture: 0);
2625 Op01 = Match0 ? Op01 : BinOp->getOperand(i_nocapture: 0);
2626 Op10 = Match1 ? Op10 : BinOp->getOperand(i_nocapture: 1);
2627 Op11 = Match1 ? Op11 : BinOp->getOperand(i_nocapture: 1);
2628
2629 Instruction::BinaryOps Opcode = BinOp->getOpcode();
2630 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
2631 auto *BinOpTy = dyn_cast<FixedVectorType>(Val: BinOp->getType());
2632 auto *Op0Ty = dyn_cast<FixedVectorType>(Val: Op00->getType());
2633 auto *Op1Ty = dyn_cast<FixedVectorType>(Val: Op10->getType());
2634 if (!ShuffleDstTy || !BinOpTy || !Op0Ty || !Op1Ty)
2635 return false;
2636
2637 unsigned NumSrcElts = BinOpTy->getNumElements();
2638
2639 // Don't accept shuffles that reference the second operand in
2640 // div/rem or if its an undef arg.
2641 if ((BinOp->isIntDivRem() || !isa<PoisonValue>(Val: I.getOperand(i: 1))) &&
2642 any_of(Range&: OuterMask, P: [NumSrcElts](int M) { return M >= (int)NumSrcElts; }))
2643 return false;
2644
2645 // Merge outer / inner (or identity if no match) shuffles.
2646 SmallVector<int> NewMask0, NewMask1;
2647 for (int M : OuterMask) {
2648 if (M < 0 || M >= (int)NumSrcElts) {
2649 NewMask0.push_back(Elt: PoisonMaskElem);
2650 NewMask1.push_back(Elt: PoisonMaskElem);
2651 } else {
2652 NewMask0.push_back(Elt: Match0 ? Mask0[M] : M);
2653 NewMask1.push_back(Elt: Match1 ? Mask1[M] : M);
2654 }
2655 }
2656
2657 unsigned NumOpElts = Op0Ty->getNumElements();
2658 bool IsIdentity0 = ShuffleDstTy == Op0Ty &&
2659 all_of(Range&: NewMask0, P: [NumOpElts](int M) { return M < (int)NumOpElts; }) &&
2660 ShuffleVectorInst::isIdentityMask(Mask: NewMask0, NumSrcElts: NumOpElts);
2661 bool IsIdentity1 = ShuffleDstTy == Op1Ty &&
2662 all_of(Range&: NewMask1, P: [NumOpElts](int M) { return M < (int)NumOpElts; }) &&
2663 ShuffleVectorInst::isIdentityMask(Mask: NewMask1, NumSrcElts: NumOpElts);
2664
2665 InstructionCost NewCost = 0;
2666 // Try to merge shuffles across the binop if the new shuffles are not costly.
2667 InstructionCost BinOpCost =
2668 TTI.getArithmeticInstrCost(Opcode, Ty: BinOpTy, CostKind);
2669 InstructionCost OldCost =
2670 BinOpCost + TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
2671 DstTy: ShuffleDstTy, SrcTy: BinOpTy, CostKind, Mask: OuterMask,
2672 Index: 0, SubTp: nullptr, Args: {BinOp}, CtxI: &I);
2673 if (!BinOp->hasOneUse())
2674 NewCost += BinOpCost;
2675
2676 if (Match0) {
2677 InstructionCost Shuf0Cost = TTI.getShuffleCost(
2678 Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: BinOpTy, SrcTy: Op0Ty, CostKind, Mask: Mask0,
2679 Index: 0, SubTp: nullptr, Args: {Op00, Op01}, CtxI: cast<Instruction>(Val: BinOp->getOperand(i_nocapture: 0)));
2680 OldCost += Shuf0Cost;
2681 if (!BinOp->hasOneUse() || !BinOp->getOperand(i_nocapture: 0)->hasOneUse())
2682 NewCost += Shuf0Cost;
2683 }
2684 if (Match1) {
2685 InstructionCost Shuf1Cost = TTI.getShuffleCost(
2686 Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: BinOpTy, SrcTy: Op1Ty, CostKind, Mask: Mask1,
2687 Index: 0, SubTp: nullptr, Args: {Op10, Op11}, CtxI: cast<Instruction>(Val: BinOp->getOperand(i_nocapture: 1)));
2688 OldCost += Shuf1Cost;
2689 if (!BinOp->hasOneUse() || !BinOp->getOperand(i_nocapture: 1)->hasOneUse())
2690 NewCost += Shuf1Cost;
2691 }
2692
2693 NewCost += TTI.getArithmeticInstrCost(Opcode, Ty: ShuffleDstTy, CostKind);
2694
2695 if (!IsIdentity0)
2696 NewCost +=
2697 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ShuffleDstTy,
2698 SrcTy: Op0Ty, CostKind, Mask: NewMask0, Index: 0, SubTp: nullptr, Args: {Op00, Op01});
2699 if (!IsIdentity1)
2700 NewCost +=
2701 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ShuffleDstTy,
2702 SrcTy: Op1Ty, CostKind, Mask: NewMask1, Index: 0, SubTp: nullptr, Args: {Op10, Op11});
2703
2704 LLVM_DEBUG(dbgs() << "Found a shuffle feeding a shuffled binop: " << I
2705 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2706 << "\n");
2707
2708 // If costs are equal, still fold as we reduce instruction count.
2709 if (NewCost > OldCost)
2710 return false;
2711
2712 Value *LHS =
2713 IsIdentity0 ? Op00 : Builder.CreateShuffleVector(V1: Op00, V2: Op01, Mask: NewMask0);
2714 Value *RHS =
2715 IsIdentity1 ? Op10 : Builder.CreateShuffleVector(V1: Op10, V2: Op11, Mask: NewMask1);
2716 Value *NewBO = Builder.CreateBinOp(Opc: Opcode, LHS, RHS);
2717
2718 // Intersect flags from the old binops.
2719 if (auto *NewInst = dyn_cast<Instruction>(Val: NewBO))
2720 NewInst->copyIRFlags(V: BinOp);
2721
2722 replaceValue(Old&: I, New&: *NewBO);
2723 return true;
2724}
2725
2726/// Try to convert "shuffle (binop), (binop)" into "binop (shuffle), (shuffle)".
2727/// Try to convert "shuffle (cmpop), (cmpop)" into "cmpop (shuffle), (shuffle)".
2728bool VectorCombine::foldShuffleOfBinops(Instruction &I) {
2729 ArrayRef<int> OldMask;
2730 Instruction *LHS, *RHS;
2731 if (!match(V: &I, P: m_Shuffle(v1: m_Instruction(I&: LHS), v2: m_Instruction(I&: RHS),
2732 mask: m_Mask(OldMask))))
2733 return false;
2734
2735 // TODO: Add support for addlike etc.
2736 if (LHS->getOpcode() != RHS->getOpcode())
2737 return false;
2738
2739 Value *X, *Y, *Z, *W;
2740 bool IsCommutative = false;
2741 CmpPredicate PredLHS = CmpInst::BAD_ICMP_PREDICATE;
2742 CmpPredicate PredRHS = CmpInst::BAD_ICMP_PREDICATE;
2743 if (match(V: LHS, P: m_BinOp(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
2744 match(V: RHS, P: m_BinOp(L: m_Value(V&: Z), R: m_Value(V&: W)))) {
2745 auto *BO = cast<BinaryOperator>(Val: LHS);
2746 // Don't introduce poison into div/rem.
2747 if (llvm::is_contained(Range&: OldMask, Element: PoisonMaskElem) && BO->isIntDivRem())
2748 return false;
2749 IsCommutative = BinaryOperator::isCommutative(Opcode: BO->getOpcode());
2750 } else if (match(V: LHS, P: m_Cmp(Pred&: PredLHS, L: m_Value(V&: X), R: m_Value(V&: Y))) &&
2751 match(V: RHS, P: m_Cmp(Pred&: PredRHS, L: m_Value(V&: Z), R: m_Value(V&: W))) &&
2752 (CmpInst::Predicate)PredLHS == (CmpInst::Predicate)PredRHS) {
2753 IsCommutative = cast<CmpInst>(Val: LHS)->isCommutative();
2754 } else
2755 return false;
2756
2757 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
2758 auto *BinResTy = dyn_cast<FixedVectorType>(Val: LHS->getType());
2759 auto *BinOpTy = dyn_cast<FixedVectorType>(Val: X->getType());
2760 if (!ShuffleDstTy || !BinResTy || !BinOpTy || X->getType() != Z->getType())
2761 return false;
2762
2763 bool SameBinOp = LHS == RHS;
2764 unsigned NumSrcElts = BinOpTy->getNumElements();
2765
2766 // If we have something like "add X, Y" and "add Z, X", swap ops to match.
2767 if (IsCommutative && X != Z && Y != W && (X == W || Y == Z))
2768 std::swap(a&: X, b&: Y);
2769
2770 auto ConvertToUnary = [NumSrcElts](int &M) {
2771 if (M >= (int)NumSrcElts)
2772 M -= NumSrcElts;
2773 };
2774
2775 SmallVector<int> NewMask0(OldMask);
2776 TargetTransformInfo::ShuffleKind SK0 = TargetTransformInfo::SK_PermuteTwoSrc;
2777 TTI::OperandValueInfo Op0Info = TTI.commonOperandInfo(X, Y: Z);
2778 if (X == Z) {
2779 llvm::for_each(Range&: NewMask0, F: ConvertToUnary);
2780 SK0 = TargetTransformInfo::SK_PermuteSingleSrc;
2781 Z = PoisonValue::get(T: BinOpTy);
2782 }
2783
2784 SmallVector<int> NewMask1(OldMask);
2785 TargetTransformInfo::ShuffleKind SK1 = TargetTransformInfo::SK_PermuteTwoSrc;
2786 TTI::OperandValueInfo Op1Info = TTI.commonOperandInfo(X: Y, Y: W);
2787 if (Y == W) {
2788 llvm::for_each(Range&: NewMask1, F: ConvertToUnary);
2789 SK1 = TargetTransformInfo::SK_PermuteSingleSrc;
2790 W = PoisonValue::get(T: BinOpTy);
2791 }
2792
2793 // Try to replace a binop with a shuffle if the shuffle is not costly.
2794 // When SameBinOp, only count the binop cost once.
2795 InstructionCost LHSCost = TTI.getInstructionCost(U: LHS, CostKind);
2796 InstructionCost RHSCost = TTI.getInstructionCost(U: RHS, CostKind);
2797
2798 InstructionCost OldCost = LHSCost;
2799 if (!SameBinOp) {
2800 OldCost += RHSCost;
2801 }
2802 OldCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc,
2803 DstTy: ShuffleDstTy, SrcTy: BinResTy, CostKind, Mask: OldMask, Index: 0,
2804 SubTp: nullptr, Args: {LHS, RHS}, CtxI: &I);
2805
2806 // Handle shuffle(binop(shuffle(x),y),binop(z,shuffle(w))) style patterns
2807 // where one use shuffles have gotten split across the binop/cmp. These
2808 // often allow a major reduction in total cost that wouldn't happen as
2809 // individual folds.
2810 auto MergeInner = [&](Value *&Op, int Offset, MutableArrayRef<int> Mask,
2811 TTI::TargetCostKind CostKind,
2812 Instruction *BinOp) -> bool {
2813 Value *InnerOp;
2814 ArrayRef<int> InnerMask;
2815 if (match(V: Op, P: m_OneUse(SubPattern: m_Shuffle(v1: m_Value(V&: InnerOp), v2: m_Undef(),
2816 mask: m_Mask(InnerMask)))) &&
2817 InnerOp->getType() == Op->getType() &&
2818 all_of(Range&: InnerMask,
2819 P: [NumSrcElts](int M) { return M < (int)NumSrcElts; })) {
2820 for (int &M : Mask)
2821 if (Offset <= M && M < (int)(Offset + NumSrcElts)) {
2822 M = InnerMask[M - Offset];
2823 M = 0 <= M ? M + Offset : M;
2824 }
2825 // Op is only removed if the binop using it is removed too.
2826 bool Removed = BinOp->hasOneUser();
2827 if (Removed)
2828 OldCost += TTI.getInstructionCost(U: cast<Instruction>(Val: Op), CostKind);
2829 Op = InnerOp;
2830 return Removed;
2831 }
2832 return false;
2833 };
2834 bool ReducedInstCount = false;
2835 ReducedInstCount |= MergeInner(X, 0, NewMask0, CostKind, LHS);
2836 ReducedInstCount |= MergeInner(Y, 0, NewMask1, CostKind, LHS);
2837 ReducedInstCount |= MergeInner(Z, NumSrcElts, NewMask0, CostKind, RHS);
2838 ReducedInstCount |= MergeInner(W, NumSrcElts, NewMask1, CostKind, RHS);
2839 bool SingleSrcBinOp = (X == Y) && (Z == W) && (NewMask0 == NewMask1);
2840 // SingleSrcBinOp only reduces instruction count if we also eliminate the
2841 // original binop(s). If binops have multiple uses, they won't be eliminated.
2842 ReducedInstCount |= SingleSrcBinOp && LHS->hasOneUser() && RHS->hasOneUser();
2843
2844 // For concat shuffles of i1 vectors where both binops are one-use, the
2845 // transform keeps the same instruction count but canonicalises to a single
2846 // wider binop, enabling downstream folds (e.g. NOT(XOR(concat(a,b),
2847 // concat(c,d))) -> XNOR(concat(a,b),concat(c,d)) on AVX-512 mask regs).
2848 // Restrict to BinaryOperator (not CmpInst) since narrow comparisons may
2849 // be cheaper than wide ones on some targets (e.g. AVX-512 vpcmpeq).
2850 ReducedInstCount |= cast<ShuffleVectorInst>(Val: &I)->isConcat() &&
2851 I.getType()->getScalarType()->isIntegerTy(BitWidth: 1) &&
2852 isa<BinaryOperator>(Val: LHS) && LHS->hasOneUser() &&
2853 RHS->hasOneUser();
2854
2855 auto *ShuffleCmpTy =
2856 FixedVectorType::get(ElementType: BinOpTy->getElementType(), FVTy: ShuffleDstTy);
2857 InstructionCost NewCost = TTI.getShuffleCost(
2858 Kind: SK0, DstTy: ShuffleCmpTy, SrcTy: BinOpTy, CostKind, Mask: NewMask0, Index: 0, SubTp: nullptr, Args: {X, Z});
2859 if (!SingleSrcBinOp)
2860 NewCost += TTI.getShuffleCost(Kind: SK1, DstTy: ShuffleCmpTy, SrcTy: BinOpTy, CostKind,
2861 Mask: NewMask1, Index: 0, SubTp: nullptr, Args: {Y, W});
2862
2863 if (PredLHS == CmpInst::BAD_ICMP_PREDICATE) {
2864 NewCost += TTI.getArithmeticInstrCost(Opcode: LHS->getOpcode(), Ty: ShuffleDstTy,
2865 CostKind, Opd1Info: Op0Info, Opd2Info: Op1Info);
2866 } else {
2867 NewCost +=
2868 TTI.getCmpSelInstrCost(Opcode: LHS->getOpcode(), ValTy: ShuffleCmpTy, CondTy: ShuffleDstTy,
2869 VecPred: PredLHS, CostKind, Op1Info: Op0Info, Op2Info: Op1Info);
2870 }
2871 // If LHS/RHS have other uses, we need to account for the cost of keeping
2872 // the original instructions. When SameBinOp, only add the cost once.
2873 if (!LHS->hasOneUser())
2874 NewCost += LHSCost;
2875 if (!SameBinOp && !RHS->hasOneUser())
2876 NewCost += RHSCost;
2877
2878 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two binops: " << I
2879 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2880 << "\n");
2881
2882 // If either shuffle will constant fold away, then fold for the same cost as
2883 // we will reduce the instruction count.
2884 ReducedInstCount |= (isa<Constant>(Val: X) && isa<Constant>(Val: Z)) ||
2885 (isa<Constant>(Val: Y) && isa<Constant>(Val: W));
2886 if (ReducedInstCount ? (NewCost > OldCost) : (NewCost >= OldCost))
2887 return false;
2888
2889 Value *Shuf0 = Builder.CreateShuffleVector(V1: X, V2: Z, Mask: NewMask0);
2890 Value *Shuf1 =
2891 SingleSrcBinOp ? Shuf0 : Builder.CreateShuffleVector(V1: Y, V2: W, Mask: NewMask1);
2892 Value *NewBO = PredLHS == CmpInst::BAD_ICMP_PREDICATE
2893 ? Builder.CreateBinOp(
2894 Opc: cast<BinaryOperator>(Val: LHS)->getOpcode(), LHS: Shuf0, RHS: Shuf1)
2895 : Builder.CreateCmp(Pred: PredLHS, LHS: Shuf0, RHS: Shuf1);
2896
2897 // Intersect flags from the old binops.
2898 if (auto *NewInst = dyn_cast<Instruction>(Val: NewBO)) {
2899 NewInst->copyIRFlags(V: LHS);
2900 NewInst->andIRFlags(V: RHS);
2901 }
2902
2903 replaceValue(Old&: I, New&: *NewBO);
2904 return true;
2905}
2906
2907/// Try to convert,
2908/// (shuffle(select(c1,t1,f1)), (select(c2,t2,f2)), m) into
2909/// (select (shuffle c1,c2,m), (shuffle t1,t2,m), (shuffle f1,f2,m))
2910bool VectorCombine::foldShuffleOfSelects(Instruction &I) {
2911 ArrayRef<int> Mask;
2912 Value *C1, *T1, *F1, *C2, *T2, *F2;
2913 if (!match(V: &I, P: m_Shuffle(v1: m_Select(C: m_Value(V&: C1), L: m_Value(V&: T1), R: m_Value(V&: F1)),
2914 v2: m_Select(C: m_Value(V&: C2), L: m_Value(V&: T2), R: m_Value(V&: F2)),
2915 mask: m_Mask(Mask))))
2916 return false;
2917
2918 auto *Sel1 = cast<Instruction>(Val: I.getOperand(i: 0));
2919 auto *Sel2 = cast<Instruction>(Val: I.getOperand(i: 1));
2920
2921 auto *C1VecTy = dyn_cast<FixedVectorType>(Val: C1->getType());
2922 auto *C2VecTy = dyn_cast<FixedVectorType>(Val: C2->getType());
2923 if (!C1VecTy || !C2VecTy || C1VecTy != C2VecTy)
2924 return false;
2925
2926 auto *SI0FOp = dyn_cast<FPMathOperator>(Val: I.getOperand(i: 0));
2927 auto *SI1FOp = dyn_cast<FPMathOperator>(Val: I.getOperand(i: 1));
2928 // SelectInsts must have the same FMF.
2929 if (((SI0FOp == nullptr) != (SI1FOp == nullptr)) ||
2930 ((SI0FOp != nullptr) &&
2931 (SI0FOp->getFastMathFlags() != SI1FOp->getFastMathFlags())))
2932 return false;
2933
2934 auto *SrcVecTy = cast<FixedVectorType>(Val: T1->getType());
2935 auto *DstVecTy = cast<FixedVectorType>(Val: I.getType());
2936 auto SK = TargetTransformInfo::SK_PermuteTwoSrc;
2937 auto SelOp = Instruction::Select;
2938
2939 InstructionCost CostSel1 = TTI.getCmpSelInstrCost(
2940 Opcode: SelOp, ValTy: SrcVecTy, CondTy: C1VecTy, VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
2941 InstructionCost CostSel2 = TTI.getCmpSelInstrCost(
2942 Opcode: SelOp, ValTy: SrcVecTy, CondTy: C2VecTy, VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
2943
2944 InstructionCost OldCost =
2945 CostSel1 + CostSel2 +
2946 TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask, Index: 0, SubTp: nullptr,
2947 Args: {I.getOperand(i: 0), I.getOperand(i: 1)}, CtxI: &I);
2948
2949 InstructionCost NewCost = TTI.getShuffleCost(
2950 Kind: SK, DstTy: FixedVectorType::get(ElementType: C1VecTy->getScalarType(), NumElts: Mask.size()), SrcTy: C1VecTy,
2951 CostKind, Mask, Index: 0, SubTp: nullptr, Args: {C1, C2});
2952 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask, Index: 0,
2953 SubTp: nullptr, Args: {T1, T2});
2954 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask, Index: 0,
2955 SubTp: nullptr, Args: {F1, F2});
2956 auto *C1C2ShuffledVecTy = FixedVectorType::get(
2957 ElementType: Type::getInt1Ty(C&: I.getContext()), NumElts: DstVecTy->getNumElements());
2958 NewCost += TTI.getCmpSelInstrCost(Opcode: SelOp, ValTy: DstVecTy, CondTy: C1C2ShuffledVecTy,
2959 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
2960
2961 if (!Sel1->hasOneUse())
2962 NewCost += CostSel1;
2963 if (!Sel2->hasOneUse())
2964 NewCost += CostSel2;
2965
2966 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two selects: " << I
2967 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2968 << "\n");
2969 if (NewCost > OldCost)
2970 return false;
2971
2972 Value *ShuffleCmp = Builder.CreateShuffleVector(V1: C1, V2: C2, Mask);
2973 Value *ShuffleTrue = Builder.CreateShuffleVector(V1: T1, V2: T2, Mask);
2974 Value *ShuffleFalse = Builder.CreateShuffleVector(V1: F1, V2: F2, Mask);
2975 Value *NewSel;
2976 // We presuppose that the SelectInsts have the same FMF.
2977 if (SI0FOp)
2978 NewSel = Builder.CreateSelectFMF(C: ShuffleCmp, True: ShuffleTrue, False: ShuffleFalse,
2979 FMFSource: SI0FOp->getFastMathFlags());
2980 else
2981 NewSel = Builder.CreateSelect(C: ShuffleCmp, True: ShuffleTrue, False: ShuffleFalse);
2982
2983 replaceValue(Old&: I, New&: *NewSel);
2984 return true;
2985}
2986
2987/// Try to convert "shuffle (castop), (castop)" with a shared castop operand
2988/// into "castop (shuffle)".
2989bool VectorCombine::foldShuffleOfCastops(Instruction &I) {
2990 Value *V0, *V1;
2991 ArrayRef<int> OldMask;
2992 if (!match(V: &I, P: m_Shuffle(v1: m_Value(V&: V0), v2: m_Value(V&: V1), mask: m_Mask(OldMask))))
2993 return false;
2994
2995 // Check whether this is a binary shuffle.
2996 bool IsBinaryShuffle = !isa<UndefValue>(Val: V1);
2997
2998 auto *C0 = dyn_cast<CastInst>(Val: V0);
2999 auto *C1 = dyn_cast<CastInst>(Val: V1);
3000 if (!C0 || (IsBinaryShuffle && !C1))
3001 return false;
3002
3003 Instruction::CastOps Opcode = C0->getOpcode();
3004
3005 // If this is allowed, foldShuffleOfCastops can get stuck in a loop
3006 // with foldBitcastOfShuffle. Reject in favor of foldBitcastOfShuffle.
3007 if (!IsBinaryShuffle && Opcode == Instruction::BitCast)
3008 return false;
3009
3010 if (IsBinaryShuffle) {
3011 if (C0->getSrcTy() != C1->getSrcTy())
3012 return false;
3013 // Handle shuffle(zext_nneg(x), sext(y)) -> sext(shuffle(x,y)) folds.
3014 if (Opcode != C1->getOpcode()) {
3015 if (match(V: C0, P: m_SExtLike(Op: m_Value())) && match(V: C1, P: m_SExtLike(Op: m_Value())))
3016 Opcode = Instruction::SExt;
3017 else
3018 return false;
3019 }
3020 }
3021
3022 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3023 auto *CastDstTy = dyn_cast<FixedVectorType>(Val: C0->getDestTy());
3024 auto *CastSrcTy = dyn_cast<FixedVectorType>(Val: C0->getSrcTy());
3025 if (!ShuffleDstTy || !CastDstTy || !CastSrcTy)
3026 return false;
3027
3028 unsigned NumSrcElts = CastSrcTy->getNumElements();
3029 unsigned NumDstElts = CastDstTy->getNumElements();
3030 assert((NumDstElts == NumSrcElts || Opcode == Instruction::BitCast) &&
3031 "Only bitcasts expected to alter src/dst element counts");
3032
3033 // Check for bitcasting of unscalable vector types.
3034 // e.g. <32 x i40> -> <40 x i32>
3035 if (NumDstElts != NumSrcElts && (NumSrcElts % NumDstElts) != 0 &&
3036 (NumDstElts % NumSrcElts) != 0)
3037 return false;
3038
3039 SmallVector<int, 16> NewMask;
3040 if (NumSrcElts >= NumDstElts) {
3041 // The bitcast is from wide to narrow/equal elements. The shuffle mask can
3042 // always be expanded to the equivalent form choosing narrower elements.
3043 assert(NumSrcElts % NumDstElts == 0 && "Unexpected shuffle mask");
3044 unsigned ScaleFactor = NumSrcElts / NumDstElts;
3045 narrowShuffleMaskElts(Scale: ScaleFactor, Mask: OldMask, ScaledMask&: NewMask);
3046 } else {
3047 // The bitcast is from narrow elements to wide elements. The shuffle mask
3048 // must choose consecutive elements to allow casting first.
3049 assert(NumDstElts % NumSrcElts == 0 && "Unexpected shuffle mask");
3050 unsigned ScaleFactor = NumDstElts / NumSrcElts;
3051 if (!widenShuffleMaskElts(Scale: ScaleFactor, Mask: OldMask, ScaledMask&: NewMask))
3052 return false;
3053 }
3054
3055 auto *NewShuffleDstTy =
3056 FixedVectorType::get(ElementType: CastSrcTy->getScalarType(), NumElts: NewMask.size());
3057
3058 // Try to replace a castop with a shuffle if the shuffle is not costly.
3059 InstructionCost CostC0 =
3060 TTI.getCastInstrCost(Opcode: C0->getOpcode(), Dst: CastDstTy, Src: CastSrcTy,
3061 CCH: TTI::CastContextHint::None, CostKind, I: C0);
3062
3063 TargetTransformInfo::ShuffleKind ShuffleKind;
3064 if (IsBinaryShuffle)
3065 ShuffleKind = TargetTransformInfo::SK_PermuteTwoSrc;
3066 else
3067 ShuffleKind = TargetTransformInfo::SK_PermuteSingleSrc;
3068
3069 InstructionCost OldCost = CostC0;
3070 OldCost += TTI.getShuffleCost(Kind: ShuffleKind, DstTy: ShuffleDstTy, SrcTy: CastDstTy, CostKind,
3071 Mask: OldMask, Index: 0, SubTp: nullptr, Args: {}, CtxI: &I);
3072
3073 InstructionCost NewCost = TTI.getShuffleCost(Kind: ShuffleKind, DstTy: NewShuffleDstTy,
3074 SrcTy: CastSrcTy, CostKind, Mask: NewMask);
3075 NewCost += TTI.getCastInstrCost(Opcode, Dst: ShuffleDstTy, Src: NewShuffleDstTy,
3076 CCH: TTI::CastContextHint::None, CostKind);
3077 if (!C0->hasOneUse())
3078 NewCost += CostC0;
3079 if (IsBinaryShuffle) {
3080 InstructionCost CostC1 =
3081 TTI.getCastInstrCost(Opcode: C1->getOpcode(), Dst: CastDstTy, Src: CastSrcTy,
3082 CCH: TTI::CastContextHint::None, CostKind, I: C1);
3083 OldCost += CostC1;
3084 if (!C1->hasOneUse())
3085 NewCost += CostC1;
3086 }
3087
3088 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two casts: " << I
3089 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
3090 << "\n");
3091 if (NewCost > OldCost)
3092 return false;
3093
3094 Value *Shuf;
3095 if (IsBinaryShuffle)
3096 Shuf = Builder.CreateShuffleVector(V1: C0->getOperand(i_nocapture: 0), V2: C1->getOperand(i_nocapture: 0),
3097 Mask: NewMask);
3098 else
3099 Shuf = Builder.CreateShuffleVector(V: C0->getOperand(i_nocapture: 0), Mask: NewMask);
3100
3101 Value *Cast = Builder.CreateCast(Op: Opcode, V: Shuf, DestTy: ShuffleDstTy);
3102
3103 // Intersect flags from the old casts.
3104 if (auto *NewInst = dyn_cast<Instruction>(Val: Cast)) {
3105 NewInst->copyIRFlags(V: C0);
3106 if (IsBinaryShuffle)
3107 NewInst->andIRFlags(V: C1);
3108 }
3109
3110 replaceValue(Old&: I, New&: *Cast);
3111 return true;
3112}
3113
3114/// Try to convert any of:
3115/// "shuffle (shuffle x, y), (shuffle y, x)"
3116/// "shuffle (shuffle x, undef), (shuffle y, undef)"
3117/// "shuffle (shuffle x, undef), y"
3118/// "shuffle x, (shuffle y, undef)"
3119/// into "shuffle x, y".
3120bool VectorCombine::foldShuffleOfShuffles(Instruction &I) {
3121 ArrayRef<int> OuterMask;
3122 Value *OuterV0, *OuterV1;
3123 if (!match(V: &I,
3124 P: m_Shuffle(v1: m_Value(V&: OuterV0), v2: m_Value(V&: OuterV1), mask: m_Mask(OuterMask))))
3125 return false;
3126
3127 ArrayRef<int> InnerMask0, InnerMask1;
3128 Value *X0, *X1, *Y0, *Y1;
3129 bool Match0 =
3130 match(V: OuterV0, P: m_Shuffle(v1: m_Value(V&: X0), v2: m_Value(V&: Y0), mask: m_Mask(InnerMask0)));
3131 bool Match1 =
3132 match(V: OuterV1, P: m_Shuffle(v1: m_Value(V&: X1), v2: m_Value(V&: Y1), mask: m_Mask(InnerMask1)));
3133 if (!Match0 && !Match1)
3134 return false;
3135
3136 // If the outer shuffle is a permute, then create a fake inner all-poison
3137 // shuffle. This is easier than accounting for length-changing shuffles below.
3138 SmallVector<int, 16> PoisonMask1;
3139 if (!Match1 && isa<PoisonValue>(Val: OuterV1)) {
3140 X1 = X0;
3141 Y1 = Y0;
3142 PoisonMask1.append(NumInputs: InnerMask0.size(), Elt: PoisonMaskElem);
3143 InnerMask1 = PoisonMask1;
3144 Match1 = true; // fake match
3145 }
3146
3147 X0 = Match0 ? X0 : OuterV0;
3148 Y0 = Match0 ? Y0 : OuterV0;
3149 X1 = Match1 ? X1 : OuterV1;
3150 Y1 = Match1 ? Y1 : OuterV1;
3151 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3152 auto *ShuffleSrcTy = dyn_cast<FixedVectorType>(Val: X0->getType());
3153 auto *ShuffleImmTy = dyn_cast<FixedVectorType>(Val: OuterV0->getType());
3154 if (!ShuffleDstTy || !ShuffleSrcTy || !ShuffleImmTy ||
3155 X0->getType() != X1->getType())
3156 return false;
3157
3158 unsigned NumSrcElts = ShuffleSrcTy->getNumElements();
3159 unsigned NumImmElts = ShuffleImmTy->getNumElements();
3160
3161 // Attempt to merge shuffles, matching upto 2 source operands.
3162 // Replace index to a poison arg with PoisonMaskElem.
3163 // Bail if either inner masks reference an undef arg.
3164 SmallVector<int, 16> NewMask(OuterMask);
3165 Value *NewX = nullptr, *NewY = nullptr;
3166 for (int &M : NewMask) {
3167 Value *Src = nullptr;
3168 if (0 <= M && M < (int)NumImmElts) {
3169 Src = OuterV0;
3170 if (Match0) {
3171 M = InnerMask0[M];
3172 Src = M >= (int)NumSrcElts ? Y0 : X0;
3173 M = M >= (int)NumSrcElts ? (M - NumSrcElts) : M;
3174 }
3175 } else if (M >= (int)NumImmElts) {
3176 Src = OuterV1;
3177 M -= NumImmElts;
3178 if (Match1) {
3179 M = InnerMask1[M];
3180 Src = M >= (int)NumSrcElts ? Y1 : X1;
3181 M = M >= (int)NumSrcElts ? (M - NumSrcElts) : M;
3182 }
3183 }
3184 if (Src && M != PoisonMaskElem) {
3185 assert(0 <= M && M < (int)NumSrcElts && "Unexpected shuffle mask index");
3186 if (isa<UndefValue>(Val: Src)) {
3187 // We've referenced an undef element - if its poison, update the shuffle
3188 // mask, else bail.
3189 if (!isa<PoisonValue>(Val: Src))
3190 return false;
3191 M = PoisonMaskElem;
3192 continue;
3193 }
3194 if (!NewX || NewX == Src) {
3195 NewX = Src;
3196 continue;
3197 }
3198 if (!NewY || NewY == Src) {
3199 M += NumSrcElts;
3200 NewY = Src;
3201 continue;
3202 }
3203 return false;
3204 }
3205 }
3206
3207 if (!NewX) {
3208 replaceValue(Old&: I, New&: *PoisonValue::get(T: ShuffleDstTy));
3209 return true;
3210 }
3211
3212 if (!NewY)
3213 NewY = PoisonValue::get(T: ShuffleSrcTy);
3214
3215 // Have we folded to an Identity shuffle?
3216 if (ShuffleVectorInst::isIdentityMask(Mask: NewMask, NumSrcElts)) {
3217 replaceValue(Old&: I, New&: *NewX);
3218 return true;
3219 }
3220
3221 // Try to merge the shuffles if the new shuffle is not costly.
3222 InstructionCost InnerCost0 = 0;
3223 if (Match0)
3224 InnerCost0 = TTI.getInstructionCost(U: cast<User>(Val: OuterV0), CostKind);
3225
3226 InstructionCost InnerCost1 = 0;
3227 if (Match1)
3228 InnerCost1 = TTI.getInstructionCost(U: cast<User>(Val: OuterV1), CostKind);
3229
3230 InstructionCost OuterCost = TTI.getInstructionCost(U: &I, CostKind);
3231
3232 InstructionCost OldCost = InnerCost0 + InnerCost1 + OuterCost;
3233
3234 bool IsUnary = all_of(Range&: NewMask, P: [&](int M) { return M < (int)NumSrcElts; });
3235 TargetTransformInfo::ShuffleKind SK =
3236 IsUnary ? TargetTransformInfo::SK_PermuteSingleSrc
3237 : TargetTransformInfo::SK_PermuteTwoSrc;
3238 InstructionCost NewCost =
3239 TTI.getShuffleCost(Kind: SK, DstTy: ShuffleDstTy, SrcTy: ShuffleSrcTy, CostKind, Mask: NewMask, Index: 0,
3240 SubTp: nullptr, Args: {NewX, NewY});
3241 if (!OuterV0->hasOneUse())
3242 NewCost += InnerCost0;
3243 if (!OuterV1->hasOneUse())
3244 NewCost += InnerCost1;
3245
3246 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two shuffles: " << I
3247 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
3248 << "\n");
3249 if (NewCost > OldCost)
3250 return false;
3251
3252 Value *Shuf = Builder.CreateShuffleVector(V1: NewX, V2: NewY, Mask: NewMask);
3253 replaceValue(Old&: I, New&: *Shuf);
3254 return true;
3255}
3256
3257/// Try to convert a chain of length-preserving shuffles that are fed by
3258/// length-changing shuffles from the same source, e.g. a chain of length 3:
3259///
3260/// "shuffle (shuffle (shuffle x, (shuffle y, undef)),
3261/// (shuffle y, undef)),
3262// (shuffle y, undef)"
3263///
3264/// into a single shuffle fed by a length-changing shuffle:
3265///
3266/// "shuffle x, (shuffle y, undef)"
3267///
3268/// Such chains arise e.g. from folding extract/insert sequences.
3269bool VectorCombine::foldShufflesOfLengthChangingShuffles(Instruction &I) {
3270 FixedVectorType *TrunkType = dyn_cast<FixedVectorType>(Val: I.getType());
3271 if (!TrunkType)
3272 return false;
3273
3274 unsigned ChainLength = 0;
3275 SmallVector<int> Mask;
3276 SmallVector<int> YMask;
3277 InstructionCost OldCost = 0;
3278 InstructionCost NewCost = 0;
3279 Value *Trunk = &I;
3280 unsigned NumTrunkElts = TrunkType->getNumElements();
3281 Value *Y = nullptr;
3282
3283 for (;;) {
3284 // Match the current trunk against (commutations of) the pattern
3285 // "shuffle trunk', (shuffle y, undef)"
3286 ArrayRef<int> OuterMask;
3287 Value *OuterV0, *OuterV1;
3288 if (ChainLength != 0 && !Trunk->hasOneUse())
3289 break;
3290 if (!match(V: Trunk, P: m_Shuffle(v1: m_Value(V&: OuterV0), v2: m_Value(V&: OuterV1),
3291 mask: m_Mask(OuterMask))))
3292 break;
3293 if (OuterV0->getType() != TrunkType) {
3294 // This shuffle is not length-preserving, so it cannot be part of the
3295 // chain.
3296 break;
3297 }
3298
3299 ArrayRef<int> InnerMask0, InnerMask1;
3300 Value *A0, *A1, *B0, *B1;
3301 bool Match0 =
3302 match(V: OuterV0, P: m_Shuffle(v1: m_Value(V&: A0), v2: m_Value(V&: B0), mask: m_Mask(InnerMask0)));
3303 bool Match1 =
3304 match(V: OuterV1, P: m_Shuffle(v1: m_Value(V&: A1), v2: m_Value(V&: B1), mask: m_Mask(InnerMask1)));
3305 bool Match0Leaf = Match0 && A0->getType() != I.getType();
3306 bool Match1Leaf = Match1 && A1->getType() != I.getType();
3307 if (Match0Leaf == Match1Leaf) {
3308 // Only handle the case of exactly one leaf in each step. The "two leaves"
3309 // case is handled by foldShuffleOfShuffles.
3310 break;
3311 }
3312
3313 SmallVector<int> CommutedOuterMask;
3314 if (Match0Leaf) {
3315 std::swap(a&: OuterV0, b&: OuterV1);
3316 std::swap(a&: InnerMask0, b&: InnerMask1);
3317 std::swap(a&: A0, b&: A1);
3318 std::swap(a&: B0, b&: B1);
3319 llvm::append_range(C&: CommutedOuterMask, R&: OuterMask);
3320 for (int &M : CommutedOuterMask) {
3321 if (M == PoisonMaskElem)
3322 continue;
3323 if (M < (int)NumTrunkElts)
3324 M += NumTrunkElts;
3325 else
3326 M -= NumTrunkElts;
3327 }
3328 OuterMask = CommutedOuterMask;
3329 }
3330 if (!OuterV1->hasOneUse())
3331 break;
3332
3333 if (!isa<UndefValue>(Val: A1)) {
3334 if (!Y)
3335 Y = A1;
3336 else if (Y != A1)
3337 break;
3338 }
3339 if (!isa<UndefValue>(Val: B1)) {
3340 if (!Y)
3341 Y = B1;
3342 else if (Y != B1)
3343 break;
3344 }
3345
3346 auto *YType = cast<FixedVectorType>(Val: A1->getType());
3347 int NumLeafElts = YType->getNumElements();
3348 SmallVector<int> LocalYMask(InnerMask1);
3349 for (int &M : LocalYMask) {
3350 if (M >= NumLeafElts)
3351 M -= NumLeafElts;
3352 }
3353
3354 InstructionCost LocalOldCost =
3355 TTI.getInstructionCost(U: cast<User>(Val: Trunk), CostKind) +
3356 TTI.getInstructionCost(U: cast<User>(Val: OuterV1), CostKind);
3357
3358 // Handle the initial (start of chain) case.
3359 if (!ChainLength) {
3360 Mask.assign(AR: OuterMask);
3361 YMask.assign(RHS: LocalYMask);
3362 OldCost = NewCost = LocalOldCost;
3363 Trunk = OuterV0;
3364 ChainLength++;
3365 continue;
3366 }
3367
3368 // For the non-root case, first attempt to combine masks.
3369 SmallVector<int> NewYMask(YMask);
3370 bool Valid = true;
3371 for (auto [CombinedM, LeafM] : llvm::zip(t&: NewYMask, u&: LocalYMask)) {
3372 if (LeafM == -1 || CombinedM == LeafM)
3373 continue;
3374 if (CombinedM == -1) {
3375 CombinedM = LeafM;
3376 } else {
3377 Valid = false;
3378 break;
3379 }
3380 }
3381 if (!Valid)
3382 break;
3383
3384 SmallVector<int> NewMask;
3385 NewMask.reserve(N: NumTrunkElts);
3386 for (int M : Mask) {
3387 if (M < 0 || M >= static_cast<int>(NumTrunkElts))
3388 NewMask.push_back(Elt: M);
3389 else
3390 NewMask.push_back(Elt: OuterMask[M]);
3391 }
3392
3393 // Break the chain if adding this new step complicates the shuffles such
3394 // that it would increase the new cost by more than the old cost of this
3395 // step.
3396 InstructionCost LocalNewCost =
3397 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc, DstTy: TrunkType,
3398 SrcTy: YType, CostKind, Mask: NewYMask) +
3399 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: TrunkType,
3400 SrcTy: TrunkType, CostKind, Mask: NewMask);
3401
3402 if (LocalNewCost >= NewCost && LocalOldCost < LocalNewCost - NewCost)
3403 break;
3404
3405 LLVM_DEBUG({
3406 if (ChainLength == 1) {
3407 dbgs() << "Found chain of shuffles fed by length-changing shuffles: "
3408 << I << '\n';
3409 }
3410 dbgs() << " next chain link: " << *Trunk << '\n'
3411 << " old cost: " << (OldCost + LocalOldCost)
3412 << " new cost: " << LocalNewCost << '\n';
3413 });
3414
3415 Mask = NewMask;
3416 YMask = NewYMask;
3417 OldCost += LocalOldCost;
3418 NewCost = LocalNewCost;
3419 Trunk = OuterV0;
3420 ChainLength++;
3421 }
3422 if (ChainLength <= 1)
3423 return false;
3424
3425 // Bail out if all leaves were poison.
3426 if (!Y)
3427 return false;
3428
3429 if (llvm::all_of(Range&: Mask, P: [&](int M) {
3430 return M < 0 || M >= static_cast<int>(NumTrunkElts);
3431 })) {
3432 // Produce a canonical simplified form if all elements are sourced from Y.
3433 for (int &M : Mask) {
3434 if (M >= static_cast<int>(NumTrunkElts))
3435 M = YMask[M - NumTrunkElts];
3436 }
3437 Value *Root =
3438 Builder.CreateShuffleVector(V1: Y, V2: PoisonValue::get(T: Y->getType()), Mask);
3439 replaceValue(Old&: I, New&: *Root);
3440 return true;
3441 }
3442
3443 Value *Leaf =
3444 Builder.CreateShuffleVector(V1: Y, V2: PoisonValue::get(T: Y->getType()), Mask: YMask);
3445 Value *Root = Builder.CreateShuffleVector(V1: Trunk, V2: Leaf, Mask);
3446 replaceValue(Old&: I, New&: *Root);
3447 return true;
3448}
3449
3450/// Try to convert
3451/// "shuffle (intrinsic), (intrinsic)" into "intrinsic (shuffle), (shuffle)".
3452bool VectorCombine::foldShuffleOfIntrinsics(Instruction &I) {
3453 Value *V0, *V1;
3454 ArrayRef<int> OldMask;
3455 if (!match(V: &I, P: m_Shuffle(v1: m_Value(V&: V0), v2: m_Value(V&: V1), mask: m_Mask(OldMask))))
3456 return false;
3457
3458 auto *II0 = dyn_cast<IntrinsicInst>(Val: V0);
3459 auto *II1 = dyn_cast<IntrinsicInst>(Val: V1);
3460 if (!II0 || !II1)
3461 return false;
3462
3463 Intrinsic::ID IID = II0->getIntrinsicID();
3464 if (IID != II1->getIntrinsicID())
3465 return false;
3466 InstructionCost CostII0 =
3467 TTI.getIntrinsicInstrCost(ICA: IntrinsicCostAttributes(IID, *II0), CostKind);
3468 InstructionCost CostII1 =
3469 TTI.getIntrinsicInstrCost(ICA: IntrinsicCostAttributes(IID, *II1), CostKind);
3470
3471 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3472 auto *II0Ty = dyn_cast<FixedVectorType>(Val: II0->getType());
3473 if (!ShuffleDstTy || !II0Ty)
3474 return false;
3475
3476 if (!isTriviallyVectorizable(ID: IID))
3477 return false;
3478
3479 for (unsigned Idx = 0, E = II0->arg_size(); Idx != E; ++Idx) {
3480 Value *Arg0 = II0->getArgOperand(i: Idx);
3481 Value *Arg1 = II1->getArgOperand(i: Idx);
3482 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: Idx, TTI: &TTI)) {
3483 // Scalar operands must be identical.
3484 if (Arg0 != Arg1)
3485 return false;
3486 } else if (Arg0->getType() != Arg1->getType()) {
3487 // The corresponding vector operands are shuffled together, so they must
3488 // share the same type. For intrinsics overloaded on their operand type
3489 // (e.g. llvm.fptosi.sat), two calls can produce the same result type
3490 // from different operand types; shuffling those would be invalid.
3491 return false;
3492 }
3493 }
3494
3495 InstructionCost OldCost =
3496 CostII0 + CostII1 +
3497 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ShuffleDstTy,
3498 SrcTy: II0Ty, CostKind, Mask: OldMask, Index: 0, SubTp: nullptr, Args: {II0, II1}, CtxI: &I);
3499
3500 SmallVector<Type *> NewArgsTy;
3501 InstructionCost NewCost = 0;
3502 SmallDenseSet<std::pair<Value *, Value *>> SeenOperandPairs;
3503 for (unsigned Idx = 0, E = II0->arg_size(); Idx != E; ++Idx) {
3504 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: Idx, TTI: &TTI)) {
3505 NewArgsTy.push_back(Elt: II0->getArgOperand(i: Idx)->getType());
3506 } else {
3507 auto *VecTy = cast<FixedVectorType>(Val: II0->getArgOperand(i: Idx)->getType());
3508 auto *ArgTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
3509 NumElts: ShuffleDstTy->getNumElements());
3510 NewArgsTy.push_back(Elt: ArgTy);
3511 std::pair<Value *, Value *> OperandPair =
3512 std::make_pair(x: II0->getArgOperand(i: Idx), y: II1->getArgOperand(i: Idx));
3513 if (!SeenOperandPairs.insert(V: OperandPair).second) {
3514 // We've already computed the cost for this operand pair.
3515 continue;
3516 }
3517 NewCost += TTI.getShuffleCost(
3518 Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ArgTy, SrcTy: VecTy, CostKind,
3519 Mask: OldMask, Index: 0, SubTp: nullptr,
3520 Args: {II0->getArgOperand(i: Idx), II1->getArgOperand(i: Idx)});
3521 }
3522 }
3523 IntrinsicCostAttributes NewAttr(IID, ShuffleDstTy, NewArgsTy);
3524
3525 NewCost += TTI.getIntrinsicInstrCost(ICA: NewAttr, CostKind);
3526 if (!II0->hasOneUse())
3527 NewCost += CostII0;
3528 if (II1 != II0 && !II1->hasOneUse())
3529 NewCost += CostII1;
3530
3531 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two intrinsics: " << I
3532 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
3533 << "\n");
3534
3535 if (NewCost > OldCost)
3536 return false;
3537
3538 SmallVector<Value *> NewArgs;
3539 SmallDenseMap<std::pair<Value *, Value *>, Value *> ShuffleCache;
3540 for (unsigned Idx = 0, E = II0->arg_size(); Idx != E; ++Idx) {
3541 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: Idx, TTI: &TTI)) {
3542 NewArgs.push_back(Elt: II0->getArgOperand(i: Idx));
3543 } else {
3544 std::pair<Value *, Value *> OperandPair =
3545 std::make_pair(x: II0->getArgOperand(i: Idx), y: II1->getArgOperand(i: Idx));
3546 auto It = ShuffleCache.find(Val: OperandPair);
3547 if (It != ShuffleCache.end()) {
3548 // Reuse previously created shuffle for this operand pair.
3549 NewArgs.push_back(Elt: It->second);
3550 continue;
3551 }
3552 Value *Shuf = Builder.CreateShuffleVector(
3553 V1: II0->getArgOperand(i: Idx), V2: II1->getArgOperand(i: Idx), Mask: OldMask);
3554 ShuffleCache[OperandPair] = Shuf;
3555 NewArgs.push_back(Elt: Shuf);
3556 }
3557 }
3558 Value *NewIntrinsic = Builder.CreateIntrinsic(RetTy: ShuffleDstTy, ID: IID, Args: NewArgs);
3559
3560 // Intersect flags from the old intrinsics.
3561 if (auto *NewInst = dyn_cast<Instruction>(Val: NewIntrinsic)) {
3562 NewInst->copyIRFlags(V: II0);
3563 NewInst->andIRFlags(V: II1);
3564 }
3565
3566 replaceValue(Old&: I, New&: *NewIntrinsic);
3567 return true;
3568}
3569
3570/// Try to convert
3571/// "shuffle (intrinsic), (poison/undef)" into "intrinsic (shuffle)".
3572bool VectorCombine::foldPermuteOfIntrinsic(Instruction &I) {
3573 Value *V0;
3574 ArrayRef<int> Mask;
3575 if (!match(V: &I, P: m_Shuffle(v1: m_Value(V&: V0), v2: m_Undef(), mask: m_Mask(Mask))))
3576 return false;
3577
3578 auto *II0 = dyn_cast<IntrinsicInst>(Val: V0);
3579 if (!II0)
3580 return false;
3581
3582 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3583 auto *IntrinsicSrcTy = dyn_cast<FixedVectorType>(Val: II0->getType());
3584 if (!ShuffleDstTy || !IntrinsicSrcTy)
3585 return false;
3586
3587 // Validate it's a pure permute, mask should only reference the first vector
3588 unsigned NumSrcElts = IntrinsicSrcTy->getNumElements();
3589 if (any_of(Range&: Mask, P: [NumSrcElts](int M) { return M >= (int)NumSrcElts; }))
3590 return false;
3591
3592 Intrinsic::ID IID = II0->getIntrinsicID();
3593 if (!isTriviallyVectorizable(ID: IID))
3594 return false;
3595
3596 // Cost analysis
3597 InstructionCost IntrinsicCost =
3598 TTI.getIntrinsicInstrCost(ICA: IntrinsicCostAttributes(IID, *II0), CostKind);
3599 InstructionCost OldCost =
3600 IntrinsicCost +
3601 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc, DstTy: ShuffleDstTy,
3602 SrcTy: IntrinsicSrcTy, CostKind, Mask, Index: 0, SubTp: nullptr, Args: {V0}, CtxI: &I);
3603
3604 SmallVector<Type *> NewArgsTy;
3605 InstructionCost NewCost = 0;
3606 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I) {
3607 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3608 NewArgsTy.push_back(Elt: II0->getArgOperand(i: I)->getType());
3609 } else {
3610 auto *VecTy = cast<FixedVectorType>(Val: II0->getArgOperand(i: I)->getType());
3611 auto *ArgTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
3612 NumElts: ShuffleDstTy->getNumElements());
3613 NewArgsTy.push_back(Elt: ArgTy);
3614 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
3615 DstTy: ArgTy, SrcTy: VecTy, CostKind, Mask, Index: 0, SubTp: nullptr,
3616 Args: {II0->getArgOperand(i: I)});
3617 }
3618 }
3619 IntrinsicCostAttributes NewAttr(IID, ShuffleDstTy, NewArgsTy);
3620 NewCost += TTI.getIntrinsicInstrCost(ICA: NewAttr, CostKind);
3621
3622 // If the intrinsic has multiple uses, we need to account for the cost of
3623 // keeping the original intrinsic around.
3624 if (!II0->hasOneUse())
3625 NewCost += IntrinsicCost;
3626
3627 LLVM_DEBUG(dbgs() << "Found a permute of intrinsic: " << I << "\n OldCost: "
3628 << OldCost << " vs NewCost: " << NewCost << "\n");
3629
3630 if (NewCost > OldCost)
3631 return false;
3632
3633 // Transform
3634 SmallVector<Value *> NewArgs;
3635 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I) {
3636 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3637 NewArgs.push_back(Elt: II0->getArgOperand(i: I));
3638 } else {
3639 Value *Shuf = Builder.CreateShuffleVector(V: II0->getArgOperand(i: I), Mask);
3640 NewArgs.push_back(Elt: Shuf);
3641 }
3642 }
3643
3644 Value *NewIntrinsic = Builder.CreateIntrinsic(RetTy: ShuffleDstTy, ID: IID, Args: NewArgs);
3645
3646 if (auto *NewInst = dyn_cast<Instruction>(Val: NewIntrinsic))
3647 NewInst->copyIRFlags(V: II0);
3648
3649 replaceValue(Old&: I, New&: *NewIntrinsic);
3650 return true;
3651}
3652
3653using InstLane = std::pair<Value *, int>;
3654
3655static InstLane lookThroughShuffles(Value *V, int Lane) {
3656 while (auto *SV = dyn_cast<ShuffleVectorInst>(Val: V)) {
3657 unsigned NumElts =
3658 cast<FixedVectorType>(Val: SV->getOperand(i_nocapture: 0)->getType())->getNumElements();
3659 int M = SV->getMaskValue(Elt: Lane);
3660 if (M < 0)
3661 return {nullptr, PoisonMaskElem};
3662 if (static_cast<unsigned>(M) < NumElts) {
3663 V = SV->getOperand(i_nocapture: 0);
3664 Lane = M;
3665 } else {
3666 V = SV->getOperand(i_nocapture: 1);
3667 Lane = M - NumElts;
3668 }
3669 }
3670 return InstLane{V, Lane};
3671}
3672
3673static SmallVector<InstLane>
3674generateInstLaneVectorFromOperand(ArrayRef<InstLane> Item, int Op) {
3675 SmallVector<InstLane> NItem;
3676 for (InstLane IL : Item) {
3677 auto [U, Lane] = IL;
3678 InstLane OpLane =
3679 U ? lookThroughShuffles(V: cast<Instruction>(Val: U)->getOperand(i: Op), Lane)
3680 : InstLane{nullptr, PoisonMaskElem};
3681 NItem.emplace_back(Args&: OpLane);
3682 }
3683 return NItem;
3684}
3685
3686/// Detect concat of multiple values into a vector
3687static bool isFreeConcat(ArrayRef<InstLane> Item, TTI::TargetCostKind CostKind,
3688 const TargetTransformInfo &TTI) {
3689 auto *Ty = cast<FixedVectorType>(Val: Item.front().first->getType());
3690 unsigned NumElts = Ty->getNumElements();
3691 if (Item.size() == NumElts || NumElts == 1 || Item.size() % NumElts != 0)
3692 return false;
3693
3694 // Check that the concat is free, usually meaning that the type will be split
3695 // during legalization.
3696 SmallVector<int, 16> ConcatMask(NumElts * 2);
3697 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
3698 if (TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc,
3699 DstTy: FixedVectorType::get(ElementType: Ty->getScalarType(), NumElts: NumElts * 2),
3700 SrcTy: Ty, CostKind, Mask: ConcatMask) != 0)
3701 return false;
3702
3703 unsigned NumSlices = Item.size() / NumElts;
3704 // Currently we generate a tree of shuffles for the concats, which limits us
3705 // to a power2.
3706 if (!isPowerOf2_32(Value: NumSlices))
3707 return false;
3708 for (unsigned Slice = 0; Slice < NumSlices; ++Slice) {
3709 Value *SliceV = Item[Slice * NumElts].first;
3710 if (!SliceV || SliceV->getType() != Ty)
3711 return false;
3712 for (unsigned Elt = 0; Elt < NumElts; ++Elt) {
3713 auto [V, Lane] = Item[Slice * NumElts + Elt];
3714 if (Lane != static_cast<int>(Elt) || SliceV != V)
3715 return false;
3716 }
3717 }
3718 return true;
3719}
3720
3721static Value *
3722generateNewInstTree(ArrayRef<InstLane> Item, Use *From,
3723 const DenseSet<std::pair<Value *, Use *>> &IdentityLeafs,
3724 const DenseSet<std::pair<Value *, Use *>> &SplatLeafs,
3725 const DenseSet<std::pair<Value *, Use *>> &ConcatLeafs,
3726 IRBuilderBase &Builder, InstructionWorklist &WorkList,
3727 const TargetTransformInfo *TTI) {
3728 auto [FrontV, FrontLane] = Item.front();
3729
3730 if (IdentityLeafs.contains(V: std::make_pair(x&: FrontV, y&: From))) {
3731 return FrontV;
3732 }
3733 if (SplatLeafs.contains(V: std::make_pair(x&: FrontV, y&: From))) {
3734 SmallVector<int, 16> Mask(Item.size(), FrontLane);
3735 return Builder.CreateShuffleVector(V: FrontV, Mask);
3736 }
3737 if (ConcatLeafs.contains(V: std::make_pair(x&: FrontV, y&: From))) {
3738 unsigned NumElts =
3739 cast<FixedVectorType>(Val: FrontV->getType())->getNumElements();
3740 SmallVector<Value *> Values(Item.size() / NumElts, nullptr);
3741 for (unsigned S = 0; S < Values.size(); ++S)
3742 Values[S] = Item[S * NumElts].first;
3743
3744 while (Values.size() > 1) {
3745 NumElts *= 2;
3746 SmallVector<int, 16> Mask(NumElts, 0);
3747 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
3748 SmallVector<Value *> NewValues(Values.size() / 2, nullptr);
3749 for (unsigned S = 0; S < NewValues.size(); ++S)
3750 NewValues[S] =
3751 Builder.CreateShuffleVector(V1: Values[S * 2], V2: Values[S * 2 + 1], Mask);
3752 Values = NewValues;
3753 }
3754 return Values[0];
3755 }
3756
3757 auto *I = cast<Instruction>(Val: FrontV);
3758
3759 // Handle vector bitcasts that change element count. We cannot use
3760 // generateInstLaneVectorFromOperand for these because the lane indices
3761 // don't map 1:1 through the bitcast.
3762 if (auto *BitCast = dyn_cast<BitCastInst>(Val: I)) {
3763 auto *BCDstTy = dyn_cast<FixedVectorType>(Val: BitCast->getDestTy());
3764 auto *BCSrcTy = dyn_cast<FixedVectorType>(Val: BitCast->getSrcTy());
3765 if (BCDstTy && BCSrcTy &&
3766 BCDstTy->getElementCount() != BCSrcTy->getElementCount()) {
3767 unsigned DstElts = BCDstTy->getNumElements();
3768 unsigned SrcElts = BCSrcTy->getNumElements();
3769 SmallVector<InstLane> NewItem;
3770 if (DstElts > SrcElts) {
3771 // Widening: compress operand Item.
3772 unsigned R = DstElts / SrcElts;
3773 if (Item.size() % R != 0)
3774 return nullptr;
3775 for (unsigned Idx = 0, E = Item.size(); Idx < E; Idx += R) {
3776 auto [V, Lane] = Item[Idx];
3777 if (!V) {
3778 NewItem.push_back(Elt: {nullptr, PoisonMaskElem});
3779 continue;
3780 }
3781 NewItem.push_back(
3782 Elt: lookThroughShuffles(V: cast<Operator>(Val: V)->getOperand(i: 0), Lane: Lane / R));
3783 }
3784 } else {
3785 // Narrowing: expand operand Item.
3786 unsigned R = SrcElts / DstElts;
3787 for (auto [V, Lane] : Item) {
3788 if (!V) {
3789 NewItem.append(NumInputs: R, Elt: {nullptr, PoisonMaskElem});
3790 continue;
3791 }
3792 Value *Op = cast<Operator>(Val: V)->getOperand(i: 0);
3793 for (unsigned J = 0; J < R; ++J)
3794 NewItem.push_back(Elt: lookThroughShuffles(V: Op, Lane: Lane * R + J));
3795 }
3796 }
3797 Value *Op = generateNewInstTree(Item: NewItem, From: &BitCast->getOperandUse(i: 0),
3798 IdentityLeafs, SplatLeafs, ConcatLeafs,
3799 Builder, WorkList, TTI);
3800 return Builder.CreateBitCast(
3801 V: Op, DestTy: FixedVectorType::get(ElementType: BCDstTy->getScalarType(), NumElts: Item.size()));
3802 }
3803 }
3804 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3805 unsigned NumOps = I->getNumOperands() - (II ? 1 : 0);
3806 SmallVector<Value *> Ops(NumOps);
3807 for (unsigned Idx = 0; Idx < NumOps; Idx++) {
3808 if (II &&
3809 isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(), ScalarOpdIdx: Idx, TTI)) {
3810 Ops[Idx] = II->getOperand(i_nocapture: Idx);
3811 continue;
3812 }
3813 Ops[Idx] = generateNewInstTree(
3814 Item: generateInstLaneVectorFromOperand(Item, Op: Idx), From: &I->getOperandUse(i: Idx),
3815 IdentityLeafs, SplatLeafs, ConcatLeafs, Builder, WorkList, TTI);
3816 // Don't re-queue the operand of a bitcast we just regenerated. Doing so
3817 // lets foldBitcastShuffle sink the bitcast back into a shuffle(bitcast),
3818 // which foldShuffleToIdentity then re-matches as the same superfluous
3819 // identity - an infinite loop between the two folds.
3820 // TODO: we should fix algorithms in foldBitcastShuffle and
3821 // foldShuffleToIdentity instead of workaround the problem in worklist.
3822 if (isa<BitCastInst>(Val: I) && isa<Instruction>(Val: Ops[Idx]))
3823 WorkList.remove(I: cast<Instruction>(Val: Ops[Idx]));
3824 }
3825
3826 SmallVector<Value *, 8> ValueList;
3827 for (const auto &Lane : Item)
3828 if (Lane.first)
3829 ValueList.push_back(Elt: Lane.first);
3830
3831 Type *DstTy =
3832 FixedVectorType::get(ElementType: I->getType()->getScalarType(), NumElts: Item.size());
3833 if (auto *BI = dyn_cast<BinaryOperator>(Val: I)) {
3834 auto *Value = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)BI->getOpcode(),
3835 LHS: Ops[0], RHS: Ops[1]);
3836 propagateIRFlags(I: Value, VL: ValueList);
3837 return Value;
3838 }
3839 if (auto *CI = dyn_cast<CmpInst>(Val: I)) {
3840 auto *Value = Builder.CreateCmp(Pred: CI->getPredicate(), LHS: Ops[0], RHS: Ops[1]);
3841 propagateIRFlags(I: Value, VL: ValueList);
3842 return Value;
3843 }
3844 if (auto *SI = dyn_cast<SelectInst>(Val: I)) {
3845 auto *Value = Builder.CreateSelect(C: Ops[0], True: Ops[1], False: Ops[2], Name: "", MDFrom: SI);
3846 propagateIRFlags(I: Value, VL: ValueList);
3847 return Value;
3848 }
3849 if (auto *CI = dyn_cast<CastInst>(Val: I)) {
3850 auto *Value = Builder.CreateCast(Op: CI->getOpcode(), V: Ops[0], DestTy: DstTy);
3851 propagateIRFlags(I: Value, VL: ValueList);
3852 return Value;
3853 }
3854 if (II) {
3855 auto *Value = Builder.CreateIntrinsic(RetTy: DstTy, ID: II->getIntrinsicID(), Args: Ops);
3856 propagateIRFlags(I: Value, VL: ValueList);
3857 return Value;
3858 }
3859 assert(isa<UnaryInstruction>(I) && "Unexpected instruction type in Generate");
3860 auto *Value =
3861 Builder.CreateUnOp(Opc: (Instruction::UnaryOps)I->getOpcode(), V: Ops[0]);
3862 propagateIRFlags(I: Value, VL: ValueList);
3863 return Value;
3864}
3865
3866// Starting from a shuffle, look up through operands tracking the shuffled index
3867// of each lane. If we can simplify away the shuffles to identities then
3868// do so.
3869bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
3870 auto *Ty = dyn_cast<FixedVectorType>(Val: I.getType());
3871 if (!Ty || I.use_empty())
3872 return false;
3873
3874 SmallVector<InstLane> Start(Ty->getNumElements());
3875 for (unsigned M = 0, E = Ty->getNumElements(); M < E; ++M)
3876 Start[M] = lookThroughShuffles(V: &I, Lane: M);
3877
3878 SmallVector<std::pair<SmallVector<InstLane>, Use *>> Candidates;
3879 Candidates.push_back(Elt: std::make_pair(x&: Start, y: &*I.use_begin()));
3880 DenseSet<std::pair<Value *, Use *>> IdentityLeafs, SplatLeafs, ConcatLeafs;
3881 unsigned NumVisited = 0;
3882 bool TraversedElCountChangingBitcast = false;
3883
3884 while (!Candidates.empty()) {
3885 if (++NumVisited > Opts.vector_combine_max_scan_instrs)
3886 return false;
3887
3888 auto ItemFrom = Candidates.pop_back_val();
3889 auto Item = ItemFrom.first;
3890 auto From = ItemFrom.second;
3891 auto [FrontV, FrontLane] = Item.front();
3892
3893 // If we found an undef first lane then bail out to keep things simple.
3894 if (!FrontV)
3895 return false;
3896
3897 // Look for an identity value.
3898 if (FrontLane == 0 &&
3899 cast<FixedVectorType>(Val: FrontV->getType())->getNumElements() ==
3900 Item.size() &&
3901 all_of(Range: drop_begin(RangeOrContainer: enumerate(First&: Item)), P: [Item](const auto &E) {
3902 Value *FrontV = Item.front().first;
3903 return !E.value().first || (isEquivBitcast(E.value().first, FrontV) &&
3904 E.value().second == (int)E.index());
3905 })) {
3906 IdentityLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3907 continue;
3908 }
3909 // Look for constants, for the moment only supporting constant splats.
3910 if (auto *C = dyn_cast<Constant>(Val: FrontV);
3911 C && C->getSplatValue() &&
3912 all_of(Range: drop_begin(RangeOrContainer&: Item), P: [Item](InstLane &IL) {
3913 Value *FrontV = Item.front().first;
3914 Value *V = IL.first;
3915 return !V || (isa<Constant>(Val: V) &&
3916 cast<Constant>(Val: V)->getSplatValue() ==
3917 cast<Constant>(Val: FrontV)->getSplatValue());
3918 })) {
3919 SplatLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3920 continue;
3921 }
3922 // Look for a splat value.
3923 if (all_of(Range: drop_begin(RangeOrContainer&: Item), P: [Item](InstLane &IL) {
3924 auto [FrontV, FrontLane] = Item.front();
3925 auto [V, Lane] = IL;
3926 return !V || (V == FrontV && Lane == FrontLane);
3927 })) {
3928 SplatLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3929 continue;
3930 }
3931
3932 // We need each element to be the same type of value, and check that each
3933 // element has a single use.
3934 auto CheckLaneIsEquivalentToFirst = [Item](InstLane IL) {
3935 Value *FrontV = Item.front().first;
3936 if (!IL.first)
3937 return true;
3938 Value *V = IL.first;
3939 if (auto *I = dyn_cast<Instruction>(Val: V); I && !I->hasOneUser())
3940 return false;
3941 if (V->getValueID() != FrontV->getValueID())
3942 return false;
3943 if (auto *CI = dyn_cast<CmpInst>(Val: V))
3944 if (CI->getPredicate() != cast<CmpInst>(Val: FrontV)->getPredicate())
3945 return false;
3946 if (auto *CI = dyn_cast<CastInst>(Val: V))
3947 if (CI->getSrcTy()->getScalarType() !=
3948 cast<CastInst>(Val: FrontV)->getSrcTy()->getScalarType())
3949 return false;
3950 if (auto *SI = dyn_cast<SelectInst>(Val: V))
3951 if (!isa<VectorType>(Val: SI->getOperand(i_nocapture: 0)->getType()) ||
3952 SI->getOperand(i_nocapture: 0)->getType() !=
3953 cast<SelectInst>(Val: FrontV)->getOperand(i_nocapture: 0)->getType())
3954 return false;
3955 if (isa<CallInst>(Val: V) && !isa<IntrinsicInst>(Val: V))
3956 return false;
3957 auto *II = dyn_cast<IntrinsicInst>(Val: V);
3958 return !II || (isa<IntrinsicInst>(Val: FrontV) &&
3959 II->getIntrinsicID() ==
3960 cast<IntrinsicInst>(Val: FrontV)->getIntrinsicID() &&
3961 !II->hasOperandBundles());
3962 };
3963 if (all_of(Range: drop_begin(RangeOrContainer&: Item), P: CheckLaneIsEquivalentToFirst)) {
3964 // Check the operator is one that we support.
3965 if (isa<BinaryOperator, CmpInst>(Val: FrontV)) {
3966 // We exclude div/rem in case they hit UB from poison lanes.
3967 if (auto *BO = dyn_cast<BinaryOperator>(Val: FrontV);
3968 BO && BO->isIntDivRem())
3969 return false;
3970 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3971 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: 0));
3972 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 1),
3973 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: 1));
3974 continue;
3975 } else if (isa<UnaryOperator, TruncInst, ZExtInst, SExtInst, FPToSIInst,
3976 FPToUIInst, SIToFPInst, UIToFPInst>(Val: FrontV)) {
3977 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3978 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: 0));
3979 continue;
3980 } else if (auto *BitCast = dyn_cast<BitCastInst>(Val: FrontV)) {
3981 auto *BCDstTy = dyn_cast<FixedVectorType>(Val: BitCast->getDestTy());
3982 auto *BCSrcTy = dyn_cast<FixedVectorType>(Val: BitCast->getSrcTy());
3983 if (BCDstTy && BCSrcTy) {
3984 ElementCount DstEC = BCDstTy->getElementCount();
3985 ElementCount SrcEC = BCSrcTy->getElementCount();
3986 if (DstEC == SrcEC) {
3987 // Same element count - simple pass-through.
3988 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3989 Args: &BitCast->getOperandUse(i: 0));
3990 continue;
3991 }
3992 unsigned DstElts = DstEC.getFixedValue();
3993 unsigned SrcElts = SrcEC.getFixedValue();
3994 if (DstElts > SrcElts && DstElts % SrcElts == 0) {
3995 // Widening bitcast (e.g. <2 x i32> -> <4 x i16>). Compress
3996 // consecutive groups of R destination lanes into one source
3997 // lane.
3998 unsigned R = DstElts / SrcElts;
3999 SmallVector<InstLane> NItem;
4000 bool Valid = Item.size() % R == 0;
4001 for (unsigned Idx = 0, E = Item.size(); Valid && Idx < E;
4002 Idx += R) {
4003 auto [V0, L0] = Item[Idx];
4004 if (!V0) {
4005 if (any_of(Range: ArrayRef(Item).slice(N: Idx + 1, M: R - 1),
4006 P: [](InstLane IL) { return IL.first != nullptr; })) {
4007 Valid = false;
4008 break;
4009 }
4010 NItem.push_back(Elt: {nullptr, PoisonMaskElem});
4011 continue;
4012 }
4013 if (L0 % R != 0) {
4014 Valid = false;
4015 break;
4016 }
4017 for (unsigned J = 1; J < R; ++J) {
4018 auto [VJ, LJ] = Item[Idx + J];
4019 if (!VJ || VJ != V0 || LJ != L0 + (int)J) {
4020 Valid = false;
4021 break;
4022 }
4023 }
4024 if (!Valid)
4025 break;
4026 NItem.push_back(Elt: lookThroughShuffles(
4027 V: cast<Operator>(Val: V0)->getOperand(i: 0), Lane: L0 / R));
4028 }
4029 if (Valid) {
4030 TraversedElCountChangingBitcast = true;
4031 Candidates.emplace_back(Args&: NItem, Args: &BitCast->getOperandUse(i: 0));
4032 continue;
4033 }
4034 } else if (SrcElts > DstElts && SrcElts % DstElts == 0) {
4035 // Narrowing bitcast (e.g. <4 x i16> -> <2 x i32>). Expand
4036 // each destination lane into R source lanes.
4037 unsigned R = SrcElts / DstElts;
4038 SmallVector<InstLane> NItem;
4039 for (auto [V, Lane] : Item) {
4040 if (!V) {
4041 NItem.append(NumInputs: R, Elt: {nullptr, PoisonMaskElem});
4042 continue;
4043 }
4044 Value *Op = cast<Operator>(Val: V)->getOperand(i: 0);
4045 for (unsigned J = 0; J < R; ++J)
4046 NItem.push_back(Elt: lookThroughShuffles(V: Op, Lane: Lane * R + J));
4047 }
4048 TraversedElCountChangingBitcast = true;
4049 Candidates.emplace_back(Args&: NItem, Args: &BitCast->getOperandUse(i: 0));
4050 continue;
4051 }
4052 }
4053 } else if (auto *Sel = dyn_cast<SelectInst>(Val: FrontV)) {
4054 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
4055 Args: &Sel->getOperandUse(i: 0));
4056 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 1),
4057 Args: &Sel->getOperandUse(i: 1));
4058 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 2),
4059 Args: &Sel->getOperandUse(i: 2));
4060 continue;
4061 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: FrontV);
4062 II && isTriviallyVectorizable(ID: II->getIntrinsicID()) &&
4063 !II->hasOperandBundles()) {
4064 for (unsigned Op = 0, E = II->getNumOperands() - 1; Op < E; Op++) {
4065 if (isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(), ScalarOpdIdx: Op,
4066 TTI: &TTI)) {
4067 if (!all_of(Range: drop_begin(RangeOrContainer&: Item), P: [Item, Op](InstLane &IL) {
4068 Value *FrontV = Item.front().first;
4069 Value *V = IL.first;
4070 return !V || (cast<Instruction>(Val: V)->getOperand(i: Op) ==
4071 cast<Instruction>(Val: FrontV)->getOperand(i: Op));
4072 }))
4073 return false;
4074 continue;
4075 }
4076 Candidates.emplace_back(
4077 Args: generateInstLaneVectorFromOperand(Item, Op),
4078 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: Op));
4079 }
4080 continue;
4081 }
4082 }
4083
4084 if (isFreeConcat(Item, CostKind, TTI)) {
4085 ConcatLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
4086 continue;
4087 }
4088
4089 return false;
4090 }
4091
4092 if (NumVisited <= 1)
4093 return false;
4094
4095 // If the only non-leaf node traversed was a single bitcast that changes
4096 // element count, the fold would just commute the bitcast and shuffle.
4097 // foldBitcastShuffle does the reverse transform, causing an infinite loop.
4098 if (NumVisited == 2 && TraversedElCountChangingBitcast)
4099 return false;
4100
4101 LLVM_DEBUG(dbgs() << "Found a superfluous identity shuffle: " << I << "\n");
4102
4103 // If we got this far, we know the shuffles are superfluous and can be
4104 // removed. Scan through again and generate the new tree of instructions.
4105 Builder.SetInsertPoint(&I);
4106 Value *V =
4107 generateNewInstTree(Item: Start, From: &*I.use_begin(), IdentityLeafs, SplatLeafs,
4108 ConcatLeafs, Builder, WorkList&: Worklist, TTI: &TTI);
4109 replaceValue(Old&: I, New&: *V);
4110 return true;
4111}
4112
4113/// Given a commutative reduction, the order of the input lanes does not alter
4114/// the results. We can use this to remove certain shuffles feeding the
4115/// reduction, removing the need to shuffle at all.
4116bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
4117 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
4118 if (!II)
4119 return false;
4120 switch (II->getIntrinsicID()) {
4121 case Intrinsic::vector_reduce_add:
4122 case Intrinsic::vector_reduce_mul:
4123 case Intrinsic::vector_reduce_and:
4124 case Intrinsic::vector_reduce_or:
4125 case Intrinsic::vector_reduce_xor:
4126 case Intrinsic::vector_reduce_smin:
4127 case Intrinsic::vector_reduce_smax:
4128 case Intrinsic::vector_reduce_umin:
4129 case Intrinsic::vector_reduce_umax:
4130 break;
4131 default:
4132 return false;
4133 }
4134
4135 // Find all the inputs when looking through operations that do not alter the
4136 // lane order (binops, for example). Currently we look for a single shuffle,
4137 // and can ignore splat values.
4138 std::queue<Value *> Worklist;
4139 SmallPtrSet<Value *, 4> Visited;
4140 ShuffleVectorInst *Shuffle = nullptr;
4141 if (auto *Op = dyn_cast<Instruction>(Val: I.getOperand(i: 0)))
4142 Worklist.push(x: Op);
4143
4144 while (!Worklist.empty()) {
4145 Value *CV = Worklist.front();
4146 Worklist.pop();
4147 if (Visited.contains(Ptr: CV))
4148 continue;
4149
4150 // Splats don't change the order, so can be safely ignored.
4151 if (isSplatValue(V: CV))
4152 continue;
4153
4154 Visited.insert(Ptr: CV);
4155
4156 if (auto *CI = dyn_cast<Instruction>(Val: CV)) {
4157 if (CI->isBinaryOp()) {
4158 for (auto *Op : CI->operand_values())
4159 Worklist.push(x: Op);
4160 continue;
4161 } else if (auto *SV = dyn_cast<ShuffleVectorInst>(Val: CI)) {
4162 if (Shuffle && Shuffle != SV)
4163 return false;
4164 Shuffle = SV;
4165 continue;
4166 }
4167 }
4168
4169 // Anything else is currently an unknown node.
4170 return false;
4171 }
4172
4173 if (!Shuffle)
4174 return false;
4175
4176 // Check all uses of the binary ops and shuffles are also included in the
4177 // lane-invariant operations (Visited should be the list of lanewise
4178 // instructions, including the shuffle that we found).
4179 for (auto *V : Visited)
4180 for (auto *U : V->users())
4181 if (!Visited.contains(Ptr: U) && U != &I)
4182 return false;
4183
4184 FixedVectorType *VecType =
4185 dyn_cast<FixedVectorType>(Val: II->getOperand(i_nocapture: 0)->getType());
4186 if (!VecType)
4187 return false;
4188 FixedVectorType *ShuffleInputType =
4189 dyn_cast<FixedVectorType>(Val: Shuffle->getOperand(i_nocapture: 0)->getType());
4190 if (!ShuffleInputType)
4191 return false;
4192 unsigned NumInputElts = ShuffleInputType->getNumElements();
4193
4194 // Find the mask from sorting the lanes into order. This is most likely to
4195 // become a identity or concat mask. Undef elements are pushed to the end.
4196 SmallVector<int> ConcatMask;
4197 Shuffle->getShuffleMask(Result&: ConcatMask);
4198 sort(C&: ConcatMask, Comp: [](int X, int Y) { return (unsigned)X < (unsigned)Y; });
4199 bool UsesSecondVec =
4200 any_of(Range&: ConcatMask, P: [&](int M) { return M >= (int)NumInputElts; });
4201
4202 InstructionCost OldCost = TTI.getShuffleCost(
4203 Kind: UsesSecondVec ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc, DstTy: VecType,
4204 SrcTy: ShuffleInputType, CostKind, Mask: Shuffle->getShuffleMask());
4205 InstructionCost NewCost = TTI.getShuffleCost(
4206 Kind: UsesSecondVec ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc, DstTy: VecType,
4207 SrcTy: ShuffleInputType, CostKind, Mask: ConcatMask);
4208
4209 LLVM_DEBUG(dbgs() << "Found a reduction feeding from a shuffle: " << *Shuffle
4210 << "\n");
4211 LLVM_DEBUG(dbgs() << " OldCost: " << OldCost << " vs NewCost: " << NewCost
4212 << "\n");
4213 bool MadeChanges = false;
4214 if (NewCost < OldCost) {
4215 Builder.SetInsertPoint(Shuffle);
4216 Value *NewShuffle = Builder.CreateShuffleVector(
4217 V1: Shuffle->getOperand(i_nocapture: 0), V2: Shuffle->getOperand(i_nocapture: 1), Mask: ConcatMask);
4218 LLVM_DEBUG(dbgs() << "Created new shuffle: " << *NewShuffle << "\n");
4219 replaceValue(Old&: *Shuffle, New&: *NewShuffle);
4220 return true;
4221 }
4222
4223 // See if we can re-use foldSelectShuffle, getting it to reduce the size of
4224 // the shuffle into a nicer order, as it can ignore the order of the shuffles.
4225 MadeChanges |= foldSelectShuffle(I&: *Shuffle, FromReduction: true);
4226 return MadeChanges;
4227}
4228
4229/// Try to fold a chain of shuffles and ops feeding extractelement(..., 0)
4230/// into llvm.vector.reduce.*, by tracking which lanes contribute to the
4231/// extracted lane and reducing the widest vector whose lanes each contribute
4232/// once.
4233///
4234/// For example:
4235///
4236/// %lo = shufflevector <4 x i32> %a, poison, <2 x i32> <i32 0, i32 1>
4237/// %hi = shufflevector <4 x i32> %a, poison, <2 x i32> <i32 2, i32 3>
4238/// %s = add <2 x i32> %lo, %hi
4239/// %sh = shufflevector <2 x i32> %s, poison, <2 x i32> <i32 1, i32 poison>
4240/// %r = add <2 x i32> %s, %sh
4241/// %e = extractelement <2 x i32> %r, i64 0
4242///
4243/// transforms to:
4244///
4245/// %e = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
4246bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
4247 Value *VecOpEE;
4248 if (!match(V: &I, P: m_ExtractElt(Val: m_Value(V&: VecOpEE), Idx: m_Zero())))
4249 return false;
4250
4251 auto *FVT = dyn_cast<FixedVectorType>(Val: VecOpEE->getType());
4252 if (!FVT)
4253 return false;
4254
4255 if (FVT->getNumElements() < 2)
4256 return false;
4257
4258 std::optional<Instruction::BinaryOps> CommonBinOp;
4259 std::optional<Intrinsic::ID> CommonCallOp;
4260
4261 if (auto *BO = dyn_cast<BinaryOperator>(Val: VecOpEE)) {
4262 if (!getReductionForBinop(Opc: BO->getOpcode()))
4263 return false;
4264 CommonBinOp = BO->getOpcode();
4265 } else if (auto *MMI = dyn_cast<MinMaxIntrinsic>(Val: VecOpEE)) {
4266 CommonCallOp = MMI->getIntrinsicID();
4267 } else {
4268 return false;
4269 }
4270
4271 // For floating-point reductions, track FMF intersection across all binops.
4272 FastMathFlags CommonFMF;
4273 bool IsFloatReduction = false;
4274
4275 // A chain node is one we walk through, either a matching-opcode binop/min-max
4276 // or a single-source shuffle. Anything else is a leaf source.
4277 auto IsChainNode = [&](Value *V) {
4278 if (auto *BO = dyn_cast<BinaryOperator>(Val: V))
4279 return CommonBinOp && BO->getOpcode() == *CommonBinOp;
4280 if (auto *MMI = dyn_cast<MinMaxIntrinsic>(Val: V))
4281 return CommonCallOp && MMI->getIntrinsicID() == *CommonCallOp;
4282 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V))
4283 return isa<PoisonValue>(Val: SVI->getOperand(i_nocapture: 1));
4284 return false;
4285 };
4286
4287 // Collect the chain, building Nodes in postorder. Bail if the chain is empty
4288 // or exceeds MaxChainNodes.
4289 constexpr unsigned MaxChainNodes = 32;
4290 SmallSetVector<Value *, 16> Nodes;
4291 SmallSetVector<Value *, 4> Sources;
4292 unsigned NumVisited = 0;
4293 auto AddSource = [&](Value *V) {
4294 if (!isa<FixedVectorType>(Val: V->getType()))
4295 return false;
4296 Sources.insert(X: V);
4297 return true;
4298 };
4299 auto Walk = [&](Value *V, auto &&Walk) -> bool {
4300 if (Nodes.contains(key: V) || Sources.contains(key: V))
4301 return true;
4302 if (++NumVisited > MaxChainNodes)
4303 return false;
4304 if (!IsChainNode(V))
4305 return AddSource(V);
4306 // Chain shuffles always have poison as op1, so only op0 matters.
4307 auto *U = cast<Instruction>(Val: V);
4308 unsigned NumOps = isa<ShuffleVectorInst>(Val: U) ? 1 : 2;
4309 for (unsigned I = 0; I != NumOps; ++I)
4310 if (!Walk(U->getOperand(i: I), Walk))
4311 return false;
4312 if (isa<ShuffleVectorInst>(Val: U) || Nodes.contains(key: U->getOperand(i: 0)) ||
4313 Nodes.contains(key: U->getOperand(i: 1))) {
4314 Nodes.insert(X: V);
4315 return true;
4316 }
4317 // Both operands are leaves so treat this binop as a source rather than
4318 // walking into it.
4319 return AddSource(V);
4320 };
4321 if (!Walk(VecOpEE, Walk) || Nodes.empty())
4322 return false;
4323
4324 bool IsIdempotent =
4325 CommonCallOp || (CommonBinOp && Instruction::isIdempotent(Opcode: *CommonBinOp));
4326
4327 // For FP reductions, require reassoc on every binop and collect FMF.
4328 for (Value *V : Nodes) {
4329 auto *BinOp = dyn_cast<BinaryOperator>(Val: V);
4330 if (!BinOp || !BinOp->getType()->isFPOrFPVectorTy())
4331 continue;
4332 if (!BinOp->hasAllowReassoc())
4333 return false;
4334 if (!IsFloatReduction) {
4335 CommonFMF = BinOp->getFastMathFlags();
4336 IsFloatReduction = true;
4337 } else {
4338 CommonFMF &= BinOp->getFastMathFlags();
4339 }
4340 }
4341
4342 // Top-down demanded elements. For each chain value, track which lanes feed
4343 // the extracted lane 0 and which feed it more than once. Reverse postorder
4344 // visits every use before its value. A binop forwards its demand to both
4345 // operands and a shuffle follows its mask back to the source lane.
4346 struct Demand {
4347 APInt Lanes;
4348 APInt Duplicates;
4349 };
4350 DenseMap<Value *, Demand> Demands;
4351 auto DemandOf = [&](Value *V) -> Demand & {
4352 unsigned N = cast<FixedVectorType>(Val: V->getType())->getNumElements();
4353 Demand &D = Demands[V];
4354 if (D.Lanes.getBitWidth() != N)
4355 D.Lanes = D.Duplicates = APInt::getZero(numBits: N);
4356 return D;
4357 };
4358 DemandOf(VecOpEE).Lanes.setBit(0);
4359 for (Value *V : reverse(C&: Nodes)) {
4360 Demand DV = Demands.lookup(Val: V);
4361 if (DV.Lanes.isZero())
4362 continue;
4363 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V)) {
4364 ArrayRef<int> Mask = SVI->getShuffleMask();
4365 Demand &DS = DemandOf(SVI->getOperand(i_nocapture: 0));
4366 for (unsigned I = 0, E = Mask.size(); I != E; ++I) {
4367 // Skip lanes that are undemanded or map to poison.
4368 if (!DV.Lanes[I] || Mask[I] < 0 ||
4369 (unsigned)Mask[I] >= DS.Lanes.getBitWidth())
4370 continue;
4371 if (DS.Lanes[Mask[I]] || DV.Duplicates[I])
4372 DS.Duplicates.setBit(Mask[I]);
4373 DS.Lanes.setBit(Mask[I]);
4374 }
4375 } else {
4376 auto *U = cast<User>(Val: V);
4377 for (Value *Op : {U->getOperand(i: 0), U->getOperand(i: 1)}) {
4378 Demand &DOp = DemandOf(Op);
4379 // Lanes demanded through more than one path accumulate in Duplicates.
4380 DOp.Duplicates |= DV.Duplicates | (DOp.Lanes & DV.Lanes);
4381 DOp.Lanes |= DV.Lanes;
4382 }
4383 }
4384 }
4385
4386 // Reducing V replaces the entire chain, so every contribution to the result
4387 // must flow through V. Reject if anything above V reads outside the chain.
4388 auto CoversChain = [&](Value *V) {
4389 SmallVector<Value *, 8> Worklist(1, VecOpEE);
4390 SmallPtrSet<Value *, 8> Seen;
4391 Seen.insert(Ptr: VecOpEE);
4392 while (!Worklist.empty()) {
4393 auto *U = cast<Instruction>(Val: Worklist.pop_back_val());
4394 unsigned NumOps = isa<ShuffleVectorInst>(Val: U) ? 1 : 2;
4395 for (unsigned I = 0; I != NumOps; ++I) {
4396 Value *Op = U->getOperand(i: I);
4397 if (Op == V || !Seen.insert(Ptr: Op).second)
4398 continue;
4399 if (!Nodes.contains(key: Op))
4400 return false;
4401 Worklist.push_back(Elt: Op);
4402 }
4403 }
4404 return true;
4405 };
4406
4407 // Reduce a single cleanly demanded source if there is one, otherwise the
4408 // deepest intermediate that covers the chain.
4409 struct ReductionCut {
4410 Value *Src;
4411 APInt Elts;
4412 };
4413 std::optional<ReductionCut> Cut;
4414 for (Value *S : Sources) {
4415 auto It = Demands.find(Val: S);
4416 if (It == Demands.end() || It->second.Lanes.isZero())
4417 continue;
4418 if (!IsIdempotent && !It->second.Duplicates.isZero()) {
4419 Cut.reset();
4420 break;
4421 }
4422 if (!Cut) {
4423 Cut = ReductionCut{.Src: S, .Elts: It->second.Lanes};
4424 continue;
4425 }
4426 if (!isEquivBitcast(X: Cut->Src, Y: S)) {
4427 Cut.reset();
4428 break;
4429 }
4430 if (!IsIdempotent && !(Cut->Elts & It->second.Lanes).isZero()) {
4431 Cut.reset();
4432 break;
4433 }
4434 Cut->Elts |= It->second.Lanes;
4435 }
4436 if (!Cut) {
4437 for (Value *V : Nodes) {
4438 if (!isa<BinaryOperator>(Val: V) && !isa<MinMaxIntrinsic>(Val: V))
4439 continue;
4440 auto It = Demands.find(Val: V);
4441 if (It == Demands.end() || !It->second.Lanes.isAllOnes())
4442 continue;
4443 if (!IsIdempotent && !It->second.Duplicates.isZero())
4444 continue;
4445 if (!CoversChain(V))
4446 continue;
4447 Cut = ReductionCut{.Src: V, .Elts: It->second.Lanes};
4448 break;
4449 }
4450 }
4451 // Reducing one lane is just an extract and can refold forever.
4452 if (!Cut || Cut->Elts.popcount() < 2)
4453 return false;
4454
4455 Intrinsic::ID ReducedOp =
4456 (CommonCallOp ? getMinMaxReductionIntrinsicID(IID: *CommonCallOp)
4457 : getReductionForBinop(Opc: *CommonBinOp));
4458 if (!ReducedOp)
4459 return false;
4460
4461 InstructionCost OrigCost = 0;
4462 for (Value *V : Nodes)
4463 OrigCost += TTI.getInstructionCost(U: cast<Instruction>(Val: V), CostKind);
4464
4465 auto *SrcVT = cast<FixedVectorType>(Val: Cut->Src->getType());
4466 bool IsPartialReduction = !Cut->Elts.isAllOnes();
4467 FixedVectorType *ReduceVecTy =
4468 IsPartialReduction
4469 ? FixedVectorType::get(ElementType: FVT->getElementType(), NumElts: Cut->Elts.popcount())
4470 : SrcVT;
4471
4472 SmallVector<int> ExtractMask;
4473 InstructionCost NewCost = 0;
4474 if (IsPartialReduction) {
4475 for (unsigned I = 0, E = Cut->Elts.getBitWidth(); I != E; ++I)
4476 if (Cut->Elts[I])
4477 ExtractMask.push_back(Elt: I);
4478 unsigned SubIdx = 0, SubLen;
4479 auto SK = Cut->Elts.isShiftedMask(MaskIdx&: SubIdx, MaskLen&: SubLen)
4480 ? TargetTransformInfo::SK_ExtractSubvector
4481 : TargetTransformInfo::SK_PermuteSingleSrc;
4482 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: ReduceVecTy, SrcTy: SrcVT, CostKind, Mask: ExtractMask,
4483 Index: SubIdx, SubTp: ReduceVecTy);
4484 }
4485
4486 IntrinsicCostAttributes ICA(
4487 ReducedOp, ReduceVecTy->getElementType(),
4488 IsFloatReduction
4489 ? SmallVector<Type *, 2>{ReduceVecTy->getElementType(), ReduceVecTy}
4490 : SmallVector<Type *, 2>{ReduceVecTy},
4491 IsFloatReduction ? CommonFMF : FastMathFlags());
4492 NewCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
4493
4494 LLVM_DEBUG(dbgs() << "Found reduction shuffle chain: " << I << "\n OldCost : "
4495 << OrigCost << " vs NewCost: " << NewCost << "\n");
4496
4497 if (!OrigCost.isValid() || !NewCost.isValid())
4498 return false;
4499
4500 if (VecOpEE->hasOneUse() ? (NewCost > OrigCost) : (NewCost >= OrigCost))
4501 return false;
4502
4503 Value *ReduceInput = Cut->Src;
4504 if (IsPartialReduction)
4505 ReduceInput = Builder.CreateShuffleVector(V: Cut->Src, Mask: ExtractMask);
4506
4507 Value *ReducedResult;
4508 if (IsFloatReduction) {
4509 Value *Identity = ConstantExpr::getBinOpIdentity(
4510 Opcode: *CommonBinOp, Ty: ReduceVecTy->getElementType(), /*AllowRHSConstant=*/false,
4511 NSZ: CommonFMF.noSignedZeros());
4512 ReducedResult = Builder.CreateIntrinsic(ID: ReducedOp, OverloadTypes: {ReduceVecTy},
4513 Args: {Identity, ReduceInput}, FMFSource: CommonFMF);
4514 } else {
4515 ReducedResult =
4516 Builder.CreateIntrinsic(ID: ReducedOp, OverloadTypes: {ReduceVecTy}, Args: {ReduceInput});
4517 }
4518 replaceValue(Old&: I, New&: *ReducedResult);
4519
4520 return true;
4521}
4522
4523/// Determine if its more efficient to fold:
4524/// reduce(trunc(x)) -> trunc(reduce(x)).
4525/// reduce(sext(x)) -> sext(reduce(x)).
4526/// reduce(zext(x)) -> zext(reduce(x)).
4527bool VectorCombine::foldCastFromReductions(Instruction &I) {
4528 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
4529 if (!II)
4530 return false;
4531
4532 bool TruncOnly = false;
4533 Intrinsic::ID IID = II->getIntrinsicID();
4534 switch (IID) {
4535 case Intrinsic::vector_reduce_add:
4536 case Intrinsic::vector_reduce_mul:
4537 TruncOnly = true;
4538 break;
4539 case Intrinsic::vector_reduce_and:
4540 case Intrinsic::vector_reduce_or:
4541 case Intrinsic::vector_reduce_xor:
4542 break;
4543 default:
4544 return false;
4545 }
4546
4547 unsigned ReductionOpc = getArithmeticReductionInstruction(RdxID: IID);
4548 Value *ReductionSrc = I.getOperand(i: 0);
4549
4550 Value *Src;
4551 if (!match(V: ReductionSrc, P: m_OneUse(SubPattern: m_Trunc(Op: m_Value(V&: Src)))) &&
4552 (TruncOnly || !match(V: ReductionSrc, P: m_OneUse(SubPattern: m_ZExtOrSExt(Op: m_Value(V&: Src))))))
4553 return false;
4554
4555 auto CastOpc =
4556 (Instruction::CastOps)cast<Instruction>(Val: ReductionSrc)->getOpcode();
4557
4558 auto *SrcTy = cast<VectorType>(Val: Src->getType());
4559 auto *ReductionSrcTy = cast<VectorType>(Val: ReductionSrc->getType());
4560 Type *ResultTy = I.getType();
4561
4562 InstructionCost OldCost = TTI.getArithmeticReductionCost(
4563 Opcode: ReductionOpc, Ty: ReductionSrcTy, FMF: std::nullopt, CostKind);
4564 OldCost += TTI.getCastInstrCost(Opcode: CastOpc, Dst: ReductionSrcTy, Src: SrcTy,
4565 CCH: TTI::CastContextHint::None, CostKind,
4566 I: cast<CastInst>(Val: ReductionSrc));
4567 InstructionCost NewCost =
4568 TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: SrcTy, FMF: std::nullopt,
4569 CostKind) +
4570 TTI.getCastInstrCost(Opcode: CastOpc, Dst: ResultTy, Src: ReductionSrcTy->getScalarType(),
4571 CCH: TTI::CastContextHint::None, CostKind);
4572
4573 if (OldCost <= NewCost || !NewCost.isValid())
4574 return false;
4575
4576 Value *NewReduction = Builder.CreateIntrinsic(RetTy: SrcTy->getScalarType(),
4577 ID: II->getIntrinsicID(), Args: {Src});
4578 Value *NewCast = Builder.CreateCast(Op: CastOpc, V: NewReduction, DestTy: ResultTy);
4579 replaceValue(Old&: I, New&: *NewCast);
4580 return true;
4581}
4582
4583/// Fold:
4584/// icmp pred (reduce.{add,or,and,umax,umin}(signbit_extract(x))), C
4585/// into:
4586/// icmp sgt/slt (reduce.{or,umax,and,umin}(x)), -1/0
4587///
4588/// Sign-bit reductions produce values with known semantics:
4589/// - reduce.{or,umax}: 0 if no element is negative, 1 if any is
4590/// - reduce.{and,umin}: 1 if all elements are negative, 0 if any isn't
4591/// - reduce.add: count of negative elements (0 to NumElts)
4592///
4593/// Both lshr and ashr are supported:
4594/// - lshr produces 0 or 1, so reduce.add range is [0, N]
4595/// - ashr produces 0 or -1, so reduce.add range is [-N, 0]
4596///
4597/// The fold generalizes to multiple source vectors combined with the same
4598/// operation as the reduction. For example:
4599/// reduce.or(or(shr A, shr B)) conceptually extends the vector
4600/// For reduce.add, this changes the count to M*N where M is the number of
4601/// source vectors.
4602///
4603/// We transform to a direct sign check on the original vector using
4604/// reduce.{or,umax} or reduce.{and,umin}.
4605///
4606/// In spirit, it's similar to foldSignBitCheck in InstCombine.
4607bool VectorCombine::foldSignBitReductionCmp(Instruction &I) {
4608 CmpPredicate Pred;
4609 IntrinsicInst *ReduceOp;
4610 const APInt *CmpVal;
4611 if (!match(V: &I,
4612 P: m_ICmp(Pred, L: m_OneUse(SubPattern: m_AnyIntrinsic(I&: ReduceOp)), R: m_APInt(Res&: CmpVal))))
4613 return false;
4614
4615 Intrinsic::ID OrigIID = ReduceOp->getIntrinsicID();
4616 switch (OrigIID) {
4617 case Intrinsic::vector_reduce_or:
4618 case Intrinsic::vector_reduce_umax:
4619 case Intrinsic::vector_reduce_and:
4620 case Intrinsic::vector_reduce_umin:
4621 case Intrinsic::vector_reduce_add:
4622 break;
4623 default:
4624 return false;
4625 }
4626
4627 Value *ReductionSrc = ReduceOp->getArgOperand(i: 0);
4628 auto *VecTy = dyn_cast<FixedVectorType>(Val: ReductionSrc->getType());
4629 if (!VecTy)
4630 return false;
4631
4632 unsigned BitWidth = VecTy->getScalarSizeInBits();
4633 if (BitWidth == 1)
4634 return false;
4635
4636 unsigned NumElts = VecTy->getNumElements();
4637
4638 // Determine the expected tree opcode for multi-vector patterns.
4639 // The tree opcode must match the reduction's underlying operation.
4640 //
4641 // TODO: for pairs of equivalent operators, we should match both,
4642 // not only the most common.
4643 Instruction::BinaryOps TreeOpcode;
4644 switch (OrigIID) {
4645 case Intrinsic::vector_reduce_or:
4646 case Intrinsic::vector_reduce_umax:
4647 TreeOpcode = Instruction::Or;
4648 break;
4649 case Intrinsic::vector_reduce_and:
4650 case Intrinsic::vector_reduce_umin:
4651 TreeOpcode = Instruction::And;
4652 break;
4653 case Intrinsic::vector_reduce_add:
4654 TreeOpcode = Instruction::Add;
4655 break;
4656 default:
4657 llvm_unreachable("Unexpected intrinsic");
4658 }
4659
4660 // Collect sign-bit extraction leaves from an associative tree of TreeOpcode.
4661 // The tree conceptually extends the vector being reduced.
4662 SmallVector<Value *, 8> Worklist;
4663 SmallVector<Value *, 8> Sources; // Original vectors (X in shr X, BW-1)
4664 Worklist.push_back(Elt: ReductionSrc);
4665 std::optional<bool> IsAShr;
4666 constexpr unsigned MaxSources = 8;
4667
4668 // Calculate old cost: all shifts + tree ops + reduction
4669 InstructionCost OldCost = TTI.getInstructionCost(U: ReduceOp, CostKind);
4670
4671 while (!Worklist.empty() && Worklist.size() <= MaxSources &&
4672 Sources.size() <= MaxSources) {
4673 Value *V = Worklist.pop_back_val();
4674
4675 // Try to match sign-bit extraction: shr X, (bitwidth-1)
4676 Value *X;
4677 if (match(V, P: m_OneUse(SubPattern: m_Shr(L: m_Value(V&: X), R: m_SpecificInt(V: BitWidth - 1))))) {
4678 auto *Shr = cast<Instruction>(Val: V);
4679
4680 // All shifts must be the same type (all lshr or all ashr)
4681 bool ThisIsAShr = Shr->getOpcode() == Instruction::AShr;
4682 if (!IsAShr)
4683 IsAShr = ThisIsAShr;
4684 else if (*IsAShr != ThisIsAShr)
4685 return false;
4686
4687 Sources.push_back(Elt: X);
4688
4689 // As part of the fold, we remove all of the shifts, so we need to keep
4690 // track of their costs.
4691 OldCost += TTI.getInstructionCost(U: Shr, CostKind);
4692
4693 continue;
4694 }
4695
4696 // Try to extend through a tree node of the expected opcode
4697 Value *A, *B;
4698 if (!match(V, P: m_OneUse(SubPattern: m_BinOp(Opcode: TreeOpcode, L: m_Value(V&: A), R: m_Value(V&: B)))))
4699 return false;
4700
4701 // We are potentially replacing these operations as well, so we add them
4702 // to the costs.
4703 OldCost += TTI.getInstructionCost(U: cast<Instruction>(Val: V), CostKind);
4704
4705 Worklist.push_back(Elt: A);
4706 Worklist.push_back(Elt: B);
4707 }
4708
4709 // Must have at least one source and not exceed limit
4710 if (Sources.empty() || Sources.size() > MaxSources ||
4711 Worklist.size() > MaxSources || !IsAShr)
4712 return false;
4713
4714 unsigned NumSources = Sources.size();
4715
4716 // For reduce.add, the total count must fit as a signed integer.
4717 // Range is [0, M*N] for lshr or [-M*N, 0] for ashr.
4718 if (OrigIID == Intrinsic::vector_reduce_add &&
4719 !isIntN(N: BitWidth, x: NumSources * NumElts))
4720 return false;
4721
4722 // Compute the boundary value when all elements are negative:
4723 // - Per-element contribution: 1 for lshr, -1 for ashr
4724 // - For add: M*N (total elements across all sources); for others: just 1
4725 unsigned Count =
4726 (OrigIID == Intrinsic::vector_reduce_add) ? NumSources * NumElts : 1;
4727 APInt NegativeVal(CmpVal->getBitWidth(), Count);
4728 if (*IsAShr)
4729 NegativeVal.negate();
4730
4731 // Range is [min(0, AllNegVal), max(0, AllNegVal)]
4732 APInt Zero = APInt::getZero(numBits: CmpVal->getBitWidth());
4733 APInt RangeLow = APIntOps::smin(A: Zero, B: NegativeVal);
4734 APInt RangeHigh = APIntOps::smax(A: Zero, B: NegativeVal);
4735
4736 // Determine comparison semantics:
4737 // - IsEq: true for equality test, false for inequality
4738 // - TestsNegative: true if testing against AllNegVal, false for zero
4739 //
4740 // In addition to EQ/NE against 0 or AllNegVal, we support inequalities
4741 // that fold to boundary tests given the narrow value range:
4742 // < RangeHigh -> != RangeHigh
4743 // > RangeHigh-1 -> == RangeHigh
4744 // > RangeLow -> != RangeLow
4745 // < RangeLow+1 -> == RangeLow
4746 //
4747 // For inequalities, we work with signed predicates only. Unsigned predicates
4748 // are canonicalized to signed when the range is non-negative (where they are
4749 // equivalent). When the range includes negative values, unsigned predicates
4750 // would have different semantics due to wrap-around, so we reject them.
4751 if (!ICmpInst::isEquality(P: Pred) && !ICmpInst::isSigned(Pred)) {
4752 if (RangeLow.isNegative())
4753 return false;
4754 Pred = ICmpInst::getSignedPredicate(Pred);
4755 }
4756
4757 bool IsEq;
4758 bool TestsNegative;
4759 if (ICmpInst::isEquality(P: Pred)) {
4760 if (CmpVal->isZero()) {
4761 TestsNegative = false;
4762 } else if (*CmpVal == NegativeVal) {
4763 TestsNegative = true;
4764 } else {
4765 return false;
4766 }
4767 IsEq = Pred == ICmpInst::ICMP_EQ;
4768 } else if (Pred == ICmpInst::ICMP_SLT && *CmpVal == RangeHigh) {
4769 IsEq = false;
4770 TestsNegative = (RangeHigh == NegativeVal);
4771 } else if (Pred == ICmpInst::ICMP_SGT && *CmpVal == RangeHigh - 1) {
4772 IsEq = true;
4773 TestsNegative = (RangeHigh == NegativeVal);
4774 } else if (Pred == ICmpInst::ICMP_SGT && *CmpVal == RangeLow) {
4775 IsEq = false;
4776 TestsNegative = (RangeLow == NegativeVal);
4777 } else if (Pred == ICmpInst::ICMP_SLT && *CmpVal == RangeLow + 1) {
4778 IsEq = true;
4779 TestsNegative = (RangeLow == NegativeVal);
4780 } else {
4781 return false;
4782 }
4783
4784 // For this fold we support four types of checks:
4785 //
4786 // 1. All lanes are negative - AllNeg
4787 // 2. All lanes are non-negative - AllNonNeg
4788 // 3. At least one negative lane - AnyNeg
4789 // 4. At least one non-negative lane - AnyNonNeg
4790 //
4791 // For each case, we can generate the following code:
4792 //
4793 // 1. AllNeg - reduce.and/umin(X) < 0
4794 // 2. AllNonNeg - reduce.or/umax(X) > -1
4795 // 3. AnyNeg - reduce.or/umax(X) < 0
4796 // 4. AnyNonNeg - reduce.and/umin(X) > -1
4797 //
4798 // The table below shows the aggregation of all supported cases
4799 // using these four cases.
4800 //
4801 // Reduction | == 0 | != 0 | == MAX | != MAX
4802 // ------------+-----------+-----------+-----------+-----------
4803 // or/umax | AllNonNeg | AnyNeg | AnyNeg | AllNonNeg
4804 // and/umin | AnyNonNeg | AllNeg | AllNeg | AnyNonNeg
4805 // add | AllNonNeg | AnyNeg | AllNeg | AnyNonNeg
4806 //
4807 // NOTE: MAX = 1 for or/and/umax/umin, and the vector size N for add
4808 //
4809 // For easier codegen and check inversion, we use the following encoding:
4810 //
4811 // 1. Bit-3 === requires or/umax (1) or and/umin (0) check
4812 // 2. Bit-2 === checks < 0 (1) or > -1 (0)
4813 // 3. Bit-1 === universal (1) or existential (0) check
4814 //
4815 // AnyNeg = 0b110: uses or/umax, checks negative, any-check
4816 // AllNonNeg = 0b101: uses or/umax, checks non-neg, all-check
4817 // AnyNonNeg = 0b000: uses and/umin, checks non-neg, any-check
4818 // AllNeg = 0b011: uses and/umin, checks negative, all-check
4819 //
4820 // XOR with 0b011 inverts the check (swaps all/any and neg/non-neg).
4821 //
4822 enum CheckKind : unsigned {
4823 AnyNonNeg = 0b000,
4824 AllNeg = 0b011,
4825 AllNonNeg = 0b101,
4826 AnyNeg = 0b110,
4827 };
4828 // Return true if we fold this check into or/umax and false for and/umin
4829 auto RequiresOr = [](CheckKind C) -> bool { return C & 0b100; };
4830 // Return true if we should check if result is negative and false otherwise
4831 auto IsNegativeCheck = [](CheckKind C) -> bool { return C & 0b010; };
4832 // Logically invert the check
4833 auto Invert = [](CheckKind C) { return CheckKind(C ^ 0b011); };
4834
4835 CheckKind Base;
4836 switch (OrigIID) {
4837 case Intrinsic::vector_reduce_or:
4838 case Intrinsic::vector_reduce_umax:
4839 Base = TestsNegative ? AnyNeg : AllNonNeg;
4840 break;
4841 case Intrinsic::vector_reduce_and:
4842 case Intrinsic::vector_reduce_umin:
4843 Base = TestsNegative ? AllNeg : AnyNonNeg;
4844 break;
4845 case Intrinsic::vector_reduce_add:
4846 Base = TestsNegative ? AllNeg : AllNonNeg;
4847 break;
4848 default:
4849 llvm_unreachable("Unexpected intrinsic");
4850 }
4851
4852 CheckKind Check = IsEq ? Base : Invert(Base);
4853
4854 auto PickCheaper = [&](Intrinsic::ID Arith, Intrinsic::ID MinMax) {
4855 InstructionCost ArithCost =
4856 TTI.getArithmeticReductionCost(Opcode: getArithmeticReductionInstruction(RdxID: Arith),
4857 Ty: VecTy, FMF: std::nullopt, CostKind);
4858 InstructionCost MinMaxCost =
4859 TTI.getMinMaxReductionCost(IID: getMinMaxReductionIntrinsicOp(RdxID: MinMax), Ty: VecTy,
4860 FMF: FastMathFlags(), CostKind);
4861 return ArithCost <= MinMaxCost ? std::make_pair(x&: Arith, y&: ArithCost)
4862 : std::make_pair(x&: MinMax, y&: MinMaxCost);
4863 };
4864
4865 // Choose output reduction based on encoding's MSB
4866 auto [NewIID, NewCost] = RequiresOr(Check)
4867 ? PickCheaper(Intrinsic::vector_reduce_or,
4868 Intrinsic::vector_reduce_umax)
4869 : PickCheaper(Intrinsic::vector_reduce_and,
4870 Intrinsic::vector_reduce_umin);
4871
4872 // Add cost of combining multiple sources with or/and
4873 if (NumSources > 1) {
4874 unsigned CombineOpc =
4875 RequiresOr(Check) ? Instruction::Or : Instruction::And;
4876 NewCost += TTI.getArithmeticInstrCost(Opcode: CombineOpc, Ty: VecTy, CostKind) *
4877 (NumSources - 1);
4878 }
4879
4880 LLVM_DEBUG(dbgs() << "Found sign-bit reduction cmp: " << I << "\n OldCost: "
4881 << OldCost << " vs NewCost: " << NewCost << "\n");
4882
4883 if (NewCost > OldCost)
4884 return false;
4885
4886 // Generate the combined input and reduction
4887 Builder.SetInsertPoint(&I);
4888 Type *ScalarTy = VecTy->getScalarType();
4889
4890 Value *Input;
4891 if (NumSources == 1) {
4892 Input = Sources[0];
4893 } else {
4894 // Combine sources with or/and based on check type
4895 Input = RequiresOr(Check) ? Builder.CreateOr(Ops: Sources)
4896 : Builder.CreateAnd(Ops: Sources);
4897 }
4898
4899 Value *NewReduce = Builder.CreateIntrinsic(RetTy: ScalarTy, ID: NewIID, Args: {Input});
4900 Value *NewCmp = IsNegativeCheck(Check) ? Builder.CreateIsNeg(Arg: NewReduce)
4901 : Builder.CreateIsNotNeg(Arg: NewReduce);
4902 replaceValue(Old&: I, New&: *NewCmp);
4903 return true;
4904}
4905
4906/// Fold a zero test of reduce.or or reduce.umax into a boolean reduction.
4907///
4908/// Vectorization may produce IR that compares the result of a scalar reduction
4909/// with zero. Depending on the target, lowering a reduction and a scalar
4910/// comparison separately can cost more than reducing lane-wise comparison
4911/// results. This fold creates the latter form only when it is not costlier.
4912///
4913/// Before:
4914/// %r = call iT @llvm.vector.reduce.or.vNiT(<N x iT> %x)
4915/// %cmp = icmp ne iT %r, 0
4916///
4917/// After:
4918/// %lane.cmp = icmp ne <N x iT> %x, zeroinitializer
4919/// %cmp = call i1 @llvm.vector.reduce.or.vNi1(<N x i1> %lane.cmp)
4920///
4921/// `reduce.or` and `reduce.umax` are non-zero when at least one lane is
4922/// non-zero. Therefore, `icmp ne` uses the existential `reduce.or` test.
4923/// Conversely, `icmp eq` must check that every lane is zero, so it uses the
4924/// universal `reduce.and` test.
4925///
4926/// Before:
4927/// %r = call iT @llvm.vector.reduce.umax.vNiT(<N x iT> %x)
4928/// %cmp = icmp eq iT %r, 0
4929///
4930/// After:
4931/// %lane.cmp = icmp eq <N x iT> %x, zeroinitializer
4932/// %cmp = call i1 @llvm.vector.reduce.and.vNi1(<N x i1> %lane.cmp)
4933bool VectorCombine::foldReductionZeroTest(Instruction &I) {
4934 CmpPredicate Pred;
4935 Value *Op;
4936
4937 if (!match(V: &I, P: m_c_ICmp(Pred, L: m_Value(V&: Op), R: m_Zero())) ||
4938 !ICmpInst::isEquality(P: Pred))
4939 return false;
4940
4941 auto *II = dyn_cast<IntrinsicInst>(Val: Op);
4942 if (!II || !II->hasOneUse())
4943 return false;
4944
4945 auto ReduceID = II->getIntrinsicID();
4946 if (ReduceID != Intrinsic::vector_reduce_or &&
4947 ReduceID != Intrinsic::vector_reduce_umax)
4948 return false;
4949
4950 Value *Vec = II->getArgOperand(i: 0);
4951 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
4952 if (!VecTy || !VecTy->getElementType()->isIntegerTy())
4953 return false;
4954
4955 // Map the scalar zero test to an any-lane or all-lane boolean reduction.
4956 Intrinsic::ID NewIID = (Pred == ICmpInst::ICMP_NE)
4957 ? Intrinsic::vector_reduce_or
4958 : Intrinsic::vector_reduce_and;
4959
4960 // This is not an unconditional canonicalization: compare the cost of the
4961 // original scalar reduction and compare with the vector compare and i1
4962 // reduction replacement for both reduce.or and reduce.umax.
4963 InstructionCost OldCost = TTI.getInstructionCost(U: II, CostKind) +
4964 TTI.getInstructionCost(U: &I, CostKind);
4965
4966 auto *CmpTy = cast<VectorType>(Val: CmpInst::makeCmpResultType(opnd_type: VecTy));
4967 InstructionCost NewCost =
4968 TTI.getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: VecTy, CondTy: CmpTy, VecPred: Pred, CostKind);
4969 NewCost += TTI.getArithmeticReductionCost(
4970 Opcode: getArithmeticReductionInstruction(RdxID: NewIID), Ty: CmpTy, FMF: std::nullopt, CostKind);
4971
4972 LLVM_DEBUG(dbgs() << "Found a reduction zero test: " << I << "\n OldCost: "
4973 << OldCost << " vs NewCost: " << NewCost << "\n");
4974
4975 if (!OldCost.isValid() || !NewCost.isValid() || NewCost > OldCost)
4976 return false;
4977
4978 Builder.SetInsertPoint(&I);
4979 Value *NewCmp = Builder.CreateICmp(P: Pred, LHS: Vec, RHS: Constant::getNullValue(Ty: VecTy));
4980 Value *NewReduce = Builder.CreateIntrinsic(ID: NewIID, OverloadTypes: {CmpTy}, Args: {NewCmp});
4981 replaceValue(Old&: I, New&: *NewReduce);
4982 return true;
4983}
4984
4985/// vector.reduce.OP f(X_i) == 0 -> vector.reduce.OP X_i == 0
4986///
4987/// We can prove it for cases when:
4988///
4989/// 1. OP X_i == 0 <=> \forall i \in [1, N] X_i == 0
4990/// 1'. OP X_i == 0 <=> \exists j \in [1, N] X_j == 0
4991/// 2. f(x) == 0 <=> x == 0
4992///
4993/// From 1 and 2 (or 1' and 2), we can infer that
4994///
4995/// OP f(X_i) == 0 <=> OP X_i == 0.
4996///
4997/// (1)
4998/// OP f(X_i) == 0 <=> \forall i \in [1, N] f(X_i) == 0
4999/// (2)
5000/// <=> \forall i \in [1, N] X_i == 0
5001/// (1)
5002/// <=> OP(X_i) == 0
5003///
5004/// For some of the OP's and f's, we need to have domain constraints on X
5005/// to ensure properties 1 (or 1') and 2.
5006bool VectorCombine::foldICmpEqZeroVectorReduce(Instruction &I) {
5007 CmpPredicate Pred;
5008 Value *Op;
5009 if (!match(V: &I, P: m_ICmp(Pred, L: m_Value(V&: Op), R: m_Zero())) ||
5010 !ICmpInst::isEquality(P: Pred))
5011 return false;
5012
5013 auto *II = dyn_cast<IntrinsicInst>(Val: Op);
5014 if (!II)
5015 return false;
5016
5017 switch (II->getIntrinsicID()) {
5018 case Intrinsic::vector_reduce_add:
5019 case Intrinsic::vector_reduce_or:
5020 case Intrinsic::vector_reduce_umin:
5021 case Intrinsic::vector_reduce_umax:
5022 case Intrinsic::vector_reduce_smin:
5023 case Intrinsic::vector_reduce_smax:
5024 break;
5025 default:
5026 return false;
5027 }
5028
5029 Value *InnerOp = II->getArgOperand(i: 0);
5030
5031 // TODO: fixed vector type might be too restrictive
5032 if (!II->hasOneUse() || !isa<FixedVectorType>(Val: InnerOp->getType()))
5033 return false;
5034
5035 Value *X = nullptr;
5036
5037 // Check for zero-preserving operations where f(x) = 0 <=> x = 0
5038 //
5039 // 1. f(x) = shl nuw x, y for arbitrary y
5040 // 2. f(x) = mul nuw x, c for defined c != 0
5041 // 3. f(x) = zext x
5042 // 4. f(x) = sext x
5043 // 5. f(x) = neg x
5044 //
5045 if (!(match(V: InnerOp, P: m_NUWShl(L: m_Value(V&: X), R: m_Value())) || // Case 1
5046 match(V: InnerOp, P: m_NUWMul(L: m_Value(V&: X), R: m_NonZeroInt())) || // Case 2
5047 match(V: InnerOp, P: m_ZExt(Op: m_Value(V&: X))) || // Case 3
5048 match(V: InnerOp, P: m_SExt(Op: m_Value(V&: X))) || // Case 4
5049 match(V: InnerOp, P: m_Neg(V: m_Value(V&: X))) // Case 5
5050 ))
5051 return false;
5052
5053 SimplifyQuery S = SQ.getWithInstruction(I: &I);
5054 auto *XTy = cast<FixedVectorType>(Val: X->getType());
5055
5056 // Check for domain constraints for all supported reductions.
5057 //
5058 // a. OR X_i - has property 1 for every X
5059 // b. UMAX X_i - has property 1 for every X
5060 // c. UMIN X_i - has property 1' for every X
5061 // d. SMAX X_i - has property 1 for X >= 0
5062 // e. SMIN X_i - has property 1' for X >= 0
5063 // f. ADD X_i - has property 1 for X >= 0 && ADD X_i doesn't sign wrap
5064 //
5065 // In order for the proof to work, we need 1 (or 1') to be true for both
5066 // OP f(X_i) and OP X_i and that's why below we check constraints twice.
5067 //
5068 // NOTE: ADD X_i holds property 1 for a mirror case as well, i.e. when
5069 // X <= 0 && ADD X_i doesn't sign wrap. However, due to the nature
5070 // of known bits, we can't reasonably hold knowledge of "either 0
5071 // or negative".
5072 switch (II->getIntrinsicID()) {
5073 case Intrinsic::vector_reduce_add: {
5074 // We need to check that both X_i and f(X_i) have enough leading
5075 // zeros to not overflow.
5076 KnownBits KnownX = computeKnownBits(V: X, Q: S);
5077 KnownBits KnownFX = computeKnownBits(V: InnerOp, Q: S);
5078 unsigned NumElems = XTy->getNumElements();
5079 // Adding N elements loses at most ceil(log2(N)) leading bits.
5080 unsigned LostBits = Log2_32_Ceil(Value: NumElems);
5081 unsigned LeadingZerosX = KnownX.countMinLeadingZeros();
5082 unsigned LeadingZerosFX = KnownFX.countMinLeadingZeros();
5083 // Need at least one leading zero left after summation to ensure no overflow
5084 if (LeadingZerosX <= LostBits || LeadingZerosFX <= LostBits)
5085 return false;
5086
5087 // We are not checking whether X or f(X) are positive explicitly because
5088 // we implicitly checked for it when we checked if both cases have enough
5089 // leading zeros to not wrap addition.
5090 break;
5091 }
5092 case Intrinsic::vector_reduce_smin:
5093 case Intrinsic::vector_reduce_smax:
5094 // Check whether X >= 0 and f(X) >= 0
5095 if (!isKnownNonNegative(V: InnerOp, SQ: S) || !isKnownNonNegative(V: X, SQ: S))
5096 return false;
5097
5098 break;
5099 default:
5100 break;
5101 };
5102
5103 LLVM_DEBUG(dbgs() << "Found a reduction to 0 comparison with removable op: "
5104 << *II << "\n");
5105
5106 // For zext/sext, check if the transform is profitable using cost model.
5107 // For other operations (shl, mul, neg), we're removing an instruction
5108 // while keeping the same reduction type, so it's always profitable.
5109 if (isa<ZExtInst>(Val: InnerOp) || isa<SExtInst>(Val: InnerOp)) {
5110 auto *FXTy = cast<FixedVectorType>(Val: InnerOp->getType());
5111 Intrinsic::ID IID = II->getIntrinsicID();
5112
5113 InstructionCost ExtCost = TTI.getCastInstrCost(
5114 Opcode: cast<CastInst>(Val: InnerOp)->getOpcode(), Dst: FXTy, Src: XTy,
5115 CCH: TTI::CastContextHint::None, CostKind, I: cast<CastInst>(Val: InnerOp));
5116
5117 InstructionCost OldReduceCost, NewReduceCost;
5118 switch (IID) {
5119 case Intrinsic::vector_reduce_add:
5120 case Intrinsic::vector_reduce_or:
5121 OldReduceCost = TTI.getArithmeticReductionCost(
5122 Opcode: getArithmeticReductionInstruction(RdxID: IID), Ty: FXTy, FMF: std::nullopt, CostKind);
5123 NewReduceCost = TTI.getArithmeticReductionCost(
5124 Opcode: getArithmeticReductionInstruction(RdxID: IID), Ty: XTy, FMF: std::nullopt, CostKind);
5125 break;
5126 case Intrinsic::vector_reduce_umin:
5127 case Intrinsic::vector_reduce_umax:
5128 case Intrinsic::vector_reduce_smin:
5129 case Intrinsic::vector_reduce_smax:
5130 OldReduceCost = TTI.getMinMaxReductionCost(
5131 IID: getMinMaxReductionIntrinsicOp(RdxID: IID), Ty: FXTy, FMF: FastMathFlags(), CostKind);
5132 NewReduceCost = TTI.getMinMaxReductionCost(
5133 IID: getMinMaxReductionIntrinsicOp(RdxID: IID), Ty: XTy, FMF: FastMathFlags(), CostKind);
5134 break;
5135 default:
5136 llvm_unreachable("Unexpected reduction");
5137 }
5138
5139 InstructionCost OldCost = OldReduceCost + ExtCost;
5140 InstructionCost NewCost =
5141 NewReduceCost + (InnerOp->hasOneUse() ? 0 : ExtCost);
5142
5143 LLVM_DEBUG(dbgs() << "Found a removable extension before reduction: "
5144 << *InnerOp << "\n OldCost: " << OldCost
5145 << " vs NewCost: " << NewCost << "\n");
5146
5147 // We consider transformation to still be potentially beneficial even
5148 // when the costs are the same because we might remove a use from f(X)
5149 // and unlock other optimizations. Equal costs would just mean that we
5150 // didn't make it worse in the worst case.
5151 if (NewCost > OldCost)
5152 return false;
5153 }
5154
5155 // Since we support zext and sext as f, we might change the scalar type
5156 // of the intrinsic.
5157 Type *Ty = XTy->getScalarType();
5158 Value *NewReduce = Builder.CreateIntrinsic(RetTy: Ty, ID: II->getIntrinsicID(), Args: {X});
5159 Value *NewCmp =
5160 Builder.CreateICmp(P: Pred, LHS: NewReduce, RHS: ConstantInt::getNullValue(Ty));
5161 replaceValue(Old&: I, New&: *NewCmp);
5162 return true;
5163}
5164
5165/// Fold comparisons of reduce.or/reduce.and with reduce.umax/reduce.umin
5166/// based on cost, preserving the comparison semantics.
5167///
5168/// We use two fundamental properties for each pair:
5169///
5170/// 1. or(X) == 0 <=> umax(X) == 0
5171/// 2. or(X) == 1 <=> umax(X) == 1
5172/// 3. sign(or(X)) == sign(umax(X))
5173///
5174/// 1. and(X) == -1 <=> umin(X) == -1
5175/// 2. and(X) == -2 <=> umin(X) == -2
5176/// 3. sign(and(X)) == sign(umin(X))
5177///
5178/// From these we can infer the following transformations:
5179/// a. or(X) ==/!= 0 <-> umax(X) ==/!= 0
5180/// b. or(X) s< 0 <-> umax(X) s< 0
5181/// c. or(X) s> -1 <-> umax(X) s> -1
5182/// d. or(X) s< 1 <-> umax(X) s< 1
5183/// e. or(X) ==/!= 1 <-> umax(X) ==/!= 1
5184/// f. or(X) s< 2 <-> umax(X) s< 2
5185/// g. and(X) ==/!= -1 <-> umin(X) ==/!= -1
5186/// h. and(X) s< 0 <-> umin(X) s< 0
5187/// i. and(X) s> -1 <-> umin(X) s> -1
5188/// j. and(X) s> -2 <-> umin(X) s> -2
5189/// k. and(X) ==/!= -2 <-> umin(X) ==/!= -2
5190/// l. and(X) s> -3 <-> umin(X) s> -3
5191///
5192bool VectorCombine::foldEquivalentReductionCmp(Instruction &I) {
5193 CmpPredicate Pred;
5194 Value *ReduceOp;
5195 const APInt *CmpVal;
5196 if (!match(V: &I, P: m_ICmp(Pred, L: m_Value(V&: ReduceOp), R: m_APInt(Res&: CmpVal))))
5197 return false;
5198
5199 auto *II = dyn_cast<IntrinsicInst>(Val: ReduceOp);
5200 if (!II || !II->hasOneUse())
5201 return false;
5202
5203 const auto IsValidOrUmaxCmp = [&]() {
5204 // or === umax for i1
5205 if (CmpVal->getBitWidth() == 1)
5206 return true;
5207
5208 // Cases a and e
5209 bool IsEquality =
5210 (CmpVal->isZero() || CmpVal->isOne()) && ICmpInst::isEquality(P: Pred);
5211 // Case c
5212 bool IsPositive = CmpVal->isAllOnes() && Pred == ICmpInst::ICMP_SGT;
5213 // Cases b, d, and f
5214 bool IsNegative = (CmpVal->isZero() || CmpVal->isOne() || *CmpVal == 2) &&
5215 Pred == ICmpInst::ICMP_SLT;
5216 return IsEquality || IsPositive || IsNegative;
5217 };
5218
5219 const auto IsValidAndUminCmp = [&]() {
5220 // and === umin for i1
5221 if (CmpVal->getBitWidth() == 1)
5222 return true;
5223
5224 const auto LeadingOnes = CmpVal->countl_one();
5225
5226 // Cases g and k
5227 bool IsEquality =
5228 (CmpVal->isAllOnes() || LeadingOnes + 1 == CmpVal->getBitWidth()) &&
5229 ICmpInst::isEquality(P: Pred);
5230 // Case h
5231 bool IsNegative = CmpVal->isZero() && Pred == ICmpInst::ICMP_SLT;
5232 // Cases i, j, and l
5233 bool IsPositive =
5234 // if the number has at least N - 2 leading ones
5235 // and the two LSBs are:
5236 // - 1 x 1 -> -1
5237 // - 1 x 0 -> -2
5238 // - 0 x 1 -> -3
5239 LeadingOnes + 2 >= CmpVal->getBitWidth() &&
5240 ((*CmpVal)[0] || (*CmpVal)[1]) && Pred == ICmpInst::ICMP_SGT;
5241 return IsEquality || IsNegative || IsPositive;
5242 };
5243
5244 Intrinsic::ID OriginalIID = II->getIntrinsicID();
5245 Intrinsic::ID AlternativeIID;
5246
5247 // Check if this is a valid comparison pattern and determine the alternate
5248 // reduction intrinsic.
5249 switch (OriginalIID) {
5250 case Intrinsic::vector_reduce_or:
5251 if (!IsValidOrUmaxCmp())
5252 return false;
5253 AlternativeIID = Intrinsic::vector_reduce_umax;
5254 break;
5255 case Intrinsic::vector_reduce_umax:
5256 if (!IsValidOrUmaxCmp())
5257 return false;
5258 AlternativeIID = Intrinsic::vector_reduce_or;
5259 break;
5260 case Intrinsic::vector_reduce_and:
5261 if (!IsValidAndUminCmp())
5262 return false;
5263 AlternativeIID = Intrinsic::vector_reduce_umin;
5264 break;
5265 case Intrinsic::vector_reduce_umin:
5266 if (!IsValidAndUminCmp())
5267 return false;
5268 AlternativeIID = Intrinsic::vector_reduce_and;
5269 break;
5270 default:
5271 return false;
5272 }
5273
5274 Value *X = II->getArgOperand(i: 0);
5275 auto *VecTy = dyn_cast<FixedVectorType>(Val: X->getType());
5276 if (!VecTy)
5277 return false;
5278
5279 const auto GetReductionCost = [&](Intrinsic::ID IID) -> InstructionCost {
5280 unsigned ReductionOpc = getArithmeticReductionInstruction(RdxID: IID);
5281 if (ReductionOpc != Instruction::ICmp)
5282 return TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: VecTy, FMF: std::nullopt,
5283 CostKind);
5284 return TTI.getMinMaxReductionCost(IID: getMinMaxReductionIntrinsicOp(RdxID: IID), Ty: VecTy,
5285 FMF: FastMathFlags(), CostKind);
5286 };
5287
5288 InstructionCost OrigCost = GetReductionCost(OriginalIID);
5289 InstructionCost AltCost = GetReductionCost(AlternativeIID);
5290
5291 LLVM_DEBUG(dbgs() << "Found equivalent reduction cmp: " << I
5292 << "\n OrigCost: " << OrigCost
5293 << " vs AltCost: " << AltCost << "\n");
5294
5295 if (AltCost >= OrigCost)
5296 return false;
5297
5298 Builder.SetInsertPoint(&I);
5299 Type *ScalarTy = VecTy->getScalarType();
5300 Value *NewReduce = Builder.CreateIntrinsic(RetTy: ScalarTy, ID: AlternativeIID, Args: {X});
5301 Value *NewCmp =
5302 Builder.CreateICmp(P: Pred, LHS: NewReduce, RHS: ConstantInt::get(Ty: ScalarTy, V: *CmpVal));
5303
5304 replaceValue(Old&: I, New&: *NewCmp);
5305 return true;
5306}
5307
5308/// Used by foldReduceAddCmpZero to check if we can prove that a value is
5309/// non-positive.
5310/// KnownBits cannot see sext <? x i1> as non-positive: each top bit equals a
5311/// single unknown input bit, which a per-bit lattice cannot track. The fold's
5312/// target shape is popcount-style sums of <N x i1> valid/invalid masks (e.g.
5313/// ray-intersection hits) tested for any-hit.
5314/// Previous attempts to approximate the known bits of such expressions were
5315/// using a fully recursive value tracking approach to infer a constant range
5316/// but ultimately turned to be too expensive in compile time.
5317static bool isKnownNonPositive(const Value *V, const SimplifyQuery &SQ,
5318 unsigned Depth = 0) {
5319 constexpr unsigned MaxLocalDepth = 2;
5320 if (Depth > MaxLocalDepth)
5321 return false;
5322
5323 auto NumSignBits = [&](const Value *X) {
5324 return ComputeNumSignBits(Op: X, DL: SQ.DL, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT);
5325 };
5326 if (NumSignBits(V) == V->getType()->getScalarSizeInBits())
5327 return true;
5328
5329 Value *A, *B;
5330 if (match(V, P: m_Add(L: m_Value(V&: A), R: m_Value(V&: B))))
5331 return NumSignBits(A) >= 2 && NumSignBits(B) >= 2 &&
5332 isKnownNonPositive(V: A, SQ, Depth: Depth + 1) &&
5333 isKnownNonPositive(V: B, SQ, Depth: Depth + 1);
5334
5335 return computeKnownBits(V, Q: SQ).isNonPositive();
5336}
5337
5338/// Fold (icmp pred (reduce.add X), 0) to (icmp pred' (reduce.or X), 0) when X
5339/// has lanes known to all be non-negative or all non-positive, so that
5340/// sum == 0 iff every lane is 0. Falls back to reduce.umax if reduce.or is
5341/// more expensive on the target.
5342bool VectorCombine::foldReduceAddCmpZero(Instruction &I) {
5343 CmpPredicate Pred;
5344 Value *Vec;
5345 if (!match(V: &I, P: m_ICmp(Pred,
5346 L: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::vector_reduce_add>(
5347 Ops: m_Value(V&: Vec))),
5348 R: m_Zero())))
5349 return false;
5350
5351 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
5352 if (!VecTy || VecTy->getNumElements() < 2)
5353 return false;
5354
5355 SimplifyQuery Q = SQ.getWithInstruction(I: &I);
5356 bool IsNonNegative = isKnownNonNegative(V: Vec, SQ: Q);
5357 bool IsNonPositive = !IsNonNegative && isKnownNonPositive(V: Vec, SQ: Q);
5358 if (!IsNonNegative && !IsNonPositive)
5359 return false;
5360
5361 // Summing NumElts lanes can consume up to log2(NumElts) sign bits. Require
5362 // strictly more headroom than that so the sum cannot wrap to zero.
5363 unsigned NumElts = VecTy->getNumElements();
5364 unsigned NumSignBits = ComputeNumSignBits(Op: Vec, DL: *DL, AC: SQ.AC, CtxI: &I, DT: &DT);
5365 if (Log2_32(Value: NumElts) >= NumSignBits)
5366 return false;
5367
5368 ICmpInst::Predicate NewPred;
5369 switch (Pred) {
5370 case ICmpInst::ICMP_EQ:
5371 case ICmpInst::ICMP_ULE:
5372 case ICmpInst::ICMP_SLE:
5373 case ICmpInst::ICMP_SGE:
5374 NewPred = ICmpInst::ICMP_EQ;
5375 break;
5376 case ICmpInst::ICMP_NE:
5377 case ICmpInst::ICMP_UGT:
5378 case ICmpInst::ICMP_SGT:
5379 case ICmpInst::ICMP_SLT:
5380 NewPred = ICmpInst::ICMP_NE;
5381 break;
5382 default:
5383 return false;
5384 }
5385
5386 // SGT and SLE on a non-positive tree, and SLT and SGE on a non-negative
5387 // tree, are tautologies (always true or always false). Leave those to
5388 // InstCombine rather than mapping them here. Remaining signed inequalities
5389 // also need one extra sign bit so the sum cannot flip sign.
5390 if (!IsNonNegative &&
5391 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE))
5392 return false;
5393 if (!IsNonPositive &&
5394 (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE))
5395 return false;
5396 if ((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE ||
5397 Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) &&
5398 Log2_32(Value: NumElts) >= NumSignBits - 1)
5399 return false;
5400
5401 InstructionCost OrigCost = TTI.getArithmeticReductionCost(
5402 Opcode: Instruction::Add, Ty: VecTy, FMF: std::nullopt, CostKind);
5403 InstructionCost OrCost = TTI.getArithmeticReductionCost(
5404 Opcode: Instruction::Or, Ty: VecTy, FMF: std::nullopt, CostKind);
5405 InstructionCost UmaxCost = TTI.getMinMaxReductionCost(
5406 IID: Intrinsic::umax, Ty: VecTy, FMF: FastMathFlags(), CostKind);
5407 if (!OrCost.isValid() && !UmaxCost.isValid())
5408 return false;
5409 bool UseOr = OrCost.isValid() && (!UmaxCost.isValid() || OrCost <= UmaxCost);
5410 InstructionCost AltCost = UseOr ? OrCost : UmaxCost;
5411 if (AltCost > OrigCost)
5412 return false;
5413
5414 Builder.SetInsertPoint(&I);
5415 Value *NewReduce = UseOr ? Builder.CreateOrReduce(Src: Vec)
5416 : Builder.CreateIntrinsic(
5417 ID: Intrinsic::vector_reduce_umax, OverloadTypes: {VecTy}, Args: {Vec});
5418 Value *NewCmp = Builder.CreateICmp(
5419 P: NewPred, LHS: NewReduce, RHS: ConstantInt::getNullValue(Ty: VecTy->getScalarType()));
5420 replaceValue(Old&: I, New&: *NewCmp);
5421 return true;
5422}
5423
5424/// Returns true if this ShuffleVectorInst eventually feeds into a
5425/// vector reduction intrinsic (e.g., vector_reduce_add) by only following
5426/// chains of shuffles and binary operators (in any combination/order).
5427/// The search does not go deeper than the given Depth.
5428static bool feedsIntoVectorReduction(ShuffleVectorInst *SVI) {
5429 constexpr unsigned MaxVisited = 32;
5430 SmallPtrSet<Instruction *, 8> Visited;
5431 SmallVector<Instruction *, 4> WorkList;
5432 bool FoundReduction = false;
5433
5434 WorkList.push_back(Elt: SVI);
5435 while (!WorkList.empty()) {
5436 Instruction *I = WorkList.pop_back_val();
5437 for (User *U : I->users()) {
5438 auto *UI = cast<Instruction>(Val: U);
5439 if (!UI || !Visited.insert(Ptr: UI).second)
5440 continue;
5441 if (Visited.size() > MaxVisited)
5442 return false;
5443 if (auto *II = dyn_cast<IntrinsicInst>(Val: UI)) {
5444 // More than one reduction reached
5445 if (FoundReduction)
5446 return false;
5447 switch (II->getIntrinsicID()) {
5448 case Intrinsic::vector_reduce_add:
5449 case Intrinsic::vector_reduce_mul:
5450 case Intrinsic::vector_reduce_and:
5451 case Intrinsic::vector_reduce_or:
5452 case Intrinsic::vector_reduce_xor:
5453 case Intrinsic::vector_reduce_smin:
5454 case Intrinsic::vector_reduce_smax:
5455 case Intrinsic::vector_reduce_umin:
5456 case Intrinsic::vector_reduce_umax:
5457 FoundReduction = true;
5458 continue;
5459 default:
5460 return false;
5461 }
5462 }
5463
5464 if (!isa<BinaryOperator>(Val: UI) && !isa<ShuffleVectorInst>(Val: UI))
5465 return false;
5466
5467 WorkList.emplace_back(Args&: UI);
5468 }
5469 }
5470 return FoundReduction;
5471}
5472
5473/// This method looks for groups of shuffles acting on binops, of the form:
5474/// %x = shuffle ...
5475/// %y = shuffle ...
5476/// %a = binop %x, %y
5477/// %b = binop %x, %y
5478/// shuffle %a, %b, selectmask
5479/// We may, especially if the shuffle is wider than legal, be able to convert
5480/// the shuffle to a form where only parts of a and b need to be computed. On
5481/// architectures with no obvious "select" shuffle, this can reduce the total
5482/// number of operations if the target reports them as cheaper.
5483bool VectorCombine::foldSelectShuffle(Instruction &I, bool FromReduction) {
5484 auto *SVI = cast<ShuffleVectorInst>(Val: &I);
5485 auto *VT = cast<FixedVectorType>(Val: I.getType());
5486 auto *Op0 = dyn_cast<Instruction>(Val: SVI->getOperand(i_nocapture: 0));
5487 auto *Op1 = dyn_cast<Instruction>(Val: SVI->getOperand(i_nocapture: 1));
5488 if (!Op0 || !Op1 || Op0 == Op1 || !Op0->isBinaryOp() || !Op1->isBinaryOp() ||
5489 VT != Op0->getType())
5490 return false;
5491
5492 auto *SVI0A = dyn_cast<Instruction>(Val: Op0->getOperand(i: 0));
5493 auto *SVI0B = dyn_cast<Instruction>(Val: Op0->getOperand(i: 1));
5494 auto *SVI1A = dyn_cast<Instruction>(Val: Op1->getOperand(i: 0));
5495 auto *SVI1B = dyn_cast<Instruction>(Val: Op1->getOperand(i: 1));
5496 SmallPtrSet<Instruction *, 4> InputShuffles({SVI0A, SVI0B, SVI1A, SVI1B});
5497 auto checkSVNonOpUses = [&](Instruction *I) {
5498 if (!I || I->getOperand(i: 0)->getType() != VT)
5499 return true;
5500 return any_of(Range: I->users(), P: [&](User *U) {
5501 return U != Op0 && U != Op1 &&
5502 !(isa<ShuffleVectorInst>(Val: U) &&
5503 (InputShuffles.contains(Ptr: cast<Instruction>(Val: U)) ||
5504 isInstructionTriviallyDead(I: cast<Instruction>(Val: U))));
5505 });
5506 };
5507 if (checkSVNonOpUses(SVI0A) || checkSVNonOpUses(SVI0B) ||
5508 checkSVNonOpUses(SVI1A) || checkSVNonOpUses(SVI1B))
5509 return false;
5510
5511 // Collect all the uses that are shuffles that we can transform together. We
5512 // may not have a single shuffle, but a group that can all be transformed
5513 // together profitably.
5514 SmallVector<ShuffleVectorInst *> Shuffles;
5515 auto collectShuffles = [&](Instruction *I) {
5516 for (auto *U : I->users()) {
5517 auto *SV = dyn_cast<ShuffleVectorInst>(Val: U);
5518 if (!SV || SV->getType() != VT)
5519 return false;
5520 if ((SV->getOperand(i_nocapture: 0) != Op0 && SV->getOperand(i_nocapture: 0) != Op1) ||
5521 (SV->getOperand(i_nocapture: 1) != Op0 && SV->getOperand(i_nocapture: 1) != Op1))
5522 return false;
5523 if (!llvm::is_contained(Range&: Shuffles, Element: SV))
5524 Shuffles.push_back(Elt: SV);
5525 }
5526 return true;
5527 };
5528 if (!collectShuffles(Op0) || !collectShuffles(Op1))
5529 return false;
5530 // From a reduction, we need to be processing a single shuffle, otherwise the
5531 // other uses will not be lane-invariant.
5532 if (FromReduction && Shuffles.size() > 1)
5533 return false;
5534
5535 // Add any shuffle uses for the shuffles we have found, to include them in our
5536 // cost calculations.
5537 if (!FromReduction) {
5538 for (size_t Idx = 0, E = Shuffles.size(); Idx != E; ++Idx) {
5539 for (auto *U : Shuffles[Idx]->users()) {
5540 ShuffleVectorInst *SSV = dyn_cast<ShuffleVectorInst>(Val: U);
5541 if (SSV && isa<UndefValue>(Val: SSV->getOperand(i_nocapture: 1)) && SSV->getType() == VT)
5542 Shuffles.push_back(Elt: SSV);
5543 }
5544 }
5545 }
5546
5547 // For each of the output shuffles, we try to sort all the first vector
5548 // elements to the beginning, followed by the second array elements at the
5549 // end. If the binops are legalized to smaller vectors, this may reduce total
5550 // number of binops. We compute the ReconstructMask mask needed to convert
5551 // back to the original lane order.
5552 SmallVector<std::pair<int, int>> V1, V2;
5553 SmallVector<SmallVector<int>> OrigReconstructMasks;
5554 int MaxV1Elt = 0, MaxV2Elt = 0;
5555 unsigned NumElts = VT->getNumElements();
5556 for (ShuffleVectorInst *SVN : Shuffles) {
5557 SmallVector<int> Mask;
5558 SVN->getShuffleMask(Result&: Mask);
5559
5560 // Check the operands are the same as the original, or reversed (in which
5561 // case we need to commute the mask).
5562 Value *SVOp0 = SVN->getOperand(i_nocapture: 0);
5563 Value *SVOp1 = SVN->getOperand(i_nocapture: 1);
5564 if (isa<UndefValue>(Val: SVOp1)) {
5565 auto *SSV = cast<ShuffleVectorInst>(Val: SVOp0);
5566 SVOp0 = SSV->getOperand(i_nocapture: 0);
5567 SVOp1 = SSV->getOperand(i_nocapture: 1);
5568 for (int &Elem : Mask) {
5569 if (Elem >= static_cast<int>(SSV->getShuffleMask().size()))
5570 return false;
5571 Elem = Elem < 0 ? Elem : SSV->getMaskValue(Elt: Elem);
5572 }
5573 }
5574 if (SVOp0 == Op1 && SVOp1 == Op0) {
5575 std::swap(a&: SVOp0, b&: SVOp1);
5576 ShuffleVectorInst::commuteShuffleMask(Mask, InVecNumElts: NumElts);
5577 }
5578 if (SVOp0 != Op0 || SVOp1 != Op1)
5579 return false;
5580
5581 // Calculate the reconstruction mask for this shuffle, as the mask needed to
5582 // take the packed values from Op0/Op1 and reconstructing to the original
5583 // order.
5584 SmallVector<int> ReconstructMask;
5585 for (unsigned I = 0; I < Mask.size(); I++) {
5586 if (Mask[I] < 0) {
5587 ReconstructMask.push_back(Elt: -1);
5588 } else if (Mask[I] < static_cast<int>(NumElts)) {
5589 MaxV1Elt = std::max(a: MaxV1Elt, b: Mask[I]);
5590 auto It = find_if(Range&: V1, P: [&](const std::pair<int, int> &A) {
5591 return Mask[I] == A.first;
5592 });
5593 if (It != V1.end())
5594 ReconstructMask.push_back(Elt: It - V1.begin());
5595 else {
5596 ReconstructMask.push_back(Elt: V1.size());
5597 V1.emplace_back(Args&: Mask[I], Args: V1.size());
5598 }
5599 } else {
5600 MaxV2Elt = std::max<int>(a: MaxV2Elt, b: Mask[I] - NumElts);
5601 auto It = find_if(Range&: V2, P: [&](const std::pair<int, int> &A) {
5602 return Mask[I] - static_cast<int>(NumElts) == A.first;
5603 });
5604 if (It != V2.end())
5605 ReconstructMask.push_back(Elt: NumElts + It - V2.begin());
5606 else {
5607 ReconstructMask.push_back(Elt: NumElts + V2.size());
5608 V2.emplace_back(Args: Mask[I] - NumElts, Args: NumElts + V2.size());
5609 }
5610 }
5611 }
5612
5613 // For reductions, we know that the lane ordering out doesn't alter the
5614 // result. In-order can help simplify the shuffle away.
5615 if (FromReduction)
5616 sort(C&: ReconstructMask);
5617 OrigReconstructMasks.push_back(Elt: std::move(ReconstructMask));
5618 }
5619
5620 // If the Maximum element used from V1 and V2 are not larger than the new
5621 // vectors, the vectors are already packes and performing the optimization
5622 // again will likely not help any further. This also prevents us from getting
5623 // stuck in a cycle in case the costs do not also rule it out.
5624 if (V1.empty() || V2.empty() ||
5625 (MaxV1Elt == static_cast<int>(V1.size()) - 1 &&
5626 MaxV2Elt == static_cast<int>(V2.size()) - 1))
5627 return false;
5628
5629 // GetBaseMaskValue takes one of the inputs, which may either be a shuffle, a
5630 // shuffle of another shuffle, or not a shuffle (that is treated like a
5631 // identity shuffle).
5632 auto GetBaseMaskValue = [&](Instruction *I, int M) {
5633 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5634 if (!SV)
5635 return M;
5636 if (isa<UndefValue>(Val: SV->getOperand(i_nocapture: 1)))
5637 if (auto *SSV = dyn_cast<ShuffleVectorInst>(Val: SV->getOperand(i_nocapture: 0)))
5638 if (InputShuffles.contains(Ptr: SSV))
5639 return SSV->getMaskValue(Elt: SV->getMaskValue(Elt: M));
5640 return SV->getMaskValue(Elt: M);
5641 };
5642
5643 // Attempt to sort the inputs my ascending mask values to make simpler input
5644 // shuffles and push complex shuffles down to the uses. We sort on the first
5645 // of the two input shuffle orders, to try and get at least one input into a
5646 // nice order.
5647 auto SortBase = [&](Instruction *A, std::pair<int, int> X,
5648 std::pair<int, int> Y) {
5649 int MXA = GetBaseMaskValue(A, X.first);
5650 int MYA = GetBaseMaskValue(A, Y.first);
5651 return MXA < MYA;
5652 };
5653 stable_sort(Range&: V1, C: [&](std::pair<int, int> A, std::pair<int, int> B) {
5654 return SortBase(SVI0A, A, B);
5655 });
5656 stable_sort(Range&: V2, C: [&](std::pair<int, int> A, std::pair<int, int> B) {
5657 return SortBase(SVI1A, A, B);
5658 });
5659 // Calculate our ReconstructMasks from the OrigReconstructMasks and the
5660 // modified order of the input shuffles.
5661 SmallVector<SmallVector<int>> ReconstructMasks;
5662 for (const auto &Mask : OrigReconstructMasks) {
5663 SmallVector<int> ReconstructMask;
5664 for (int M : Mask) {
5665 auto FindIndex = [](const SmallVector<std::pair<int, int>> &V, int M) {
5666 auto It = find_if(Range: V, P: [M](auto A) { return A.second == M; });
5667 assert(It != V.end() && "Expected all entries in Mask");
5668 return std::distance(first: V.begin(), last: It);
5669 };
5670 if (M < 0)
5671 ReconstructMask.push_back(Elt: -1);
5672 else if (M < static_cast<int>(NumElts)) {
5673 ReconstructMask.push_back(Elt: FindIndex(V1, M));
5674 } else {
5675 ReconstructMask.push_back(Elt: NumElts + FindIndex(V2, M));
5676 }
5677 }
5678 ReconstructMasks.push_back(Elt: std::move(ReconstructMask));
5679 }
5680
5681 // Calculate the masks needed for the new input shuffles, which get padded
5682 // with undef
5683 SmallVector<int> V1A, V1B, V2A, V2B;
5684 for (unsigned I = 0; I < V1.size(); I++) {
5685 V1A.push_back(Elt: GetBaseMaskValue(SVI0A, V1[I].first));
5686 V1B.push_back(Elt: GetBaseMaskValue(SVI0B, V1[I].first));
5687 }
5688 for (unsigned I = 0; I < V2.size(); I++) {
5689 V2A.push_back(Elt: GetBaseMaskValue(SVI1A, V2[I].first));
5690 V2B.push_back(Elt: GetBaseMaskValue(SVI1B, V2[I].first));
5691 }
5692 while (V1A.size() < NumElts) {
5693 V1A.push_back(Elt: PoisonMaskElem);
5694 V1B.push_back(Elt: PoisonMaskElem);
5695 }
5696 while (V2A.size() < NumElts) {
5697 V2A.push_back(Elt: PoisonMaskElem);
5698 V2B.push_back(Elt: PoisonMaskElem);
5699 }
5700
5701 auto AddShuffleCost = [&](InstructionCost C, Instruction *I) {
5702 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5703 if (!SV)
5704 return C;
5705 return C + TTI.getShuffleCost(Kind: isa<UndefValue>(Val: SV->getOperand(i_nocapture: 1))
5706 ? TTI::SK_PermuteSingleSrc
5707 : TTI::SK_PermuteTwoSrc,
5708 DstTy: VT, SrcTy: VT, CostKind, Mask: SV->getShuffleMask());
5709 };
5710 auto AddShuffleMaskCost = [&](InstructionCost C, ArrayRef<int> Mask) {
5711 return C +
5712 TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: VT, SrcTy: VT, CostKind, Mask);
5713 };
5714
5715 unsigned ElementSize = VT->getElementType()->getPrimitiveSizeInBits();
5716 unsigned MaxVectorSize =
5717 TTI.getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector);
5718 unsigned MaxElementsInVector = MaxVectorSize / ElementSize;
5719 if (MaxElementsInVector == 0)
5720 return false;
5721 // When there are multiple shufflevector operations on the same input,
5722 // especially when the vector length is larger than the register size,
5723 // identical shuffle patterns may occur across different groups of elements.
5724 // To avoid overestimating the cost by counting these repeated shuffles more
5725 // than once, we only account for unique shuffle patterns. This adjustment
5726 // prevents inflated costs in the cost model for wide vectors split into
5727 // several register-sized groups.
5728 std::set<SmallVector<int, 4>> UniqueShuffles;
5729 auto AddShuffleMaskAdjustedCost = [&](InstructionCost C, ArrayRef<int> Mask) {
5730 // Compute the cost for performing the shuffle over the full vector.
5731 auto ShuffleCost =
5732 TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: VT, SrcTy: VT, CostKind, Mask);
5733 unsigned NumFullVectors = Mask.size() / MaxElementsInVector;
5734 if (NumFullVectors < 2)
5735 return C + ShuffleCost;
5736 SmallVector<int, 4> SubShuffle(MaxElementsInVector);
5737 unsigned NumUniqueGroups = 0;
5738 unsigned NumGroups = Mask.size() / MaxElementsInVector;
5739 // For each group of MaxElementsInVector contiguous elements,
5740 // collect their shuffle pattern and insert into the set of unique patterns.
5741 for (unsigned I = 0; I < NumFullVectors; ++I) {
5742 for (unsigned J = 0; J < MaxElementsInVector; ++J)
5743 SubShuffle[J] = Mask[MaxElementsInVector * I + J];
5744 if (UniqueShuffles.insert(x: SubShuffle).second)
5745 NumUniqueGroups += 1;
5746 }
5747 return C + ShuffleCost * NumUniqueGroups / NumGroups;
5748 };
5749 auto AddShuffleAdjustedCost = [&](InstructionCost C, Instruction *I) {
5750 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5751 if (!SV)
5752 return C;
5753 SmallVector<int, 16> Mask;
5754 SV->getShuffleMask(Result&: Mask);
5755 return AddShuffleMaskAdjustedCost(C, Mask);
5756 };
5757 // Check that input consists of ShuffleVectors applied to the same input
5758 auto AllShufflesHaveSameOperands =
5759 [](SmallPtrSetImpl<Instruction *> &InputShuffles) {
5760 if (InputShuffles.size() < 2)
5761 return false;
5762 ShuffleVectorInst *FirstSV =
5763 dyn_cast<ShuffleVectorInst>(Val: *InputShuffles.begin());
5764 if (!FirstSV)
5765 return false;
5766
5767 Value *In0 = FirstSV->getOperand(i_nocapture: 0), *In1 = FirstSV->getOperand(i_nocapture: 1);
5768 return std::all_of(
5769 first: std::next(x: InputShuffles.begin()), last: InputShuffles.end(),
5770 pred: [&](Instruction *I) {
5771 ShuffleVectorInst *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5772 return SV && SV->getOperand(i_nocapture: 0) == In0 && SV->getOperand(i_nocapture: 1) == In1;
5773 });
5774 };
5775
5776 // Get the costs of the shuffles + binops before and after with the new
5777 // shuffle masks.
5778 InstructionCost CostBefore =
5779 TTI.getArithmeticInstrCost(Opcode: Op0->getOpcode(), Ty: VT, CostKind) +
5780 TTI.getArithmeticInstrCost(Opcode: Op1->getOpcode(), Ty: VT, CostKind);
5781 CostBefore += std::accumulate(first: Shuffles.begin(), last: Shuffles.end(),
5782 init: InstructionCost(0), binary_op: AddShuffleCost);
5783 if (AllShufflesHaveSameOperands(InputShuffles)) {
5784 UniqueShuffles.clear();
5785 CostBefore += std::accumulate(first: InputShuffles.begin(), last: InputShuffles.end(),
5786 init: InstructionCost(0), binary_op: AddShuffleAdjustedCost);
5787 } else {
5788 CostBefore += std::accumulate(first: InputShuffles.begin(), last: InputShuffles.end(),
5789 init: InstructionCost(0), binary_op: AddShuffleCost);
5790 }
5791
5792 // The new binops will be unused for lanes past the used shuffle lengths.
5793 // These types attempt to get the correct cost for that from the target.
5794 FixedVectorType *Op0SmallVT =
5795 FixedVectorType::get(ElementType: VT->getScalarType(), NumElts: V1.size());
5796 FixedVectorType *Op1SmallVT =
5797 FixedVectorType::get(ElementType: VT->getScalarType(), NumElts: V2.size());
5798 InstructionCost CostAfter =
5799 TTI.getArithmeticInstrCost(Opcode: Op0->getOpcode(), Ty: Op0SmallVT, CostKind) +
5800 TTI.getArithmeticInstrCost(Opcode: Op1->getOpcode(), Ty: Op1SmallVT, CostKind);
5801 UniqueShuffles.clear();
5802 CostAfter += std::accumulate(first: ReconstructMasks.begin(), last: ReconstructMasks.end(),
5803 init: InstructionCost(0), binary_op: AddShuffleMaskAdjustedCost);
5804 std::set<SmallVector<int>> OutputShuffleMasks({V1A, V1B, V2A, V2B});
5805 CostAfter +=
5806 std::accumulate(first: OutputShuffleMasks.begin(), last: OutputShuffleMasks.end(),
5807 init: InstructionCost(0), binary_op: AddShuffleMaskCost);
5808
5809 LLVM_DEBUG(dbgs() << "Found a binop select shuffle pattern: " << I << "\n");
5810 LLVM_DEBUG(dbgs() << " CostBefore: " << CostBefore
5811 << " vs CostAfter: " << CostAfter << "\n");
5812 if (CostBefore < CostAfter ||
5813 (CostBefore == CostAfter && !feedsIntoVectorReduction(SVI)))
5814 return false;
5815
5816 // The cost model has passed, create the new instructions.
5817 auto GetShuffleOperand = [&](Instruction *I, unsigned Op) -> Value * {
5818 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5819 if (!SV)
5820 return I;
5821 if (isa<UndefValue>(Val: SV->getOperand(i_nocapture: 1)))
5822 if (auto *SSV = dyn_cast<ShuffleVectorInst>(Val: SV->getOperand(i_nocapture: 0)))
5823 if (InputShuffles.contains(Ptr: SSV))
5824 return SSV->getOperand(i_nocapture: Op);
5825 return SV->getOperand(i_nocapture: Op);
5826 };
5827 Builder.SetInsertPoint(*SVI0A->getInsertionPointAfterDef());
5828 Value *NSV0A = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI0A, 0),
5829 V2: GetShuffleOperand(SVI0A, 1), Mask: V1A);
5830 Builder.SetInsertPoint(*SVI0B->getInsertionPointAfterDef());
5831 Value *NSV0B = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI0B, 0),
5832 V2: GetShuffleOperand(SVI0B, 1), Mask: V1B);
5833 Builder.SetInsertPoint(*SVI1A->getInsertionPointAfterDef());
5834 Value *NSV1A = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI1A, 0),
5835 V2: GetShuffleOperand(SVI1A, 1), Mask: V2A);
5836 Builder.SetInsertPoint(*SVI1B->getInsertionPointAfterDef());
5837 Value *NSV1B = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI1B, 0),
5838 V2: GetShuffleOperand(SVI1B, 1), Mask: V2B);
5839 Builder.SetInsertPoint(Op0);
5840 Value *NOp0 = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Op0->getOpcode(),
5841 LHS: NSV0A, RHS: NSV0B);
5842 if (auto *I = dyn_cast<Instruction>(Val: NOp0))
5843 I->copyIRFlags(V: Op0, IncludeWrapFlags: true);
5844 Builder.SetInsertPoint(Op1);
5845 Value *NOp1 = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Op1->getOpcode(),
5846 LHS: NSV1A, RHS: NSV1B);
5847 if (auto *I = dyn_cast<Instruction>(Val: NOp1))
5848 I->copyIRFlags(V: Op1, IncludeWrapFlags: true);
5849
5850 for (int S = 0, E = ReconstructMasks.size(); S != E; S++) {
5851 Builder.SetInsertPoint(Shuffles[S]);
5852 Value *NSV = Builder.CreateShuffleVector(V1: NOp0, V2: NOp1, Mask: ReconstructMasks[S]);
5853 replaceValue(Old&: *Shuffles[S], New&: *NSV, Erase: false);
5854 }
5855
5856 return true;
5857}
5858
5859/// Check if instruction depends on ZExt and this ZExt can be moved after the
5860/// instruction. Move ZExt if it is profitable. For example:
5861/// logic(zext(x),y) -> zext(logic(x,trunc(y)))
5862/// lshr((zext(x),y) -> zext(lshr(x,trunc(y)))
5863/// Cost model calculations takes into account if zext(x) has other users and
5864/// whether it can be propagated through them too.
5865bool VectorCombine::shrinkType(Instruction &I) {
5866 Value *ZExted, *OtherOperand;
5867 if (!match(V: &I, P: m_c_BitwiseLogic(L: m_ZExt(Op: m_Value(V&: ZExted)),
5868 R: m_Value(V&: OtherOperand))) &&
5869 !match(V: &I, P: m_LShr(L: m_ZExt(Op: m_Value(V&: ZExted)), R: m_Value(V&: OtherOperand))))
5870 return false;
5871
5872 Value *ZExtOperand = I.getOperand(i: I.getOperand(i: 0) == OtherOperand ? 1 : 0);
5873
5874 auto *BigTy = cast<FixedVectorType>(Val: I.getType());
5875 auto *SmallTy = cast<FixedVectorType>(Val: ZExted->getType());
5876 unsigned BW = SmallTy->getElementType()->getPrimitiveSizeInBits();
5877
5878 if (I.getOpcode() == Instruction::LShr) {
5879 // Check that the shift amount is less than the number of bits in the
5880 // smaller type. Otherwise, the smaller lshr will return a poison value.
5881 KnownBits ShAmtKB = computeKnownBits(V: I.getOperand(i: 1), DL: *DL);
5882 if (ShAmtKB.getMaxValue().uge(RHS: BW))
5883 return false;
5884 } else {
5885 // Check that the expression overall uses at most the same number of bits as
5886 // ZExted
5887 KnownBits KB = computeKnownBits(V: &I, DL: *DL);
5888 if (KB.countMaxActiveBits() > BW)
5889 return false;
5890 }
5891
5892 // Calculate costs of leaving current IR as it is and moving ZExt operation
5893 // later, along with adding truncates if needed
5894 InstructionCost ZExtCost = TTI.getCastInstrCost(
5895 Opcode: Instruction::ZExt, Dst: BigTy, Src: SmallTy,
5896 CCH: TargetTransformInfo::CastContextHint::None, CostKind);
5897 InstructionCost CurrentCost = ZExtCost;
5898 InstructionCost ShrinkCost = 0;
5899
5900 // Calculate total cost and check that we can propagate through all ZExt users
5901 for (User *U : ZExtOperand->users()) {
5902 auto *UI = cast<Instruction>(Val: U);
5903 if (UI == &I) {
5904 CurrentCost +=
5905 TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: BigTy, CostKind);
5906 ShrinkCost +=
5907 TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: SmallTy, CostKind);
5908 ShrinkCost += ZExtCost;
5909 continue;
5910 }
5911
5912 if (!Instruction::isBinaryOp(Opcode: UI->getOpcode()))
5913 return false;
5914
5915 // Check if we can propagate ZExt through its other users
5916 KnownBits KB = computeKnownBits(V: UI, DL: *DL);
5917 if (KB.countMaxActiveBits() > BW)
5918 return false;
5919
5920 CurrentCost += TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: BigTy, CostKind);
5921 ShrinkCost +=
5922 TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: SmallTy, CostKind);
5923 ShrinkCost += ZExtCost;
5924 }
5925
5926 // If the other instruction operand is not a constant, we'll need to
5927 // generate a truncate instruction. So we have to adjust cost
5928 if (!isa<Constant>(Val: OtherOperand))
5929 ShrinkCost += TTI.getCastInstrCost(
5930 Opcode: Instruction::Trunc, Dst: SmallTy, Src: BigTy,
5931 CCH: TargetTransformInfo::CastContextHint::None, CostKind);
5932
5933 // If the cost of shrinking types and leaving the IR is the same, we'll lean
5934 // towards modifying the IR because shrinking opens opportunities for other
5935 // shrinking optimisations.
5936 if (ShrinkCost > CurrentCost)
5937 return false;
5938
5939 Builder.SetInsertPoint(&I);
5940 Value *Op0 = ZExted;
5941 Value *Op1 = Builder.CreateTrunc(V: OtherOperand, DestTy: SmallTy);
5942 // Keep the order of operands the same
5943 if (I.getOperand(i: 0) == OtherOperand)
5944 std::swap(a&: Op0, b&: Op1);
5945 Value *NewBinOp =
5946 Builder.CreateBinOp(Opc: (Instruction::BinaryOps)I.getOpcode(), LHS: Op0, RHS: Op1);
5947 if (auto *NewBinOpI = dyn_cast<Instruction>(Val: NewBinOp)) {
5948 NewBinOpI->copyIRFlags(V: &I);
5949 NewBinOpI->copyMetadata(SrcInst: I);
5950 }
5951 Value *NewZExtr = Builder.CreateZExt(V: NewBinOp, DestTy: BigTy);
5952 replaceValue(Old&: I, New&: *NewZExtr);
5953 return true;
5954}
5955
5956/// insert (DstVec, (extract SrcVec, ExtIdx), InsIdx) -->
5957/// shuffle (DstVec, SrcVec, Mask)
5958bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
5959 Value *DstVec, *SrcVec;
5960 uint64_t ExtIdx, InsIdx;
5961 if (!match(V: &I,
5962 P: m_InsertElt(Val: m_Value(V&: DstVec),
5963 Elt: m_ExtractElt(Val: m_Value(V&: SrcVec), Idx: m_ConstantInt(V&: ExtIdx)),
5964 Idx: m_ConstantInt(V&: InsIdx))))
5965 return false;
5966
5967 auto *DstVecTy = dyn_cast<FixedVectorType>(Val: I.getType());
5968 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcVec->getType());
5969 // We can try combining vectors with different element sizes.
5970 if (!DstVecTy || !SrcVecTy ||
5971 SrcVecTy->getElementType() != DstVecTy->getElementType())
5972 return false;
5973
5974 unsigned NumDstElts = DstVecTy->getNumElements();
5975 unsigned NumSrcElts = SrcVecTy->getNumElements();
5976 if (InsIdx >= NumDstElts || ExtIdx >= NumSrcElts || NumDstElts == 1)
5977 return false;
5978
5979 // Insertion into poison is a cheaper single operand shuffle.
5980 TargetTransformInfo::ShuffleKind SK;
5981 SmallVector<int> Mask(NumDstElts, PoisonMaskElem);
5982
5983 bool NeedExpOrNarrow = NumSrcElts != NumDstElts;
5984 bool NeedDstSrcSwap = isa<PoisonValue>(Val: DstVec) && !isa<UndefValue>(Val: SrcVec);
5985 if (NeedDstSrcSwap) {
5986 SK = TargetTransformInfo::SK_PermuteSingleSrc;
5987 Mask[InsIdx] = ExtIdx % NumDstElts;
5988 std::swap(a&: DstVec, b&: SrcVec);
5989 } else {
5990 SK = TargetTransformInfo::SK_PermuteTwoSrc;
5991 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
5992 Mask[InsIdx] = (ExtIdx % NumDstElts) + NumDstElts;
5993 }
5994
5995 // Cost
5996 auto *Ins = cast<InsertElementInst>(Val: &I);
5997 auto *Ext = cast<ExtractElementInst>(Val: I.getOperand(i: 1));
5998 InstructionCost InsCost =
5999 TTI.getVectorInstrCost(I: *Ins, Val: DstVecTy, CostKind, Index: InsIdx);
6000 InstructionCost ExtCost =
6001 TTI.getVectorInstrCost(I: *Ext, Val: DstVecTy, CostKind, Index: ExtIdx);
6002 InstructionCost OldCost = ExtCost + InsCost;
6003
6004 InstructionCost NewCost = 0;
6005 SmallVector<int> ExtToVecMask;
6006 if (!NeedExpOrNarrow) {
6007 // Ignore 'free' identity insertion shuffle.
6008 // TODO: getShuffleCost should return TCC_Free for Identity shuffles.
6009 if (!ShuffleVectorInst::isIdentityMask(Mask, NumSrcElts))
6010 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: DstVecTy, CostKind, Mask, Index: 0,
6011 SubTp: nullptr, Args: {DstVec, SrcVec});
6012 } else {
6013 // When creating a length-changing-vector, always try to keep the relevant
6014 // element in an equivalent position, so that bulk shuffles are more likely
6015 // to be useful.
6016 ExtToVecMask.assign(NumElts: NumDstElts, Elt: PoisonMaskElem);
6017 ExtToVecMask[ExtIdx % NumDstElts] = ExtIdx;
6018 // Add cost for expanding or narrowing
6019 NewCost = TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
6020 DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask: ExtToVecMask);
6021 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: DstVecTy, CostKind, Mask);
6022 }
6023
6024 if (!Ext->hasOneUse())
6025 NewCost += ExtCost;
6026
6027 LLVM_DEBUG(dbgs() << "Found a insert/extract shuffle-like pair: " << I
6028 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
6029 << "\n");
6030
6031 if (OldCost < NewCost)
6032 return false;
6033
6034 if (NeedExpOrNarrow) {
6035 if (!NeedDstSrcSwap)
6036 SrcVec = Builder.CreateShuffleVector(V: SrcVec, Mask: ExtToVecMask);
6037 else
6038 DstVec = Builder.CreateShuffleVector(V: DstVec, Mask: ExtToVecMask);
6039 }
6040
6041 // Canonicalize undef param to RHS to help further folds.
6042 if (isa<UndefValue>(Val: DstVec) && !isa<UndefValue>(Val: SrcVec)) {
6043 ShuffleVectorInst::commuteShuffleMask(Mask, InVecNumElts: NumDstElts);
6044 std::swap(a&: DstVec, b&: SrcVec);
6045 }
6046
6047 Value *Shuf = Builder.CreateShuffleVector(V1: DstVec, V2: SrcVec, Mask);
6048 replaceValue(Old&: I, New&: *Shuf);
6049
6050 return true;
6051}
6052
6053/// Try to replace a chain of insertelements of parts of the same scalar with a
6054/// bitcast and a shuffle (little endian):
6055/// insert (insert poison, (trunc (lshr X, 32)), 0), (trunc X), 1 -->
6056/// shuffle (bitcast X to <2 x i32>), poison, <1, 0>
6057bool VectorCombine::foldInsertScalarPartsToShuffle(Instruction &I) {
6058 auto *VecTy = dyn_cast<FixedVectorType>(Val: I.getType());
6059 if (!VecTy)
6060 return false;
6061
6062 // Start from the last insertelement of the chain.
6063 if (I.hasOneUse() && isa<InsertElementInst>(Val: I.user_back()))
6064 return false;
6065
6066 Type *EltTy = VecTy->getElementType();
6067 if ((!EltTy->isIntegerTy() && !EltTy->isIEEELikeFPTy()) ||
6068 !DL->typeSizeEqualsStoreSize(Ty: EltTy))
6069 return false;
6070 unsigned EltBits = EltTy->getPrimitiveSizeInBits();
6071 unsigned NumElts = VecTy->getNumElements();
6072
6073 Value *Src = nullptr;
6074 unsigned NumSrcElts = 0;
6075 SmallVector<int> Mask(NumElts, PoisonMaskElem);
6076 APInt DemandedElts = APInt::getZero(numBits: NumElts);
6077 InstructionCost OldCost = 0;
6078 Value *Vec = &I;
6079 while (auto *Ins = dyn_cast<InsertElementInst>(Val: Vec)) {
6080 if (Ins != &I && !Ins->hasOneUse())
6081 return false;
6082 uint64_t Idx;
6083 if (!match(V: Ins->getOperand(i_nocapture: 2), P: m_ConstantInt(V&: Idx)) || Idx >= NumElts)
6084 return false;
6085 Vec = Ins->getOperand(i_nocapture: 0);
6086 // A later insert to the same element overrides this one.
6087 if (DemandedElts[Idx])
6088 continue;
6089 DemandedElts.setBit(Idx);
6090
6091 // Match (bitcast (trunc (lshr X, ShAmt))), the bitcast and shift being
6092 // optional.
6093 Value *Elt = Ins->getOperand(i_nocapture: 1);
6094 Value *Trunc = Elt;
6095 match(V: Trunc, P: m_BitCast(Op: m_Value(V&: Trunc)));
6096 Value *X;
6097 if (!match(V: Trunc, P: m_Trunc(Op: m_Value(V&: X))) || !X->getType()->isIntegerTy() ||
6098 Trunc->getType()->getPrimitiveSizeInBits() != EltBits)
6099 return false;
6100 Value *Shift = nullptr;
6101 uint64_t ShAmt = 0;
6102 if (match(V: X, P: m_LShr(L: m_Value(), R: m_ConstantInt(V&: ShAmt)))) {
6103 Shift = X;
6104 X = cast<Instruction>(Val: Shift)->getOperand(i: 0);
6105 }
6106
6107 if (!Src) {
6108 unsigned SrcBits = X->getType()->getIntegerBitWidth();
6109 if (SrcBits % EltBits)
6110 return false;
6111 Src = X;
6112 NumSrcElts = SrcBits / EltBits;
6113 } else if (X != Src) {
6114 return false;
6115 }
6116 uint64_t Part = ShAmt / EltBits;
6117 if (ShAmt % EltBits || Part >= NumSrcElts)
6118 return false;
6119 Mask[Idx] = DL->isBigEndian() ? NumSrcElts - 1 - Part : Part;
6120
6121 // The scalar ops die with the chain if it is their only user.
6122 for (Value *V : {Elt == Trunc ? nullptr : Elt, Trunc, Shift}) {
6123 if (!V)
6124 continue;
6125 if (!V->hasOneUse())
6126 break;
6127 OldCost += TTI.getInstructionCost(U: cast<Instruction>(Val: V), CostKind);
6128 }
6129 }
6130 // Elements that are not inserted become poison, so the base must be poison
6131 // unless every element is inserted.
6132 if (!Src || (!isa<PoisonValue>(Val: Vec) && !DemandedElts.isAllOnes()))
6133 return false;
6134 // Inserting a single part is a scalar insert or a splat, whose canonical
6135 // insertelement (+ splat shuffle) form is better left alone.
6136 if (all_equal(
6137 Range: make_filter_range(Range&: Mask, Pred: [](int M) { return M != PoisonMaskElem; })))
6138 return false;
6139
6140 OldCost += TTI.getScalarizationOverhead(Ty: VecTy, DemandedElts, /*Insert=*/true,
6141 /*Extract=*/false, CostKind);
6142
6143 auto *SrcVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: NumSrcElts);
6144 InstructionCost NewCost =
6145 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: SrcVecTy, Src: Src->getType(),
6146 CCH: TTI::CastContextHint::None, CostKind);
6147 bool IsIdentity = NumSrcElts == NumElts &&
6148 ShuffleVectorInst::isIdentityMask(Mask, NumSrcElts);
6149 if (!IsIdentity)
6150 NewCost += TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: VecTy, SrcTy: SrcVecTy,
6151 CostKind, Mask);
6152
6153 LLVM_DEBUG(dbgs() << "Found an insertelement chain of scalar parts: " << I
6154 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
6155 << "\n");
6156 if (!OldCost.isValid() || !NewCost.isValid() || NewCost >= OldCost)
6157 return false;
6158
6159 Value *Cast = Builder.CreateBitCast(V: Src, DestTy: SrcVecTy);
6160 Value *Shuf = IsIdentity ? Cast : Builder.CreateShuffleVector(V: Cast, Mask);
6161 replaceValue(Old&: I, New&: *Shuf);
6162 return true;
6163}
6164
6165/// Return the number of data operands of \p Inst.
6166static unsigned getNumDataOperands(const Instruction *Inst) {
6167 if (auto *CB = dyn_cast<CallBase>(Val: Inst))
6168 return CB->arg_size(); // Exclude callee operand and bundles.
6169 return Inst->getNumOperands();
6170}
6171
6172/// Return true if \p Inst is an elementwise operation that can be rebuilt at a
6173/// wider element count.
6174static bool isSupportedElementwise(Instruction *Inst) {
6175 auto *ResultTy = dyn_cast<VectorType>(Val: Inst->getType());
6176 if (!ResultTy || !isSafeToSpeculativelyExecute(I: Inst))
6177 return false;
6178
6179 if (auto *II = dyn_cast<IntrinsicInst>(Val: Inst)) {
6180 if (II->hasOperandBundles() ||
6181 !isTriviallyVectorizable(ID: II->getIntrinsicID()))
6182 return false;
6183 } else if (!isa<BinaryOperator, UnaryOperator, CastInst, CmpInst, SelectInst,
6184 FreezeInst>(Val: Inst)) {
6185 return false;
6186 }
6187
6188 // Reject operations that change the element-count.
6189 // E.g., bitcast <vscale x 4 x i16> %v to <vscale x 8 x i8>
6190 for (unsigned Op = 0, E = getNumDataOperands(Inst); Op != E; ++Op) {
6191 auto *OperandTy = dyn_cast<VectorType>(Val: Inst->getOperand(i: Op)->getType());
6192 if (OperandTy &&
6193 OperandTy->getElementCount() != ResultTy->getElementCount())
6194 return false;
6195 }
6196
6197 return true;
6198}
6199
6200/// Return the common splat value of \p Values.
6201static Value *getCommonSplatValue(ArrayRef<Value *> Values) {
6202 auto GetSplatOrScalar = [](Value *V) {
6203 return isa<VectorType>(Val: V->getType()) ? getSplatValue(V) : V;
6204 };
6205
6206 Value *CommonValue = GetSplatOrScalar(Values.front());
6207 if (!CommonValue)
6208 return nullptr;
6209 for (Value *V : Values.drop_front())
6210 if (GetSplatOrScalar(V) != CommonValue)
6211 return nullptr;
6212 return CommonValue;
6213}
6214
6215/// Return the common deinterleave intrinsic if \p Members are its extracts in
6216/// field order.
6217static IntrinsicInst *getCommonDeinterleavedSource(ArrayRef<Value *> Members) {
6218 IntrinsicInst *Deinterleave = nullptr;
6219 for (const auto &[Index, Member] : enumerate(First&: Members)) {
6220 auto *Extract = dyn_cast<ExtractValueInst>(Val: Member);
6221 if (!Extract || Extract->getNumIndices() != 1 ||
6222 *Extract->idx_begin() != Index)
6223 return nullptr;
6224
6225 auto *Current = dyn_cast<IntrinsicInst>(Val: Extract->getAggregateOperand());
6226 if (!Current || (Deinterleave && Current != Deinterleave))
6227 return nullptr;
6228 Deinterleave = Current;
6229 }
6230 unsigned Factor = Members.size();
6231 if (Deinterleave->hasOperandBundles() ||
6232 getDeinterleaveIntrinsicFactor(ID: Deinterleave->getIntrinsicID()) !=
6233 Factor ||
6234 !Deinterleave->hasNUndroppableUses(N: Factor))
6235 return nullptr;
6236 return Deinterleave;
6237}
6238
6239/// Return the operand at \p OperandIndex of each \p Members.
6240static SmallVector<Value *, 8> getInstrOperandsAtIdx(ArrayRef<Value *> Members,
6241 unsigned OperandIndex) {
6242 SmallVector<Value *, 8> Operands;
6243 for (Value *Member : Members)
6244 Operands.push_back(Elt: cast<Instruction>(Val: Member)->getOperand(i: OperandIndex));
6245 return Operands;
6246}
6247
6248/// Check whether the tree of elementwise operations each feeding \p Members
6249/// can be rebuilt at the interleaved width.
6250static bool canWidenDeinterleavedOperations(const VectorizeOptions &Opts,
6251 ArrayRef<Value *> Members,
6252 unsigned &NumScanned) {
6253 unsigned Factor = Members.size();
6254 if (getCommonDeinterleavedSource(Members))
6255 return true;
6256 if (NumScanned + Factor > Opts.vector_combine_max_scan_instrs)
6257 return false;
6258 NumScanned += Factor;
6259
6260 auto *FirstInst = dyn_cast<Instruction>(Val: Members.front());
6261 if (!FirstInst || !isSupportedElementwise(Inst: FirstInst) ||
6262 !FirstInst->getSingleUndroppableUse())
6263 return false;
6264
6265 for (Value *Member : Members.drop_front()) {
6266 auto *Inst = dyn_cast<Instruction>(Val: Member);
6267 if (!Inst || !Inst->getSingleUndroppableUse() ||
6268 !FirstInst->isSameOperationAs(I: Inst, flags: Instruction::CompareCallTargets))
6269 return false;
6270 }
6271
6272 // Scalars operands should be equal among all members.
6273 // Vector operands should be a common splat value or can be widened.
6274 for (unsigned Op = 0, E = getNumDataOperands(Inst: FirstInst); Op != E; ++Op) {
6275 SmallVector<Value *, 8> Operands = getInstrOperandsAtIdx(Members, OperandIndex: Op);
6276 if (!getCommonSplatValue(Values: Operands) &&
6277 !canWidenDeinterleavedOperations(Opts, Members: Operands, NumScanned))
6278 return false;
6279 }
6280 return true;
6281}
6282
6283static Value *createWideInstruction(Instruction *NarrowInst,
6284 ArrayRef<Value *> NewOperands,
6285 VectorType *WideResultTy,
6286 IRBuilderBase &Builder) {
6287 if (isa<BinaryOperator, UnaryOperator>(Val: NarrowInst))
6288 return Builder.CreateNAryOp(Opc: NarrowInst->getOpcode(), Ops: NewOperands);
6289 if (auto *Cast = dyn_cast<CastInst>(Val: NarrowInst))
6290 return Builder.CreateCast(Op: Cast->getOpcode(), V: NewOperands[0], DestTy: WideResultTy);
6291 if (auto *Cmp = dyn_cast<CmpInst>(Val: NarrowInst))
6292 return Builder.CreateCmp(Pred: Cmp->getPredicate(), LHS: NewOperands[0],
6293 RHS: NewOperands[1]);
6294 if (isa<SelectInst>(Val: NarrowInst))
6295 return Builder.CreateSelect(
6296 C: NewOperands[0], True: NewOperands[1], False: NewOperands[2], /*Name=*/"",
6297 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : NarrowInst);
6298 if (isa<FreezeInst>(Val: NarrowInst))
6299 return Builder.CreateFreeze(V: NewOperands[0]);
6300 if (auto *II = dyn_cast<IntrinsicInst>(Val: NarrowInst))
6301 return Builder.CreateIntrinsic(RetTy: WideResultTy, ID: II->getIntrinsicID(),
6302 Args: NewOperands);
6303 llvm_unreachable("Unsupported instruction");
6304}
6305
6306static Value *widenDeinterleavedOperations(ArrayRef<Value *> Members,
6307 ElementCount WideEC,
6308 IRBuilderBase &Builder) {
6309 if (auto *Deinterleave = getCommonDeinterleavedSource(Members)) {
6310 Value *Source = Deinterleave->getArgOperand(i: 0);
6311 assert(cast<VectorType>(Source->getType())->getElementCount() == WideEC &&
6312 "deinterleave source must have the interleaved element count");
6313 return Source;
6314 }
6315
6316 auto *NarrowInst = cast<Instruction>(Val: Members.front());
6317 unsigned NumOperands = getNumDataOperands(Inst: NarrowInst);
6318 SmallVector<Value *, 4> NewOperands;
6319 NewOperands.reserve(N: NumOperands);
6320 for (unsigned Op = 0; Op != NumOperands; ++Op) {
6321 SmallVector<Value *, 8> Operands = getInstrOperandsAtIdx(Members, OperandIndex: Op);
6322 Value *NewOperand = nullptr;
6323 if ((NewOperand = getCommonSplatValue(Values: Operands))) {
6324 if (isa<VectorType>(Val: Operands.front()->getType())) {
6325 Builder.SetCurrentDebugLocation(NarrowInst->getDebugLoc());
6326 NewOperand = Builder.CreateVectorSplat(EC: WideEC, V: NewOperand);
6327 } else {
6328 assert(all_equal(Operands) && "expected all operands to be equal");
6329 }
6330 } else {
6331 NewOperand = widenDeinterleavedOperations(Members: Operands, WideEC, Builder);
6332 }
6333 NewOperands.push_back(Elt: NewOperand);
6334 }
6335
6336 Builder.SetCurrentDebugLocation(NarrowInst->getDebugLoc());
6337 auto *WideResultTy =
6338 VectorType::get(ElementType: NarrowInst->getType()->getScalarType(), EC: WideEC);
6339 Value *NewValue =
6340 createWideInstruction(NarrowInst, NewOperands, WideResultTy, Builder);
6341 if (auto *NewInst = dyn_cast<Instruction>(Val: NewValue)) {
6342 propagateIRFlags(I: NewInst, VL: Members);
6343 propagateMetadata(I: NewInst, VL: Members);
6344 }
6345 return NewValue;
6346}
6347
6348/// Fold away vector.deinterleave/interleave intrinsics with matching trees of
6349/// elementwise operations between them.
6350///
6351/// For example:
6352/// ```
6353/// %d = call { <2 x i16>, <2 x i16> } @deinterleave2.v4i16(<4 x i16> %v)
6354/// %f0 = extractvalue { <2 x i16>, <2 x i16> } %d, 0
6355/// %f1 = extractvalue { <2 x i16>, <2 x i16> } %d, 1
6356///
6357/// %u0 = add <2 x i16> %f0, splat (i16 3)
6358/// %u1 = add <2 x i16> %f1, splat (i16 3)
6359///
6360/// %r = call <4 x i16> @interleave2.v4i16(<2 x i16> %u0, <2 x i16> %u1)
6361/// ```
6362/// Folds to:
6363/// ```
6364/// %r = add <4 x i16> %v, splat (i16 3)
6365/// ```
6366/// And with two sources:
6367/// ```
6368/// %da = call { <2 x i16>, <2 x i16> } @deinterleave2.v4i16(<4 x i16> %a)
6369/// %a0 = extractvalue { <2 x i16>, <2 x i16> } %da, 0
6370/// %a1 = extractvalue { <2 x i16>, <2 x i16> } %da, 1
6371/// %db = call { <2 x i16>, <2 x i16> } @deinterleave2.v4i16(<4 x i16> %b)
6372/// %b0 = extractvalue { <2 x i16>, <2 x i16> } %db, 0
6373/// %b1 = extractvalue { <2 x i16>, <2 x i16> } %db, 1
6374///
6375/// %m0 = mul <2 x i16> %a0, %b0
6376/// %m1 = mul <2 x i16> %a1, %b1
6377///
6378/// %r = call <4 x i16> @interleave2.v4i16(<2 x i16> %m0, <2 x i16> %m1)
6379/// ```
6380/// Folds to:
6381/// ```
6382/// %r = mul <4 x i16> %a, %b
6383/// ```
6384bool VectorCombine::foldInterleaveOfDeinterleaveChains(Instruction &I) {
6385 auto *Interleave = dyn_cast<IntrinsicInst>(Val: &I);
6386 if (!Interleave)
6387 return false;
6388
6389 unsigned Factor = getInterleaveIntrinsicFactor(ID: Interleave->getIntrinsicID());
6390 if (!Factor || Interleave->hasOperandBundles())
6391 return false;
6392
6393 SmallVector<Value *, 8> RootMembers(Interleave->args());
6394 unsigned NumScanned = 0;
6395 if (!canWidenDeinterleavedOperations(Opts, Members: RootMembers, NumScanned))
6396 return false;
6397
6398 ElementCount WideEC = cast<VectorType>(Val: I.getType())->getElementCount();
6399 Builder.SetInsertPoint(Interleave);
6400 Value *WideValue = widenDeinterleavedOperations(Members: RootMembers, WideEC, Builder);
6401 assert(WideValue->getType() == Interleave->getType());
6402 replaceValue(Old&: *Interleave, New&: *WideValue);
6403 return true;
6404}
6405
6406/// If we're interleaving 2 constant splats, for instance `<vscale x 8 x i32>
6407/// <splat of 666>` and `<vscale x 8 x i32> <splat of 777>`, we can create a
6408/// larger splat `<vscale x 8 x i64> <splat of ((777 << 32) | 666)>` first
6409/// before casting it back into `<vscale x 16 x i32>`.
6410bool VectorCombine::foldInterleaveIntrinsics(Instruction &I) {
6411 if (foldInterleaveOfDeinterleaveChains(I))
6412 return true;
6413
6414 const APInt *SplatVal0, *SplatVal1;
6415 if (!match(V: &I, P: m_Intrinsic<Intrinsic::vector_interleave2>(
6416 Ops: m_APInt(Res&: SplatVal0), Ops: m_APInt(Res&: SplatVal1))))
6417 return false;
6418
6419 LLVM_DEBUG(dbgs() << "VC: Folding interleave2 with two splats: " << I
6420 << "\n");
6421
6422 auto *VTy =
6423 cast<VectorType>(Val: cast<IntrinsicInst>(Val&: I).getArgOperand(i: 0)->getType());
6424 auto *ExtVTy = VectorType::getExtendedElementVectorType(VTy);
6425 unsigned Width = VTy->getElementType()->getIntegerBitWidth();
6426
6427 // Just in case the cost of interleave2 intrinsic and bitcast are both
6428 // invalid, in which case we want to bail out, we use <= rather
6429 // than < here. Even they both have valid and equal costs, it's probably
6430 // not a good idea to emit a high-cost constant splat.
6431 if (TTI.getInstructionCost(U: &I, CostKind) <=
6432 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: I.getType(), Src: ExtVTy,
6433 CCH: TTI::CastContextHint::None, CostKind)) {
6434 LLVM_DEBUG(dbgs() << "VC: The cost to cast from " << *ExtVTy << " to "
6435 << *I.getType() << " is too high.\n");
6436 return false;
6437 }
6438
6439 // The bitcast puts element 0 of the narrow vector in the low half of each
6440 // wide element on little-endian targets, and in the high half on big-endian
6441 // targets.
6442 const APInt &LowHalf = DL->isLittleEndian() ? *SplatVal0 : *SplatVal1;
6443 const APInt &HighHalf = DL->isLittleEndian() ? *SplatVal1 : *SplatVal0;
6444 APInt NewSplatVal = HighHalf.zext(width: Width * 2);
6445 NewSplatVal <<= Width;
6446 NewSplatVal |= LowHalf.zext(width: Width * 2);
6447 auto *NewSplat = ConstantVector::getSplat(
6448 EC: ExtVTy->getElementCount(), Elt: ConstantInt::get(Context&: F.getContext(), V: NewSplatVal));
6449
6450 Builder.SetInsertPoint(&I);
6451 replaceValue(Old&: I, New&: *Builder.CreateBitCast(V: NewSplat, DestTy: I.getType()));
6452 return true;
6453}
6454
6455/// Given this sequence:
6456/// ```
6457/// %d = llvm.vector.deinterleave2 <vscale x 16 x i32> %v
6458/// %f0 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d, 0
6459/// %f1 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d, 1
6460///
6461/// %low0 = and <vscale x 8 x i32> %f0, splat (i32 65535)
6462/// %low1 = shl <vscale x 8 x i32> %f1, splat (i32 16)
6463/// %merge0 = or disjoint <vscale x 8 x i32> %low0, %low1
6464///
6465/// %high0 = and <vscale x 8 x i32> %f1, splat (i32 -65536)
6466/// %high1 = lshr <vscale x 8 x i32> %f0, splat (i32 16)
6467/// %merge1 = or disjoint <vscale x 8 x i32> %high0, %high1
6468/// ```
6469/// It is actually just de-interleaving a 16-bit vector with double the
6470/// vector length. More generally speaking, it's de-interleaving on a vector
6471/// with half the element width as the original vector.
6472///
6473/// Therefore, we can turn it into:
6474/// ```
6475/// %narrow.v = bitcast <vscale x 16 x i32> %v to <vscale x 32 x i16>
6476/// %d = llvm.vector.deinterleave2 <vscale x 32 x i16> %narrow.v
6477/// %f0 = extractvalue { <vscale x 16 x i16>, <vscale x 16 x i16> } %d, 0
6478/// %f1 = extractvalue { <vscale x 16 x i16>, <vscale x 16 x i16> } %d, 1
6479///
6480/// %merge0 = bitcast <vscale x 16 x i16> %f0 to <vscale x 8 x i32>
6481/// %merge1 = bitcast <vscale x 16 x i16> %f1 to <vscale x 8 x i32>
6482/// ```
6483bool VectorCombine::foldDeinterleaveIntrinsics(Instruction &I) {
6484 // This pattern involves bitcast that is not compatible with big endian.
6485 if (DL->isBigEndian())
6486 return false;
6487
6488 using namespace PatternMatch;
6489 Value *DeinterleavedVal;
6490 if (!match(V: &I, P: m_Deinterleave2(Op: m_Value(V&: DeinterleavedVal))))
6491 return false;
6492
6493 VectorType *VecTy = cast<VectorType>(Val: DeinterleavedVal->getType());
6494 IntegerType *ElementTy = dyn_cast<IntegerType>(Val: VecTy->getElementType());
6495 if (!ElementTy)
6496 return false;
6497 unsigned ElementWidth = ElementTy->getBitWidth();
6498 if (ElementWidth < 2 || !isPowerOf2_32(Value: ElementWidth))
6499 return false;
6500 unsigned HalfElementWidth = ElementWidth / 2;
6501
6502 if (!I.hasNUses(N: 2))
6503 return false;
6504 std::array<ExtractValueInst *, 2> OrigFields{};
6505 for (User *Usr : I.users()) {
6506 auto *E = dyn_cast<ExtractValueInst>(Val: Usr);
6507 // The deinterleave result can only be used by extractions.
6508 if (!E || E->getNumIndices() != 1)
6509 return false;
6510 unsigned Idx = *E->idx_begin();
6511 // A single field cannot be extracted more than once.
6512 if (Idx >= 2 || OrigFields[Idx] || !E->hasNUses(N: 2))
6513 return false;
6514 OrigFields[Idx] = E;
6515 }
6516
6517 // Find the merge instruction (i.e. OR) first.
6518 SmallVector<Instruction *, 2> MergeInsts;
6519 for (auto *FieldUsr : OrigFields[0]->users()) {
6520 if (!FieldUsr->hasOneUse() || !isa<Instruction>(Val: FieldUsr->user_back()))
6521 return false;
6522 MergeInsts.push_back(Elt: cast<Instruction>(Val: FieldUsr->user_back()));
6523 }
6524 assert(MergeInsts.size() == 2);
6525
6526 // Pattern match bottom-up from the merge instructions.
6527 auto MatchMerge = [&](void) -> bool {
6528 APInt LoMask = APInt::getLowBitsSet(numBits: ElementWidth, loBitsSet: HalfElementWidth);
6529 APInt HiMask = APInt::getHighBitsSet(numBits: ElementWidth, hiBitsSet: HalfElementWidth);
6530 return match(V: MergeInsts[0],
6531 P: m_c_Or(L: m_And(L: m_Specific(V: OrigFields[0]), R: m_SpecificInt(V: LoMask)),
6532 R: m_Shl(L: m_Specific(V: OrigFields[1]),
6533 R: m_SpecificInt(V: HalfElementWidth)))) &&
6534 match(V: MergeInsts[1],
6535 P: m_c_Or(L: m_And(L: m_Specific(V: OrigFields[1]), R: m_SpecificInt(V: HiMask)),
6536 R: m_LShr(L: m_Specific(V: OrigFields[0]),
6537 R: m_SpecificInt(V: HalfElementWidth))));
6538 };
6539 if (!MatchMerge()) {
6540 std::swap(a&: MergeInsts[0], b&: MergeInsts[1]);
6541 if (!MatchMerge())
6542 return false;
6543 }
6544
6545 // Profitability check.
6546 InstructionCost OldCost =
6547 TTI.getInstructionCost(U: MergeInsts[0], CostKind) +
6548 TTI.getInstructionCost(U: cast<Instruction>(Val: MergeInsts[0]->getOperand(i: 0)),
6549 CostKind) +
6550 TTI.getInstructionCost(U: cast<Instruction>(Val: MergeInsts[0]->getOperand(i: 1)),
6551 CostKind);
6552 // There are two fields (assuming SHL has the same cost as LSHR).
6553 OldCost *= 2;
6554
6555 auto *NewFieldTy = VecTy->getWithNewBitWidth(NewBitWidth: HalfElementWidth);
6556 auto *NewVecTy =
6557 VectorType::getDoubleElementsVectorType(VTy: cast<VectorType>(Val: NewFieldTy));
6558 InstructionCost NewCost =
6559 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: VecTy, Src: NewVecTy,
6560 CCH: TTI::CastContextHint::None, CostKind) +
6561 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: NewFieldTy,
6562 Src: MergeInsts[0]->getType(), CCH: TTI::CastContextHint::None,
6563 CostKind) *
6564 2;
6565 if (OldCost <= NewCost || !NewCost.isValid()) {
6566 LLVM_DEBUG(
6567 dbgs() << "VC: New deinterleave2 sequence cost (" << NewCost << ")"
6568 << " is higher than that of the old one (" << OldCost << ")\n");
6569 return false;
6570 }
6571
6572 // Do the replacement.
6573 Builder.SetInsertPoint(&I);
6574 Value *NewVecCast = Builder.CreateBitCast(V: DeinterleavedVal, DestTy: NewVecTy);
6575 Value *NewDeinterleave = Builder.CreateIntrinsic(
6576 ID: Intrinsic::vector_deinterleave2, OverloadTypes: {NewVecTy}, Args: {NewVecCast});
6577 for (auto [Idx, MergeInst] : enumerate(First&: MergeInsts)) {
6578 Value *NewField = Builder.CreateExtractValue(Agg: NewDeinterleave, Idxs: Idx);
6579 NewField = Builder.CreateBitCast(V: NewField, DestTy: MergeInst->getType());
6580 replaceValue(Old&: *MergeInst, New&: *NewField);
6581 }
6582
6583 return true;
6584}
6585
6586bool VectorCombine::foldBitcastOfVPLoad(Instruction &I) {
6587 const DataLayout &DL = I.getDataLayout();
6588 auto *Cast = dyn_cast<CastInst>(Val: &I);
6589 if (!Cast || !Cast->isNoopCast(DL) || !isa<VectorType>(Val: Cast->getDestTy()))
6590 return false;
6591
6592 // Fold away bit casts of the loaded value by loading the desired type,
6593 // if the mask is all-ones.
6594 Value *EVL;
6595 auto *II = dyn_cast<VPIntrinsic>(Val: I.getOperand(i: 0));
6596 if (!II || !match(V: II, P: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::vp_load>(
6597 Ops: m_Value(), Ops: m_AllOnes(), Ops: m_Value(V&: EVL)))))
6598 return false;
6599
6600 VectorType *OrigVecTy = cast<VectorType>(Val: II->getType());
6601 Align OrigAlign =
6602 DL.getValueOrABITypeAlignment(Alignment: II->getPointerAlignment(), Ty: OrigVecTy);
6603 ElementCount OrigVecCnt = OrigVecTy->getElementCount();
6604 VectorType *NewVecTy = cast<VectorType>(Val: Cast->getDestTy());
6605 ElementCount NewVecCnt = NewVecTy->getElementCount();
6606
6607 // Right now we only support cases where the NewVec is longer, because for
6608 // cases where it's shorter, we have to be sure that EVL can be exactly
6609 // divided, otherwise it might yield incorrect results or even page faults
6610 // (if we round-up during the division).
6611 if (!(OrigVecCnt.isScalable() == NewVecCnt.isScalable() &&
6612 NewVecCnt.hasKnownScalarFactor(RHS: OrigVecCnt)))
6613 return false;
6614
6615 InstructionCost OldCost =
6616 TTI.getMemIntrinsicInstrCost(MICA: {Intrinsic::vp_load, OrigVecTy,
6617 II->getMemoryPointerParam(), false,
6618 OrigAlign},
6619 CostKind) +
6620 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: Cast->getType(), Src: OrigVecTy,
6621 CCH: TTI::CastContextHint::None, CostKind);
6622 InstructionCost NewCost = TTI.getMemIntrinsicInstrCost(
6623 MICA: {Intrinsic::vp_load, NewVecTy, II->getMemoryPointerParam(), false,
6624 OrigAlign},
6625 CostKind);
6626 LLVM_DEBUG(dbgs() << "foldBitcastOfVPLoad: OldCost=" << OldCost
6627 << " NewCost=" << NewCost << "\n");
6628 if (NewCost > OldCost || !NewCost.isValid())
6629 return false;
6630
6631 Builder.SetInsertPoint(II);
6632 unsigned Factor = NewVecCnt.getKnownScalarFactor(RHS: OrigVecCnt);
6633 Value *NewEVL = Builder.CreateNUWMul(LHS: EVL, RHS: Builder.getInt32(C: Factor));
6634 Value *NewMask = Builder.CreateVectorSplat(EC: NewVecCnt, V: Builder.getTrue());
6635 CallInst *NewVP = Builder.CreateIntrinsicWithoutFolding(
6636 RetTy: NewVecTy, ID: Intrinsic::vp_load,
6637 Args: {II->getMemoryPointerParam(), NewMask, NewEVL});
6638 // Preserve the original alignment.
6639 NewVP->addParamAttrs(
6640 ArgNo: 0, B: AttrBuilder(II->getContext()).addAlignmentAttr(Align: OrigAlign));
6641 replaceValue(Old&: *Cast, New&: *NewVP);
6642 return true;
6643}
6644/// Fold the following cases into a single byte-level bit-reverse operation
6645/// and accepts bswap and bitreverse intrinsics:
6646/// bswap(bitreverse(x)) --> bitcast(bitreverse(bitcast(x)))
6647/// bitreverse(bswap(x)) <--> bitcast(bitreverse(bitcast(x)))
6648/// The direction of the fold is cost-model driven.
6649/// Also supports:
6650/// bitcast(bitreverse(bitcast(x))) --> bitreverse(fshl(x))
6651bool VectorCombine::foldBitOrderReverseAndSwap(Instruction &I) {
6652 Value *X;
6653
6654 if (match(V: &I, P: m_BitCast(Op: m_BitReverse(Op0: m_BitCast(Op: m_Value(V&: X)))))) {
6655 Type *Ty = X->getType();
6656 Type *VecTy = I.getOperand(i: 0)->getType();
6657 // Detect the case when bitreversing every octet in X individually. Then we
6658 // can use bswap to reorder the octets before doing a single bitreverse.
6659 bool CanUseBswap =
6660 Ty->isIntegerTy() && Ty == I.getType() && isa<FixedVectorType>(Val: VecTy) &&
6661 cast<FixedVectorType>(Val: VecTy)->getElementType()->isIntegerTy(BitWidth: 8) &&
6662 Ty->getIntegerBitWidth() % 16 == 0;
6663 // Detect the case when bitreversing upper and lower half of X
6664 // individually. Then we can use fshl as a rotate operation, to swap the
6665 // halves before doing a single bitreverse.
6666 bool CanUseFshl =
6667 Ty->isIntegerTy() && Ty == I.getType() && isa<FixedVectorType>(Val: VecTy) &&
6668 cast<FixedVectorType>(Val: VecTy)->getElementType()->isIntegerTy() &&
6669 cast<FixedVectorType>(Val: VecTy)->getNumElements() == 2;
6670 if (CanUseBswap || CanUseFshl) {
6671 auto *InnerCall = dyn_cast<Instruction>(Val: I.getOperand(i: 0));
6672 if (!InnerCall)
6673 return false;
6674 auto *InnerBitCast = dyn_cast<BitCastInst>(Val: InnerCall->getOperand(i: 0));
6675 if (!InnerBitCast)
6676 return false;
6677 Constant *HalfBW = ConstantInt::get(Ty, V: Ty->getIntegerBitWidth() / 2);
6678 InstructionCost OldCost = TTI.getInstructionCost(U: InnerBitCast, CostKind) +
6679 TTI.getInstructionCost(U: InnerCall, CostKind) +
6680 TTI.getInstructionCost(U: &I, CostKind);
6681 IntrinsicCostAttributes ICABSwap(Intrinsic::bswap, Ty, {Ty});
6682 IntrinsicCostAttributes ICABFshl(Intrinsic::fshl, Ty, {X, X, HalfBW},
6683 {Ty, Ty, Ty});
6684 IntrinsicCostAttributes ICABRev(Intrinsic::bitreverse, Ty, {Ty});
6685 InstructionCost NewCost =
6686 TTI.getIntrinsicInstrCost(ICA: CanUseBswap ? ICABSwap : ICABFshl,
6687 CostKind) +
6688 TTI.getIntrinsicInstrCost(ICA: ICABRev, CostKind);
6689 if (!InnerCall->hasOneUse())
6690 NewCost += TTI.getInstructionCost(U: InnerCall, CostKind) +
6691 TTI.getInstructionCost(U: InnerBitCast, CostKind);
6692 else if (!InnerBitCast->hasOneUse())
6693 NewCost += TTI.getInstructionCost(U: InnerBitCast, CostKind);
6694 LLVM_DEBUG(dbgs() << "Found bitreverse vector roundtrip: " << I
6695 << "\n OldCost: " << OldCost
6696 << " vs NewCost: " << NewCost << "\n");
6697 if (NewCost.isValid() && NewCost < OldCost) {
6698 Builder.SetInsertPoint(&I);
6699 Value *Swap =
6700 CanUseBswap
6701 ? Builder.CreateUnaryIntrinsic(ID: Intrinsic::bswap, Op: X)
6702 : Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::fshl, Args: {X, X, HalfBW});
6703 Value *BRev = Builder.CreateUnaryIntrinsic(ID: Intrinsic::bitreverse, Op: Swap);
6704 replaceValue(Old&: I, New&: *BRev);
6705 return true;
6706 }
6707 }
6708 }
6709
6710 if (!match(V: &I, P: m_BitReverse(Op0: m_BSwap(Op0: m_Value(V&: X)))) &&
6711 !match(V: &I, P: m_BSwap(Op0: m_BitReverse(Op0: m_Value(V&: X)))))
6712 return false;
6713 Type *Ty = I.getType();
6714 Type *I8Ty = Builder.getInt8Ty();
6715 TypeSize ElementSize = DL->getTypeStoreSize(Ty);
6716 ElementCount NewVecCnt = ElementCount::get(MinVal: ElementSize.getKnownMinValue(),
6717 Scalable: ElementSize.isScalable());
6718 Type *NewVecTy = VectorType::get(ElementType: I8Ty, EC: NewVecCnt);
6719 auto *II = cast<IntrinsicInst>(Val: &I);
6720 auto *InnerII = cast<IntrinsicInst>(Val: II->getArgOperand(i: 0));
6721 // OldCost = cost of bitreverse/bswap + cost of bswap/bitreverse
6722 InstructionCost OldCost = TTI.getInstructionCost(U: II, CostKind) +
6723 TTI.getInstructionCost(U: InnerII, CostKind);
6724 // NewCost = cost of bitcast to byte vector +
6725 // cost of bitreverse/bswap on byte vector +
6726 // cost of bitcast back to original type
6727 InstructionCost CastToVecCost = TTI.getCastInstrCost(
6728 Opcode: Instruction::BitCast, Dst: NewVecTy, Src: Ty, CCH: TTI::CastContextHint::None, CostKind);
6729 InstructionCost CastToOrigCost = TTI.getCastInstrCost(
6730 Opcode: Instruction::BitCast, Dst: Ty, Src: NewVecTy, CCH: TTI::CastContextHint::None, CostKind);
6731 IntrinsicCostAttributes ICANew(Intrinsic::bitreverse, NewVecTy, {NewVecTy});
6732 InstructionCost NewIntrinsicCost =
6733 TTI.getIntrinsicInstrCost(ICA: ICANew, CostKind);
6734 InstructionCost NewCost = CastToVecCost + NewIntrinsicCost + CastToOrigCost;
6735 if (!InnerII->hasOneUse())
6736 NewCost += TTI.getInstructionCost(U: InnerII, CostKind);
6737 LLVM_DEBUG(dbgs() << "Found bitorder reverse and swap: " << I
6738 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
6739 << "\n");
6740 if (!NewCost.isValid() || NewCost >= OldCost)
6741 return false;
6742 // Perform transform: bitcast(arg, <N x i8>), bitreverse, bitcast back
6743 Builder.SetInsertPoint(II);
6744 Value *CastToVec = Builder.CreateBitCast(V: X, DestTy: NewVecTy);
6745 Value *NewCall =
6746 Builder.CreateUnaryIntrinsic(ID: Intrinsic::bitreverse, Op: CastToVec);
6747 Value *CastToOrig = Builder.CreateBitCast(V: NewCall, DestTy: Ty);
6748 replaceValue(Old&: I, New&: *CastToOrig);
6749 return true;
6750}
6751
6752/// Given the maximum shuffle index and load vector type, compute the number of
6753/// elements for the shrunk load, rounding up to the next full vector register
6754/// boundary to avoid scalar remainders that legalize poorly.
6755static unsigned getAlignedNumElements(unsigned MaxIdx, FixedVectorType *LoadTy,
6756 const TargetTransformInfo &TTI,
6757 const DataLayout &DL) {
6758 unsigned RawNumElements = MaxIdx + 1u;
6759 Type *ElemTy = LoadTy->getElementType();
6760 // Skip alignment for illegal element types.
6761 if (!TTI.isTypeLegal(Ty: ElemTy))
6762 return RawNumElements;
6763
6764 TypeSize ElemSize = DL.getTypeSizeInBits(Ty: ElemTy);
6765 if (ElemSize.isScalable() || ElemSize.isZero())
6766 return RawNumElements;
6767
6768 TypeSize RegSize =
6769 TTI.getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector);
6770 if (RegSize.isScalable() || RegSize.isZero())
6771 return RawNumElements;
6772
6773 unsigned ElemsPerReg = RegSize.getFixedValue() / ElemSize.getFixedValue();
6774 // If the load already fits in a register, keep the exact size.
6775 // Otherwise round up to the next full register boundary.
6776 if (ElemsPerReg == 0 || RawNumElements <= ElemsPerReg)
6777 return RawNumElements;
6778
6779 return alignTo(Value: RawNumElements, Align: ElemsPerReg);
6780}
6781
6782// Attempt to shrink loads that are only used by shufflevector instructions.
6783bool VectorCombine::shrinkLoadForShuffles(Instruction &I) {
6784 auto *OldLoad = dyn_cast<LoadInst>(Val: &I);
6785 if (!OldLoad || !OldLoad->isSimple())
6786 return false;
6787
6788 auto *OldLoadTy = dyn_cast<FixedVectorType>(Val: OldLoad->getType());
6789 if (!OldLoadTy)
6790 return false;
6791
6792 unsigned const OldNumElements = OldLoadTy->getNumElements();
6793
6794 // Search all uses of load. If all uses are shufflevector instructions, and
6795 // the second operands are all poison values, find the minimum and maximum
6796 // indices of the vector elements referenced by all shuffle masks.
6797 // Otherwise return `std::nullopt`.
6798 using IndexRange = std::pair<int, int>;
6799 auto GetIndexRangeInShuffles = [&]() -> std::optional<IndexRange> {
6800 IndexRange OutputRange = IndexRange(OldNumElements, -1);
6801 for (llvm::Use &Use : I.uses()) {
6802 // Ensure all uses match the required pattern.
6803 User *Shuffle = Use.getUser();
6804 ArrayRef<int> Mask;
6805
6806 if (!match(V: Shuffle,
6807 P: m_Shuffle(v1: m_Specific(V: OldLoad), v2: m_Undef(), mask: m_Mask(Mask))))
6808 return std::nullopt;
6809
6810 // Ignore shufflevector instructions that have no uses.
6811 if (Shuffle->use_empty())
6812 continue;
6813
6814 // Find the min and max indices used by the shufflevector instruction.
6815 for (int Index : Mask) {
6816 if (Index >= 0 && Index < static_cast<int>(OldNumElements)) {
6817 OutputRange.first = std::min(a: Index, b: OutputRange.first);
6818 OutputRange.second = std::max(a: Index, b: OutputRange.second);
6819 }
6820 }
6821 }
6822
6823 if (OutputRange.second < OutputRange.first)
6824 return std::nullopt;
6825
6826 return OutputRange;
6827 };
6828
6829 // Get the range of vector elements used by shufflevector instructions.
6830 if (std::optional<IndexRange> Indices = GetIndexRangeInShuffles()) {
6831 unsigned const NewNumElements =
6832 getAlignedNumElements(MaxIdx: Indices->second, LoadTy: OldLoadTy, TTI, DL: *DL);
6833
6834 // If the range of vector elements is smaller than the full load, attempt
6835 // to create a smaller load.
6836 if (NewNumElements < OldNumElements) {
6837 Builder.SetInsertPoint(&I);
6838 Builder.SetCurrentDebugLocation(I.getDebugLoc());
6839
6840 // Calculate costs of old and new ops.
6841 Type *ElemTy = OldLoadTy->getElementType();
6842 FixedVectorType *NewLoadTy = FixedVectorType::get(ElementType: ElemTy, NumElts: NewNumElements);
6843 Value *PtrOp = OldLoad->getPointerOperand();
6844
6845 InstructionCost OldCost = TTI.getMemoryOpCost(
6846 Opcode: Instruction::Load, Src: OldLoad->getType(), Alignment: OldLoad->getAlign(),
6847 AddressSpace: OldLoad->getPointerAddressSpace(), CostKind);
6848 InstructionCost NewCost =
6849 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: NewLoadTy, Alignment: OldLoad->getAlign(),
6850 AddressSpace: OldLoad->getPointerAddressSpace(), CostKind);
6851
6852 using UseEntry = std::pair<ShuffleVectorInst *, std::vector<int>>;
6853 SmallVector<UseEntry, 4u> NewUses;
6854 unsigned const MaxIndex = NewNumElements * 2u;
6855
6856 for (llvm::Use &Use : I.uses()) {
6857 auto *Shuffle = cast<ShuffleVectorInst>(Val: Use.getUser());
6858
6859 // Ignore shufflevector instructions that have no uses.
6860 if (Shuffle->use_empty())
6861 continue;
6862
6863 ArrayRef<int> OldMask = Shuffle->getShuffleMask();
6864
6865 // Create entry for new use.
6866 NewUses.push_back(Elt: {Shuffle, OldMask});
6867
6868 // Validate mask indices.
6869 for (int Index : OldMask) {
6870 if (Index >= static_cast<int>(MaxIndex))
6871 return false;
6872 }
6873
6874 // Update costs.
6875 OldCost +=
6876 TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: Shuffle->getType(),
6877 SrcTy: OldLoadTy, CostKind, Mask: OldMask);
6878 NewCost +=
6879 TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: Shuffle->getType(),
6880 SrcTy: NewLoadTy, CostKind, Mask: OldMask);
6881 }
6882
6883 LLVM_DEBUG(
6884 dbgs() << "Found a load used only by shufflevector instructions: "
6885 << I << "\n OldCost: " << OldCost
6886 << " vs NewCost: " << NewCost << "\n");
6887
6888 if (OldCost < NewCost || !NewCost.isValid())
6889 return false;
6890
6891 // Create new load of smaller vector.
6892 auto *NewLoad = cast<LoadInst>(
6893 Val: Builder.CreateAlignedLoad(Ty: NewLoadTy, Ptr: PtrOp, Align: OldLoad->getAlign()));
6894 NewLoad->copyMetadata(SrcInst: I);
6895
6896 // Replace all uses.
6897 for (UseEntry &Use : NewUses) {
6898 ShuffleVectorInst *Shuffle = Use.first;
6899 std::vector<int> &NewMask = Use.second;
6900
6901 Builder.SetInsertPoint(Shuffle);
6902 Builder.SetCurrentDebugLocation(Shuffle->getDebugLoc());
6903 Value *NewShuffle = Builder.CreateShuffleVector(
6904 V1: NewLoad, V2: PoisonValue::get(T: NewLoadTy), Mask: NewMask);
6905
6906 replaceValue(Old&: *Shuffle, New&: *NewShuffle, Erase: false);
6907 }
6908
6909 return true;
6910 }
6911 }
6912 return false;
6913}
6914
6915// Attempt to combine two adjacent fixed-length vector loads, that only feed
6916// shufflevector instructions, into a single wider load, rewriting every such
6917// shuffle so that operand 0 is the wide load and operand 1 is poison.
6918// clang-format off
6919// e.g.
6920// %loadA = load <16 x i8>, ptr %a
6921// %gep = getelementptr inbounds <16 x i8>, ptr %a, i64 1
6922// %loadB = load <16 x i8>, ptr %gep
6923// %shuffle0 = shufflevector <16 x i8> %loadA, <16 x i8> %loadB,
6924// <32 x i8> <...>
6925// %shuffle1 = shufflevector <16 x i8> %loadA, <16 x i8> %loadB,
6926// <32 x i8> <...>
6927//
6928// The fold would transform this to:
6929// %loadAB = load <32 x i8>, ptr %a
6930// %shuffle0 = shufflevector <32 x i8> %loadAB, <32 x i8> poison,
6931// <32 x i8> <...>
6932// %shuffle1 = shufflevector <32 x i8> %loadAB, <32 x i8> poison,
6933// <32 x i8> <...>
6934//
6935// clang-format on
6936// Matching this pattern in codegen becomes difficult and hence, we prefer doing
6937// this here.
6938bool VectorCombine::foldShuffleOfAdjacentLoads(Instruction &I) {
6939 auto *SV = cast<ShuffleVectorInst>(Val: &I);
6940
6941 // The two operands must be distinct loads of the same fixed vector type.
6942 auto *Load0 = dyn_cast<LoadInst>(Val: SV->getOperand(i_nocapture: 0));
6943 auto *Load1 = dyn_cast<LoadInst>(Val: SV->getOperand(i_nocapture: 1));
6944 if (!Load0 || !Load1 || Load0 == Load1 || !Load0->isSimple() ||
6945 !Load1->isSimple())
6946 return false;
6947
6948 // Confirm both loads are of fixed vector type.
6949 auto *LoadTy = dyn_cast<FixedVectorType>(Val: Load0->getType());
6950 if (!LoadTy)
6951 return false;
6952
6953 // We restrict to loads occurring in the same BB for now.
6954 if (Load0->getParent() != Load1->getParent())
6955 return false;
6956
6957 if (Load0->getPointerAddressSpace() != Load1->getPointerAddressSpace())
6958 return false;
6959
6960 // Check that the original load type has no padding bits otherwise the wide
6961 // load would be incorrect.
6962 if (DL->getTypeSizeInBits(Ty: LoadTy) != 8 * DL->getTypeStoreSize(Ty: LoadTy))
6963 return false;
6964
6965 const unsigned NumElts = LoadTy->getNumElements();
6966 // Avoid overflow/wraparound issues.
6967 if (NumElts > INT_MAX / 2)
6968 return false;
6969
6970 // Determine which load is at the lower address and confirm the two loads are
6971 // exactly contiguous: both pointers must strip to the same base, and their
6972 // constant offsets from it must differ by exactly the store size of LoadTy.
6973 unsigned AS = Load0->getPointerAddressSpace();
6974 unsigned IdxWidth = DL->getIndexSizeInBits(AS);
6975 APInt Offset0(IdxWidth, 0), Offset1(IdxWidth, 0);
6976 const Value *Base0 =
6977 Load0->getPointerOperand()->stripAndAccumulateConstantOffsets(
6978 DL: *DL, Offset&: Offset0, /*AllowNonInbounds=*/true);
6979 const Value *Base1 =
6980 Load1->getPointerOperand()->stripAndAccumulateConstantOffsets(
6981 DL: *DL, Offset&: Offset1, /*AllowNonInbounds=*/true);
6982 // Offsets accumulated across an addrspacecast are not comparable.
6983 if (Base0 != Base1 || Base0->getType()->getPointerAddressSpace() != AS)
6984 return false;
6985
6986 int64_t LoadSize = DL->getTypeStoreSize(Ty: LoadTy).getFixedValue();
6987 std::optional<int64_t> Dist = (Offset1 - Offset0).trySExtValue();
6988 LoadInst *LowLoad, *HighLoad;
6989 if (Dist == LoadSize) {
6990 LowLoad = Load0;
6991 HighLoad = Load1;
6992 } else if (Dist == -LoadSize) {
6993 LowLoad = Load1;
6994 HighLoad = Load0;
6995 } else {
6996 return false;
6997 }
6998
6999 // 1. Check all users of both loads are shuffles.
7000 // 2. Check that both loads feed exactly the same set of shuffles.
7001 SmallPtrSet<ShuffleVectorInst *, 4> Shuffles;
7002 if (!std::is_permutation(first1: LowLoad->user_begin(), last1: LowLoad->user_end(),
7003 first2: HighLoad->user_begin(), last2: HighLoad->user_end()) ||
7004 !all_of(Range: LowLoad->users(), P: [&Shuffles](User *U) {
7005 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: U)) {
7006 Shuffles.insert(Ptr: SVI);
7007 return true;
7008 }
7009 return false;
7010 }))
7011 return false;
7012
7013 // The value loaded by either load must not be clobbered in between the loads.
7014 auto *WideTy = FixedVectorType::get(ElementType: LoadTy->getElementType(), NumElts: NumElts * 2);
7015 LoadInst *FirstLoad = LowLoad, *LastLoad = HighLoad;
7016 if (HighLoad->comesBefore(Other: LowLoad))
7017 std::swap(a&: FirstLoad, b&: LastLoad);
7018 MemoryLocation FirstLoc = MemoryLocation::get(LI: FirstLoad);
7019 if (isMemModifiedBetween(Opts, Begin: std::next(x: FirstLoad->getIterator()),
7020 End: LastLoad->getIterator(), Loc: FirstLoc, AA))
7021 return false;
7022
7023 // case 1: wide load = LowLoad + HighLoad ,
7024 // shuffle 0th operand = LowLoad
7025 // shuffle 1st operand = HighLoad
7026 // Implication with this is shuffle mask for the wide load remains unchanged
7027 // case 2: wide load = LowLoad + HighLoad ,
7028 // shuffle 0th operand = HighLoad
7029 // shuffle 1st operand = LowLoad
7030 // Implication with this is shuffle mask for the wide load changes
7031 auto RemapMask = [&](ShuffleVectorInst *SV, SmallVectorImpl<int> &NewMask) {
7032 Value *SVOp0 = SV->getOperand(i_nocapture: 0);
7033 assert(((SVOp0 == LowLoad && SV->getOperand(1) == HighLoad) ||
7034 (SVOp0 == HighLoad && SV->getOperand(1) == LowLoad)) &&
7035 "Shuffle operands must be exactly {LowLoad, HighLoad} or {HighLoad, "
7036 "LowLoad}");
7037 SV->getShuffleMask(Result&: NewMask);
7038 if (SVOp0 == HighLoad)
7039 ShuffleVectorInst::commuteShuffleMask(Mask: NewMask, InVecNumElts: NumElts);
7040 };
7041
7042 // Cost model checks
7043 Value *Poison = PoisonValue::get(T: WideTy);
7044 InstructionCost OldCost =
7045 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: LoadTy, Alignment: LowLoad->getAlign(),
7046 AddressSpace: LowLoad->getPointerAddressSpace(), CostKind);
7047 OldCost +=
7048 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: LoadTy, Alignment: HighLoad->getAlign(),
7049 AddressSpace: HighLoad->getPointerAddressSpace(), CostKind);
7050 InstructionCost NewCost =
7051 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: WideTy, Alignment: LowLoad->getAlign(),
7052 AddressSpace: LowLoad->getPointerAddressSpace(), CostKind);
7053 for (ShuffleVectorInst *SV : Shuffles) {
7054 OldCost += TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SV->getType(), SrcTy: LoadTy,
7055 CostKind, Mask: SV->getShuffleMask());
7056 SmallVector<int, 32> NewMask;
7057 RemapMask(SV, NewMask);
7058 // LoadSz = initial load size
7059 // WideSz = 2 * LoadSz
7060 // MaxMaskSize = WideSz * 2
7061 // Check if MaxMaskSize fits within an integer range.
7062 if (!ShuffleVectorInst::isValidOperands(V1: Poison, V2: Poison, Mask: NewMask))
7063 return false;
7064 NewCost += TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: SV->getType(),
7065 SrcTy: WideTy, CostKind, Mask: NewMask);
7066 }
7067
7068 LLVM_DEBUG(dbgs() << "Found adjacent loads feeding shuffles: " << *LowLoad
7069 << ", " << *HighLoad << "\n OldCost: " << OldCost
7070 << " vs NewCost: " << NewCost << "\n");
7071
7072 if (!NewCost.isValid() || NewCost > OldCost)
7073 return false;
7074
7075 // Insert the wide load at whichever original load comes last, so that both
7076 // halves of the contiguous range are known to be dereferenceable there.
7077 LoadInst *InsertPt = LastLoad;
7078
7079 // Build the wide load at the insertion point using the low load's pointer and
7080 // alignment, intersecting alias metadata from both original loads.
7081 Builder.SetInsertPoint(InsertPt);
7082 Builder.SetCurrentDebugLocation(InsertPt->getDebugLoc());
7083 LoadInst *WideLoad = Builder.CreateAlignedLoad(
7084 Ty: WideTy, Ptr: LowLoad->getPointerOperand(), Align: LowLoad->getAlign());
7085
7086 // Set the metadata on the wide load. copyMetadataForLoad seeds it from
7087 // LowLoad, then combineMetadataForCSE intersects every known kind against
7088 // HighLoad (taking the most-generic value where applicable, keeping facts
7089 // only where both loads agree, and dropping unknown metadata), so nothing is
7090 // asserted over the combined load unless justified by both halves.
7091 copyMetadataForLoad(Dest&: *WideLoad, Source: *LowLoad);
7092 combineMetadataForCSE(K: WideLoad, J: HighLoad, /*DoesKMove=*/true);
7093
7094 for (ShuffleVectorInst *SV : Shuffles) {
7095 SmallVector<int, 32> NewMask;
7096 RemapMask(SV, NewMask);
7097
7098 Builder.SetInsertPoint(SV);
7099 Builder.SetCurrentDebugLocation(SV->getDebugLoc());
7100 Value *NewShuf = Builder.CreateShuffleVector(V1: WideLoad, V2: Poison, Mask: NewMask);
7101 // We do not want to erase shuffles immediately because they may invalidate
7102 // the NextInst pointer in the caller's BB traversal.
7103 replaceValue(Old&: *SV, New&: *NewShuf, /*Erase=*/false);
7104 }
7105 return true;
7106}
7107
7108// Attempt to narrow a phi of shufflevector instructions where the two incoming
7109// values have the same operands but different masks. If the two shuffle masks
7110// are offsets of one another we can use one branch to rotate the incoming
7111// vector and perform one larger shuffle after the phi.
7112bool VectorCombine::shrinkPhiOfShuffles(Instruction &I) {
7113 auto *Phi = dyn_cast<PHINode>(Val: &I);
7114 if (!Phi || Phi->getNumIncomingValues() != 2u)
7115 return false;
7116
7117 Value *Op = nullptr;
7118 ArrayRef<int> Mask0;
7119 ArrayRef<int> Mask1;
7120
7121 if (!match(V: Phi->getOperand(i_nocapture: 0u),
7122 P: m_OneUse(SubPattern: m_Shuffle(v1: m_Value(V&: Op), v2: m_Poison(), mask: m_Mask(Mask0)))) ||
7123 !match(V: Phi->getOperand(i_nocapture: 1u),
7124 P: m_OneUse(SubPattern: m_Shuffle(v1: m_Specific(V: Op), v2: m_Poison(), mask: m_Mask(Mask1)))))
7125 return false;
7126
7127 auto *Shuf = cast<ShuffleVectorInst>(Val: Phi->getOperand(i_nocapture: 0u));
7128
7129 // Ensure result vectors are wider than the argument vector.
7130 auto *InputVT = cast<FixedVectorType>(Val: Op->getType());
7131 auto *ResultVT = cast<FixedVectorType>(Val: Shuf->getType());
7132 auto const InputNumElements = InputVT->getNumElements();
7133
7134 if (InputNumElements >= ResultVT->getNumElements())
7135 return false;
7136
7137 // Take the difference of the two shuffle masks at each index. Ignore poison
7138 // values at the same index in both masks.
7139 SmallVector<int, 16> NewMask;
7140 NewMask.reserve(N: Mask0.size());
7141
7142 auto IsPoison = [InputNumElements](int M) {
7143 return M < 0 || (unsigned)M >= InputNumElements;
7144 };
7145
7146 for (auto [M0, M1] : zip(t&: Mask0, u&: Mask1)) {
7147 bool P0 = IsPoison(M0);
7148 bool P1 = IsPoison(M1);
7149 if (!P0 && !P1)
7150 NewMask.push_back(Elt: M0 - M1);
7151 else if (P0 && P1)
7152 continue;
7153 else
7154 return false;
7155 }
7156
7157 // Ensure all elements of the new mask are equal. If the difference between
7158 // the incoming mask elements is the same, the two must be constant offsets
7159 // of one another.
7160 if (NewMask.empty() || !all_equal(Range&: NewMask))
7161 return false;
7162
7163 // Create new mask using difference of the two incoming masks.
7164 int MaskOffset = NewMask[0u];
7165 unsigned Index = (InputNumElements + MaskOffset) % InputNumElements;
7166 NewMask.clear();
7167
7168 for (unsigned I = 0u; I < InputNumElements; ++I) {
7169 NewMask.push_back(Elt: Index);
7170 Index = (Index + 1u) % InputNumElements;
7171 }
7172
7173 // Calculate costs for worst cases and compare.
7174 auto const Kind = TTI::SK_PermuteSingleSrc;
7175 auto OldCost =
7176 std::max(a: TTI.getShuffleCost(Kind, DstTy: ResultVT, SrcTy: InputVT, CostKind, Mask: Mask0),
7177 b: TTI.getShuffleCost(Kind, DstTy: ResultVT, SrcTy: InputVT, CostKind, Mask: Mask1));
7178 auto NewCost = TTI.getShuffleCost(Kind, DstTy: InputVT, SrcTy: InputVT, CostKind, Mask: NewMask) +
7179 TTI.getShuffleCost(Kind, DstTy: ResultVT, SrcTy: InputVT, CostKind, Mask: Mask1);
7180
7181 LLVM_DEBUG(dbgs() << "Found a phi of mergeable shuffles: " << I
7182 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
7183 << "\n");
7184
7185 if (NewCost > OldCost)
7186 return false;
7187
7188 // Create new shuffles and narrowed phi.
7189 Builder.SetInsertPoint(Shuf);
7190 Builder.SetCurrentDebugLocation(Shuf->getDebugLoc());
7191 auto *PoisonVal = PoisonValue::get(T: InputVT);
7192 auto *NewShuf0 = Builder.CreateShuffleVector(V1: Op, V2: PoisonVal, Mask: NewMask);
7193
7194 Builder.SetInsertPoint(Phi);
7195 Builder.SetCurrentDebugLocation(Phi->getDebugLoc());
7196 auto *NewPhi = Builder.CreatePHI(Ty: NewShuf0->getType(), NumReservedValues: 2u);
7197 NewPhi->addIncoming(V: NewShuf0, BB: Phi->getIncomingBlock(i: 0u));
7198 NewPhi->addIncoming(V: Op, BB: Phi->getIncomingBlock(i: 1u));
7199
7200 Builder.SetInsertPoint(*NewPhi->getInsertionPointAfterDef());
7201 PoisonVal = PoisonValue::get(T: NewPhi->getType());
7202 auto *NewShuf1 = Builder.CreateShuffleVector(V1: NewPhi, V2: PoisonVal, Mask: Mask1);
7203
7204 replaceValue(Old&: *Phi, New&: *NewShuf1);
7205 return true;
7206}
7207
7208/// This is the entry point for all transforms. Pass manager differences are
7209/// handled in the callers of this function.
7210bool VectorCombine::run() {
7211 if (Opts.disable_vector_combine)
7212 return false;
7213
7214 // Don't attempt vectorization if the target does not support vectors.
7215 if (!TTI.getNumberOfRegisters(ClassID: TTI.getRegisterClassForType(/*Vector*/ true)))
7216 return false;
7217
7218 LLVM_DEBUG(dbgs() << "\n\nVECTORCOMBINE on " << F.getName() << "\n");
7219
7220 auto FoldInst = [this](Instruction &I) {
7221 Builder.SetInsertPoint(&I);
7222 bool IsVectorType = isa<VectorType>(Val: I.getType());
7223 bool IsFixedVectorType = isa<FixedVectorType>(Val: I.getType());
7224 auto Opcode = I.getOpcode();
7225
7226 LLVM_DEBUG(dbgs() << "VC: Visiting: " << I << '\n');
7227
7228 // These folds should be beneficial regardless of when this pass is run
7229 // in the optimization pipeline.
7230 // The type checking is for run-time efficiency. We can avoid wasting time
7231 // dispatching to folding functions if there's no chance of matching.
7232 if (IsFixedVectorType) {
7233 switch (Opcode) {
7234 case Instruction::InsertElement:
7235 if (vectorizeLoadInsert(I))
7236 return true;
7237 break;
7238 case Instruction::ShuffleVector:
7239 if (widenSubvectorLoad(I))
7240 return true;
7241 break;
7242 default:
7243 break;
7244 }
7245 }
7246
7247 // This transform works with scalable and fixed vectors
7248 // TODO: Identify and allow other scalable transforms
7249 if (IsVectorType) {
7250 if (scalarizeOpOrCmp(I))
7251 return true;
7252 if (scalarizeLoad(I))
7253 return true;
7254 if (scalarizeExtExtract(I))
7255 return true;
7256 if (foldInterleaveIntrinsics(I))
7257 return true;
7258 if (foldBitcastOfVPLoad(I))
7259 return true;
7260 }
7261
7262 if (foldDeinterleaveIntrinsics(I))
7263 return true;
7264
7265 if (Opcode == Instruction::Store)
7266 if (foldInsertElementsToStores(I))
7267 return true;
7268
7269 // If this is an early pipeline invocation of this pass, we are done.
7270 if (TryEarlyFoldsOnly)
7271 return false;
7272
7273 if (Opcode == Instruction::Call)
7274 if (foldBitOrderReverseAndSwap(I))
7275 return true;
7276 if (Opcode == Instruction::BitCast)
7277 if (foldBitOrderReverseAndSwap(I))
7278 return true;
7279
7280 // Otherwise, try folds that improve codegen but may interfere with
7281 // early IR canonicalizations.
7282 // The type checking is for run-time efficiency. We can avoid wasting time
7283 // dispatching to folding functions if there's no chance of matching.
7284 if (IsFixedVectorType) {
7285 switch (Opcode) {
7286 case Instruction::InsertElement:
7287 if (foldInsExtFNeg(I))
7288 return true;
7289 if (foldInsExtBinop(I))
7290 return true;
7291 if (foldInsExtVectorToShuffle(I))
7292 return true;
7293 if (foldInsertScalarPartsToShuffle(I))
7294 return true;
7295 break;
7296 case Instruction::ShuffleVector:
7297 if (foldPermuteOfBinops(I))
7298 return true;
7299 if (foldShuffleOfBinops(I))
7300 return true;
7301 if (foldShuffleOfSelects(I))
7302 return true;
7303 if (foldShuffleOfCastops(I))
7304 return true;
7305 if (foldShuffleOfShuffles(I))
7306 return true;
7307 if (foldShuffleOfAdjacentLoads(I))
7308 return true;
7309 if (foldPermuteOfIntrinsic(I))
7310 return true;
7311 if (foldShufflesOfLengthChangingShuffles(I))
7312 return true;
7313 if (foldShuffleOfIntrinsics(I))
7314 return true;
7315 if (foldSelectShuffle(I))
7316 return true;
7317 if (foldShuffleToIdentity(I))
7318 return true;
7319 break;
7320 case Instruction::Load:
7321 if (shrinkLoadForShuffles(I))
7322 return true;
7323 break;
7324 case Instruction::BitCast:
7325 if (foldBitcastShuffle(I))
7326 return true;
7327 if (foldSelectsFromBitcast(I))
7328 return true;
7329 break;
7330 case Instruction::And:
7331 case Instruction::Or:
7332 case Instruction::Xor:
7333 if (foldBitOpOfCastops(I))
7334 return true;
7335 if (foldBitOpOfCastConstant(I))
7336 return true;
7337 break;
7338 case Instruction::PHI:
7339 if (shrinkPhiOfShuffles(I))
7340 return true;
7341 break;
7342 default:
7343 if (shrinkType(I))
7344 return true;
7345 break;
7346 }
7347 } else {
7348 switch (Opcode) {
7349 case Instruction::Call:
7350 if (foldShuffleFromReductions(I))
7351 return true;
7352 if (foldCastFromReductions(I))
7353 return true;
7354 break;
7355 case Instruction::ExtractElement:
7356 if (foldShuffleChainsToReduce(I))
7357 return true;
7358 break;
7359 case Instruction::ICmp:
7360 if (foldSignBitReductionCmp(I))
7361 return true;
7362 if (foldICmpEqZeroVectorReduce(I))
7363 return true;
7364 if (foldReductionZeroTest(I))
7365 return true;
7366 if (foldEquivalentReductionCmp(I))
7367 return true;
7368 if (foldReduceAddCmpZero(I))
7369 return true;
7370 [[fallthrough]];
7371 case Instruction::FCmp:
7372 if (foldExtractExtract(I))
7373 return true;
7374 break;
7375 case Instruction::Or:
7376 if (foldConcatOfBoolMasks(I))
7377 return true;
7378 [[fallthrough]];
7379 default:
7380 if (Instruction::isBinaryOp(Opcode)) {
7381 if (foldExtractExtract(I))
7382 return true;
7383 if (foldExtractedCmps(I))
7384 return true;
7385 if (foldBinopOfReductions(I))
7386 return true;
7387 }
7388 break;
7389 }
7390 }
7391 return false;
7392 };
7393
7394 bool MadeChange = false;
7395 for (BasicBlock &BB : F) {
7396 // Ignore unreachable basic blocks.
7397 if (!DT.isReachableFromEntry(A: &BB))
7398 continue;
7399 // Use early increment range so that we can erase instructions in loop.
7400 // make_early_inc_range is not applicable here, as the next iterator may
7401 // be invalidated by RecursivelyDeleteTriviallyDeadInstructions.
7402 // We manually maintain the next instruction and update it when it is about
7403 // to be deleted.
7404 Instruction *I = &BB.front();
7405 while (I) {
7406 NextInst = I->getNextNode();
7407 MadeChange |= FoldInst(*I);
7408 I = NextInst;
7409 }
7410 }
7411
7412 NextInst = nullptr;
7413
7414 while (!Worklist.isEmpty()) {
7415 // Push deferred instructions in reverse order, so that they'll end up
7416 // popped from the worklist in-order.
7417 while (Instruction *I = Worklist.popDeferred()) {
7418 if (isInstructionTriviallyDead(I)) {
7419 eraseInstruction(I&: *I);
7420 continue;
7421 }
7422
7423 Worklist.push(I);
7424 }
7425
7426 Instruction *I = Worklist.removeOne();
7427 if (!I)
7428 continue;
7429
7430 if (isInstructionTriviallyDead(I)) {
7431 eraseInstruction(I&: *I);
7432 continue;
7433 }
7434
7435 MadeChange |= FoldInst(*I);
7436 }
7437
7438 return MadeChange;
7439}
7440
7441PreservedAnalyses VectorCombinePass::run(Function &F,
7442 FunctionAnalysisManager &FAM) {
7443 auto &AC = FAM.getResult<AssumptionAnalysis>(IR&: F);
7444 TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(IR&: F);
7445 DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(IR&: F);
7446 AAResults &AA = FAM.getResult<AAManager>(IR&: F);
7447 const DataLayout *DL = &F.getDataLayout();
7448 TTI::TargetCostKind CostKind =
7449 F.hasOptSize() ? TTI::TCK_CodeSize : TTI::TCK_RecipThroughput;
7450 VectorCombine Combiner(getVectorizeOptions(F), F, TTI, DT, AA, AC, DL,
7451 CostKind, TryEarlyFoldsOnly);
7452 if (!Combiner.run())
7453 return PreservedAnalyses::all();
7454 PreservedAnalyses PA;
7455 PA.preserveSet<CFGAnalyses>();
7456 return PA;
7457}
7458