1//===------- VectorCombine.cpp - Optimize partial vector operations -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass optimizes scalar/vector interactions using target cost models. The
10// transforms implemented here may not fit in traditional loop-based or SLP
11// vectorization passes.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Vectorize/VectorCombine.h"
16#include "llvm/ADT/DenseMap.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/ScopeExit.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/SmallVectorExtras.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/Analysis/AssumptionCache.h"
23#include "llvm/Analysis/BasicAliasAnalysis.h"
24#include "llvm/Analysis/GlobalsModRef.h"
25#include "llvm/Analysis/InstSimplifyFolder.h"
26#include "llvm/Analysis/Loads.h"
27#include "llvm/Analysis/TargetFolder.h"
28#include "llvm/Analysis/TargetTransformInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/IR/Dominators.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/IRBuilder.h"
34#include "llvm/IR/Instructions.h"
35#include "llvm/IR/PatternMatch.h"
36#include "llvm/IR/ProfDataUtils.h"
37#include "llvm/Support/CommandLine.h"
38#include "llvm/Support/KnownBits.h"
39#include "llvm/Support/MathExtras.h"
40#include "llvm/Transforms/Utils/Local.h"
41#include "llvm/Transforms/Utils/LoopUtils.h"
42#include <numeric>
43#include <optional>
44#include <queue>
45#include <set>
46
47#define DEBUG_TYPE "vector-combine"
48#include "llvm/Transforms/Utils/InstructionWorklist.h"
49
50using namespace llvm;
51using namespace llvm::PatternMatch;
52
53STATISTIC(NumVecLoad, "Number of vector loads formed");
54STATISTIC(NumVecCmp, "Number of vector compares formed");
55STATISTIC(NumVecBO, "Number of vector binops formed");
56STATISTIC(NumVecCmpBO, "Number of vector compare + binop formed");
57STATISTIC(NumShufOfBitcast, "Number of shuffles moved after bitcast");
58STATISTIC(NumScalarOps, "Number of scalar unary + binary ops formed");
59STATISTIC(NumScalarCmp, "Number of scalar compares formed");
60STATISTIC(NumScalarIntrinsic, "Number of scalar intrinsic calls formed");
61
62static cl::opt<bool> DisableVectorCombine(
63 "disable-vector-combine", cl::init(Val: false), cl::Hidden,
64 cl::desc("Disable all vector combine transforms"));
65
66static cl::opt<bool> DisableBinopExtractShuffle(
67 "disable-binop-extract-shuffle", cl::init(Val: false), cl::Hidden,
68 cl::desc("Disable binop extract to shuffle transforms"));
69
70static cl::opt<unsigned> MaxInstrsToScan(
71 "vector-combine-max-scan-instrs", cl::init(Val: 30), cl::Hidden,
72 cl::desc("Max number of instructions to scan for vector combining."));
73
74static const unsigned InvalidIndex = std::numeric_limits<unsigned>::max();
75
76namespace {
77class VectorCombine {
78public:
79 VectorCombine(Function &F, const TargetTransformInfo &TTI,
80 const DominatorTree &DT, AAResults &AA, AssumptionCache &AC,
81 const DataLayout *DL, TTI::TargetCostKind CostKind,
82 bool TryEarlyFoldsOnly)
83 : F(F), Builder(F.getContext(), InstSimplifyFolder(*DL)), TTI(TTI),
84 DT(DT), AA(AA), DL(DL), CostKind(CostKind),
85 SQ(*DL, /*TLI=*/nullptr, &DT, &AC),
86 TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
87
88 bool run();
89
90private:
91 Function &F;
92 IRBuilder<InstSimplifyFolder> Builder;
93 const TargetTransformInfo &TTI;
94 const DominatorTree &DT;
95 AAResults &AA;
96 const DataLayout *DL;
97 TTI::TargetCostKind CostKind;
98 const SimplifyQuery SQ;
99
100 /// If true, only perform beneficial early IR transforms. Do not introduce new
101 /// vector operations.
102 bool TryEarlyFoldsOnly;
103
104 InstructionWorklist Worklist;
105
106 /// Next instruction to iterate. It will be updated when it is erased by
107 /// RecursivelyDeleteTriviallyDeadInstructions.
108 Instruction *NextInst;
109
110 // TODO: Direct calls from the top-level "run" loop use a plain "Instruction"
111 // parameter. That should be updated to specific sub-classes because the
112 // run loop was changed to dispatch on opcode.
113 bool vectorizeLoadInsert(Instruction &I);
114 bool widenSubvectorLoad(Instruction &I);
115 ExtractElementInst *getShuffleExtract(ExtractElementInst *Ext0,
116 ExtractElementInst *Ext1,
117 unsigned PreferredExtractIndex) const;
118 bool isExtractExtractCheap(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
119 const Instruction &I,
120 ExtractElementInst *&ConvertToShuffle,
121 unsigned PreferredExtractIndex);
122 Value *foldExtExtCmp(Value *V0, Value *V1, Value *ExtIndex, Instruction &I);
123 Value *foldExtExtBinop(Value *V0, Value *V1, Value *ExtIndex, Instruction &I);
124 bool foldExtractExtract(Instruction &I);
125 bool foldInsExtFNeg(Instruction &I);
126 bool foldInsExtBinop(Instruction &I);
127 bool foldInsExtVectorToShuffle(Instruction &I);
128 bool foldBitOpOfCastops(Instruction &I);
129 bool foldBitOpOfCastConstant(Instruction &I);
130 bool foldBitcastShuffle(Instruction &I);
131 bool scalarizeOpOrCmp(Instruction &I);
132 bool foldExtractedCmps(Instruction &I);
133 bool foldSelectsFromBitcast(Instruction &I);
134 bool foldBinopOfReductions(Instruction &I);
135 bool foldInsertElementsToStores(Instruction &I);
136 bool scalarizeLoad(Instruction &I);
137 bool scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy, Value *Ptr);
138 bool scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy, Value *Ptr);
139 bool scalarizeExtExtract(Instruction &I);
140 bool foldConcatOfBoolMasks(Instruction &I);
141 bool foldPermuteOfBinops(Instruction &I);
142 bool foldShuffleOfBinops(Instruction &I);
143 bool foldShuffleOfSelects(Instruction &I);
144 bool foldShuffleOfCastops(Instruction &I);
145 bool foldShuffleOfShuffles(Instruction &I);
146 bool foldPermuteOfIntrinsic(Instruction &I);
147 bool foldShufflesOfLengthChangingShuffles(Instruction &I);
148 bool foldShuffleOfIntrinsics(Instruction &I);
149 bool foldShuffleToIdentity(Instruction &I);
150 bool foldShuffleFromReductions(Instruction &I);
151 bool foldShuffleChainsToReduce(Instruction &I);
152 bool foldCastFromReductions(Instruction &I);
153 bool foldSignBitReductionCmp(Instruction &I);
154 bool foldReductionZeroTest(Instruction &I);
155 bool foldICmpEqZeroVectorReduce(Instruction &I);
156 bool foldEquivalentReductionCmp(Instruction &I);
157 bool foldReduceAddCmpZero(Instruction &I);
158 bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
159 bool foldInterleaveIntrinsics(Instruction &I);
160 bool foldDeinterleaveIntrinsics(Instruction &I);
161 bool foldBitcastOfVPLoad(Instruction &I);
162 bool foldBitOrderReverseAndSwap(Instruction &I);
163 bool shrinkType(Instruction &I);
164 bool shrinkLoadForShuffles(Instruction &I);
165 bool shrinkPhiOfShuffles(Instruction &I);
166 bool foldDeinterleaveInterleavePair(Instruction &I);
167
168 void replaceValue(Instruction &Old, Value &New, bool Erase = true) {
169 LLVM_DEBUG(dbgs() << "VC: Replacing: " << Old << '\n');
170 LLVM_DEBUG(dbgs() << " With: " << New << '\n');
171 Old.replaceAllUsesWith(V: &New);
172 if (auto *NewI = dyn_cast<Instruction>(Val: &New)) {
173 New.takeName(V: &Old);
174 Worklist.pushUsersToWorkList(I&: *NewI);
175 Worklist.pushValue(V: NewI);
176 }
177 if (Erase && isInstructionTriviallyDead(I: &Old)) {
178 eraseInstruction(I&: Old);
179 } else {
180 Worklist.push(I: &Old);
181 }
182 }
183
184 void eraseInstruction(Instruction &I) {
185 LLVM_DEBUG(dbgs() << "VC: Erasing: " << I << '\n');
186 SmallVector<Value *> Ops(I.operands());
187 Worklist.remove(I: &I);
188 I.eraseFromParent();
189
190 // Push remaining users of the operands and then the operand itself - allows
191 // further folds that were hindered by OneUse limits.
192 SmallPtrSet<Value *, 4> Visited;
193 for (Value *Op : Ops) {
194 if (!Visited.contains(Ptr: Op)) {
195 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
196 if (RecursivelyDeleteTriviallyDeadInstructions(
197 V: OpI, TLI: nullptr, MSSAU: nullptr, AboutToDeleteCallback: [&](Value *V) {
198 if (auto *I = dyn_cast<Instruction>(Val: V)) {
199 LLVM_DEBUG(dbgs() << "VC: Erased: " << *I << '\n');
200 Worklist.remove(I);
201 if (I == NextInst)
202 NextInst = NextInst->getNextNode();
203 Visited.insert(Ptr: I);
204 }
205 }))
206 continue;
207 Worklist.pushUsersToWorkList(I&: *OpI);
208 Worklist.pushValue(V: OpI);
209 }
210 }
211 }
212 }
213};
214} // namespace
215
216/// Return the source operand of a potentially bitcasted value. If there is no
217/// bitcast, return the input value itself.
218static Value *peekThroughBitcasts(Value *V) {
219 while (auto *BitCast = dyn_cast<BitCastInst>(Val: V))
220 V = BitCast->getOperand(i_nocapture: 0);
221 return V;
222}
223
224/// Helper to peek through bitcasts to the same value.
225static bool isEquivBitcast(Value *X, Value *Y) {
226 return X->getType() == Y->getType() &&
227 peekThroughBitcasts(V: X) == peekThroughBitcasts(V: Y);
228}
229
230static bool canWidenLoad(LoadInst *Load, const TargetTransformInfo &TTI) {
231 // Do not widen load if atomic/volatile or under asan/hwasan/memtag/tsan.
232 // The widened load may load data from dirty regions or create data races
233 // non-existent in the source.
234 if (!Load || !Load->isSimple() || !Load->hasOneUse() ||
235 Load->getFunction()->hasFnAttribute(Kind: Attribute::SanitizeMemTag) ||
236 mustSuppressSpeculation(LI: *Load))
237 return false;
238
239 // We are potentially transforming byte-sized (8-bit) memory accesses, so make
240 // sure we have all of our type-based constraints in place for this target.
241 Type *ScalarTy = Load->getType()->getScalarType();
242 uint64_t ScalarSize = ScalarTy->getPrimitiveSizeInBits();
243 unsigned MinVectorSize = TTI.getMinVectorRegisterBitWidth();
244 if (!ScalarSize || !MinVectorSize || MinVectorSize % ScalarSize != 0 ||
245 ScalarSize % 8 != 0)
246 return false;
247
248 return true;
249}
250
251bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
252 // Match insert into fixed vector of scalar value.
253 // TODO: Handle non-zero insert index.
254 Value *Scalar;
255 if (!match(V: &I,
256 P: m_InsertElt(Val: m_Poison(), Elt: m_OneUse(SubPattern: m_Value(V&: Scalar)), Idx: m_ZeroInt())))
257 return false;
258
259 // Optionally match an extract from another vector.
260 Value *X;
261 bool HasExtract = match(V: Scalar, P: m_ExtractElt(Val: m_Value(V&: X), Idx: m_ZeroInt()));
262 if (!HasExtract)
263 X = Scalar;
264
265 auto *Load = dyn_cast<LoadInst>(Val: X);
266 if (!canWidenLoad(Load, TTI))
267 return false;
268
269 Type *ScalarTy = Scalar->getType();
270 uint64_t ScalarSize = ScalarTy->getPrimitiveSizeInBits();
271 unsigned MinVectorSize = TTI.getMinVectorRegisterBitWidth();
272
273 // Check safety of replacing the scalar load with a larger vector load.
274 // We use minimal alignment (maximum flexibility) because we only care about
275 // the dereferenceable region. When calculating cost and creating a new op,
276 // we may use a larger value based on alignment attributes.
277 Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts();
278 assert(isa<PointerType>(SrcPtr->getType()) && "Expected a pointer type");
279
280 unsigned MinVecNumElts = MinVectorSize / ScalarSize;
281 auto *MinVecTy = VectorType::get(ElementType: ScalarTy, NumElements: MinVecNumElts, Scalable: false);
282 unsigned OffsetEltIndex = 0;
283 Align Alignment = Load->getAlign();
284 if (!isSafeToLoadUnconditionally(V: SrcPtr, Ty: MinVecTy, Alignment: Align(1),
285 SQ: SQ.getWithInstruction(I: Load))) {
286 // It is not safe to load directly from the pointer, but we can still peek
287 // through gep offsets and check if it safe to load from a base address with
288 // updated alignment. If it is, we can shuffle the element(s) into place
289 // after loading.
290 unsigned OffsetBitWidth = DL->getIndexTypeSizeInBits(Ty: SrcPtr->getType());
291 APInt Offset(OffsetBitWidth, 0);
292 SrcPtr = SrcPtr->stripAndAccumulateInBoundsConstantOffsets(DL: *DL, Offset);
293
294 // We want to shuffle the result down from a high element of a vector, so
295 // the offset must be positive.
296 if (Offset.isNegative())
297 return false;
298
299 // The offset must be a multiple of the scalar element to shuffle cleanly
300 // in the element's size.
301 uint64_t ScalarSizeInBytes = ScalarSize / 8;
302 if (Offset.urem(RHS: ScalarSizeInBytes) != 0)
303 return false;
304
305 // If we load MinVecNumElts, will our target element still be loaded?
306 APInt OffsetEltIndexAP = Offset.udiv(RHS: ScalarSizeInBytes);
307 if (OffsetEltIndexAP.uge(RHS: MinVecNumElts))
308 return false;
309 OffsetEltIndex = OffsetEltIndexAP.getZExtValue();
310
311 if (!isSafeToLoadUnconditionally(V: SrcPtr, Ty: MinVecTy, Alignment: Align(1),
312 SQ: SQ.getWithInstruction(I: Load)))
313 return false;
314
315 // Update alignment with offset value. Note that the offset could be negated
316 // to more accurately represent "(new) SrcPtr - Offset = (old) SrcPtr", but
317 // negation does not change the result of the alignment calculation.
318 Alignment = commonAlignment(A: Alignment, Offset: Offset.getZExtValue());
319 }
320
321 // Original pattern: insertelt undef, load [free casts of] PtrOp, 0
322 // Use the greater of the alignment on the load or its source pointer.
323 Alignment = std::max(a: SrcPtr->getPointerAlignment(DL: *DL), b: Alignment);
324 Type *LoadTy = Load->getType();
325 unsigned AS = Load->getPointerAddressSpace();
326 InstructionCost OldCost =
327 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: LoadTy, Alignment, AddressSpace: AS, CostKind);
328 APInt DemandedElts = APInt::getOneBitSet(numBits: MinVecNumElts, BitNo: 0);
329 OldCost +=
330 TTI.getScalarizationOverhead(Ty: MinVecTy, DemandedElts,
331 /* Insert */ true, Extract: HasExtract, CostKind);
332
333 // New pattern: load VecPtr
334 InstructionCost NewCost =
335 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: MinVecTy, Alignment, AddressSpace: AS, CostKind);
336 // Optionally, we are shuffling the loaded vector element(s) into place.
337 // For the mask set everything but element 0 to undef to prevent poison from
338 // propagating from the extra loaded memory. This will also optionally
339 // shrink/grow the vector from the loaded size to the output size.
340 // We assume this operation has no cost in codegen if there was no offset.
341 // Note that we could use freeze to avoid poison problems, but then we might
342 // still need a shuffle to change the vector size.
343 auto *Ty = cast<FixedVectorType>(Val: I.getType());
344 unsigned OutputNumElts = Ty->getNumElements();
345 SmallVector<int, 16> Mask(OutputNumElts, PoisonMaskElem);
346 assert(OffsetEltIndex < MinVecNumElts && "Address offset too big");
347 Mask[0] = OffsetEltIndex;
348 if (OffsetEltIndex)
349 NewCost += TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: Ty, SrcTy: MinVecTy,
350 CostKind, Mask);
351
352 // We can aggressively convert to the vector form because the backend can
353 // invert this transform if it does not result in a performance win.
354 if (OldCost < NewCost || !NewCost.isValid())
355 return false;
356
357 // It is safe and potentially profitable to load a vector directly:
358 // inselt undef, load Scalar, 0 --> load VecPtr
359 IRBuilder<> Builder(Load);
360 Value *CastedPtr =
361 Builder.CreatePointerBitCastOrAddrSpaceCast(V: SrcPtr, DestTy: Builder.getPtrTy(AddrSpace: AS));
362 Value *VecLd = Builder.CreateAlignedLoad(Ty: MinVecTy, Ptr: CastedPtr, Align: Alignment);
363 VecLd = Builder.CreateShuffleVector(V: VecLd, Mask);
364
365 replaceValue(Old&: I, New&: *VecLd);
366 ++NumVecLoad;
367 return true;
368}
369
370/// If we are loading a vector and then inserting it into a larger vector with
371/// undefined elements, try to load the larger vector and eliminate the insert.
372/// This removes a shuffle in IR and may allow combining of other loaded values.
373bool VectorCombine::widenSubvectorLoad(Instruction &I) {
374 // Match subvector insert of fixed vector.
375 auto *Shuf = cast<ShuffleVectorInst>(Val: &I);
376 if (!Shuf->isIdentityWithPadding())
377 return false;
378
379 // Allow a non-canonical shuffle mask that is choosing elements from op1.
380 unsigned NumOpElts =
381 cast<FixedVectorType>(Val: Shuf->getOperand(i_nocapture: 0)->getType())->getNumElements();
382 unsigned OpIndex = any_of(Range: Shuf->getShuffleMask(), P: [&NumOpElts](int M) {
383 return M >= (int)(NumOpElts);
384 });
385
386 auto *Load = dyn_cast<LoadInst>(Val: Shuf->getOperand(i_nocapture: OpIndex));
387 if (!canWidenLoad(Load, TTI))
388 return false;
389
390 // We use minimal alignment (maximum flexibility) because we only care about
391 // the dereferenceable region. When calculating cost and creating a new op,
392 // we may use a larger value based on alignment attributes.
393 auto *Ty = cast<FixedVectorType>(Val: I.getType());
394 Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts();
395 assert(isa<PointerType>(SrcPtr->getType()) && "Expected a pointer type");
396 Align Alignment = Load->getAlign();
397 if (!isSafeToLoadUnconditionally(V: SrcPtr, Ty, Alignment: Align(1),
398 SQ: SQ.getWithInstruction(I: Load)))
399 return false;
400
401 Alignment = std::max(a: SrcPtr->getPointerAlignment(DL: *DL), b: Alignment);
402 Type *LoadTy = Load->getType();
403 unsigned AS = Load->getPointerAddressSpace();
404
405 // Original pattern: insert_subvector (load PtrOp)
406 // This conservatively assumes that the cost of a subvector insert into an
407 // undef value is 0. We could add that cost if the cost model accurately
408 // reflects the real cost of that operation.
409 InstructionCost OldCost =
410 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: LoadTy, Alignment, AddressSpace: AS, CostKind);
411
412 // New pattern: load PtrOp
413 InstructionCost NewCost =
414 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment, AddressSpace: AS, CostKind);
415
416 // We can aggressively convert to the vector form because the backend can
417 // invert this transform if it does not result in a performance win.
418 if (OldCost < NewCost || !NewCost.isValid())
419 return false;
420
421 IRBuilder<> Builder(Load);
422 Value *CastedPtr =
423 Builder.CreatePointerBitCastOrAddrSpaceCast(V: SrcPtr, DestTy: Builder.getPtrTy(AddrSpace: AS));
424 Value *VecLd = Builder.CreateAlignedLoad(Ty, Ptr: CastedPtr, Align: Alignment);
425 replaceValue(Old&: I, New&: *VecLd);
426 ++NumVecLoad;
427 return true;
428}
429
430/// Determine which, if any, of the inputs should be replaced by a shuffle
431/// followed by extract from a different index.
432ExtractElementInst *VectorCombine::getShuffleExtract(
433 ExtractElementInst *Ext0, ExtractElementInst *Ext1,
434 unsigned PreferredExtractIndex = InvalidIndex) const {
435 auto *Index0C = dyn_cast<ConstantInt>(Val: Ext0->getIndexOperand());
436 auto *Index1C = dyn_cast<ConstantInt>(Val: Ext1->getIndexOperand());
437 assert(Index0C && Index1C && "Expected constant extract indexes");
438
439 unsigned Index0 = Index0C->getZExtValue();
440 unsigned Index1 = Index1C->getZExtValue();
441
442 // If the extract indexes are identical, no shuffle is needed.
443 if (Index0 == Index1)
444 return nullptr;
445
446 Type *VecTy = Ext0->getVectorOperand()->getType();
447 assert(VecTy == Ext1->getVectorOperand()->getType() && "Need matching types");
448 InstructionCost Cost0 =
449 TTI.getVectorInstrCost(I: *Ext0, Val: VecTy, CostKind, Index: Index0);
450 InstructionCost Cost1 =
451 TTI.getVectorInstrCost(I: *Ext1, Val: VecTy, CostKind, Index: Index1);
452
453 // If both costs are invalid no shuffle is needed
454 if (!Cost0.isValid() && !Cost1.isValid())
455 return nullptr;
456
457 // We are extracting from 2 different indexes, so one operand must be shuffled
458 // before performing a vector operation and/or extract. The more expensive
459 // extract will be replaced by a shuffle.
460 if (Cost0 > Cost1)
461 return Ext0;
462 if (Cost1 > Cost0)
463 return Ext1;
464
465 // If the costs are equal and there is a preferred extract index, shuffle the
466 // opposite operand.
467 if (PreferredExtractIndex == Index0)
468 return Ext1;
469 if (PreferredExtractIndex == Index1)
470 return Ext0;
471
472 // Otherwise, replace the extract with the higher index.
473 return Index0 > Index1 ? Ext0 : Ext1;
474}
475
476/// Compare the relative costs of 2 extracts followed by scalar operation vs.
477/// vector operation(s) followed by extract. Return true if the existing
478/// instructions are cheaper than a vector alternative. Otherwise, return false
479/// and if one of the extracts should be transformed to a shufflevector, set
480/// \p ConvertToShuffle to that extract instruction.
481bool VectorCombine::isExtractExtractCheap(ExtractElementInst *Ext0,
482 ExtractElementInst *Ext1,
483 const Instruction &I,
484 ExtractElementInst *&ConvertToShuffle,
485 unsigned PreferredExtractIndex) {
486 auto *Ext0IndexC = dyn_cast<ConstantInt>(Val: Ext0->getIndexOperand());
487 auto *Ext1IndexC = dyn_cast<ConstantInt>(Val: Ext1->getIndexOperand());
488 assert(Ext0IndexC && Ext1IndexC && "Expected constant extract indexes");
489
490 unsigned Opcode = I.getOpcode();
491 Value *Ext0Src = Ext0->getVectorOperand();
492 Value *Ext1Src = Ext1->getVectorOperand();
493 Type *ScalarTy = Ext0->getType();
494 auto *VecTy = cast<VectorType>(Val: Ext0Src->getType());
495 InstructionCost ScalarOpCost, VectorOpCost;
496
497 // Get cost estimates for scalar and vector versions of the operation.
498 bool IsBinOp = Instruction::isBinaryOp(Opcode);
499 if (IsBinOp) {
500 ScalarOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: ScalarTy, CostKind);
501 VectorOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
502 } else {
503 assert((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
504 "Expected a compare");
505 CmpInst::Predicate Pred = cast<CmpInst>(Val: I).getPredicate();
506 ScalarOpCost = TTI.getCmpSelInstrCost(
507 Opcode, ValTy: ScalarTy, CondTy: CmpInst::makeCmpResultType(opnd_type: ScalarTy), VecPred: Pred, CostKind);
508 VectorOpCost = TTI.getCmpSelInstrCost(
509 Opcode, ValTy: VecTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VecTy), VecPred: Pred, CostKind);
510 }
511
512 // Get cost estimates for the extract elements. These costs will factor into
513 // both sequences.
514 unsigned Ext0Index = Ext0IndexC->getZExtValue();
515 unsigned Ext1Index = Ext1IndexC->getZExtValue();
516
517 InstructionCost Extract0Cost =
518 TTI.getVectorInstrCost(I: *Ext0, Val: VecTy, CostKind, Index: Ext0Index);
519 InstructionCost Extract1Cost =
520 TTI.getVectorInstrCost(I: *Ext1, Val: VecTy, CostKind, Index: Ext1Index);
521
522 // A more expensive extract will always be replaced by a splat shuffle.
523 // For example, if Ext0 is more expensive:
524 // opcode (extelt V0, Ext0), (ext V1, Ext1) -->
525 // extelt (opcode (splat V0, Ext0), V1), Ext1
526 // TODO: Evaluate whether that always results in lowest cost. Alternatively,
527 // check the cost of creating a broadcast shuffle and shuffling both
528 // operands to element 0.
529 unsigned BestExtIndex = Extract0Cost > Extract1Cost ? Ext0Index : Ext1Index;
530 unsigned BestInsIndex = Extract0Cost > Extract1Cost ? Ext1Index : Ext0Index;
531 InstructionCost CheapExtractCost = std::min(a: Extract0Cost, b: Extract1Cost);
532
533 // Extra uses of the extracts mean that we include those costs in the
534 // vector total because those instructions will not be eliminated.
535 InstructionCost OldCost, NewCost;
536 if (Ext0Src == Ext1Src && Ext0Index == Ext1Index) {
537 // Handle a special case. If the 2 extracts are identical, adjust the
538 // formulas to account for that. The extra use charge allows for either the
539 // CSE'd pattern or an unoptimized form with identical values:
540 // opcode (extelt V, C), (extelt V, C) --> extelt (opcode V, V), C
541 bool HasUseTax = Ext0 == Ext1 ? !Ext0->hasNUses(N: 2)
542 : !Ext0->hasOneUse() || !Ext1->hasOneUse();
543 OldCost = CheapExtractCost + ScalarOpCost;
544 NewCost = VectorOpCost + CheapExtractCost + HasUseTax * CheapExtractCost;
545 } else {
546 // Handle the general case. Each extract is actually a different value:
547 // opcode (extelt V0, C0), (extelt V1, C1) --> extelt (opcode V0, V1), C
548 OldCost = Extract0Cost + Extract1Cost + ScalarOpCost;
549 NewCost = VectorOpCost + CheapExtractCost +
550 !Ext0->hasOneUse() * Extract0Cost +
551 !Ext1->hasOneUse() * Extract1Cost;
552 }
553
554 ConvertToShuffle = getShuffleExtract(Ext0, Ext1, PreferredExtractIndex);
555 if (ConvertToShuffle) {
556 if (IsBinOp && DisableBinopExtractShuffle)
557 return true;
558
559 // If we are extracting from 2 different indexes, then one operand must be
560 // shuffled before performing the vector operation. The shuffle mask is
561 // poison except for 1 lane that is being translated to the remaining
562 // extraction lane. Therefore, it is a splat shuffle. Ex:
563 // ShufMask = { poison, poison, 0, poison }
564 // TODO: The cost model has an option for a "broadcast" shuffle
565 // (splat-from-element-0), but no option for a more general splat.
566 if (auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: VecTy)) {
567 SmallVector<int> ShuffleMask(FixedVecTy->getNumElements(),
568 PoisonMaskElem);
569 ShuffleMask[BestInsIndex] = BestExtIndex;
570 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
571 DstTy: VecTy, SrcTy: VecTy, CostKind, Mask: ShuffleMask, Index: 0,
572 SubTp: nullptr, Args: {ConvertToShuffle});
573 } else {
574 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
575 DstTy: VecTy, SrcTy: VecTy, CostKind, Mask: {}, Index: 0, SubTp: nullptr,
576 Args: {ConvertToShuffle});
577 }
578 }
579
580 LLVM_DEBUG(dbgs() << "Found a binop of extractions: " << I << "\n OldCost: "
581 << OldCost << " vs NewCost: " << NewCost << "\n");
582
583 // Aggressively form a vector op if the cost is equal because the transform
584 // may enable further optimization.
585 // Codegen can reverse this transform (scalarize) if it was not profitable.
586 return OldCost < NewCost;
587}
588
589/// Create a shuffle that translates (shifts) 1 element from the input vector
590/// to a new element location.
591static Value *createShiftShuffle(Value *Vec, unsigned OldIndex,
592 unsigned NewIndex, IRBuilderBase &Builder) {
593 // The shuffle mask is poison except for 1 lane that is being translated
594 // to the new element index. Example for OldIndex == 2 and NewIndex == 0:
595 // ShufMask = { 2, poison, poison, poison }
596 auto *VecTy = cast<FixedVectorType>(Val: Vec->getType());
597 SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
598 ShufMask[NewIndex] = OldIndex;
599 return Builder.CreateShuffleVector(V: Vec, Mask: ShufMask, Name: "shift");
600}
601
602/// Given an extract element instruction with constant index operand, shuffle
603/// the source vector (shift the scalar element) to a NewIndex for extraction.
604/// Return null if the input can be constant folded, so that we are not creating
605/// unnecessary instructions.
606static Value *translateExtract(ExtractElementInst *ExtElt, unsigned NewIndex,
607 IRBuilderBase &Builder) {
608 // Shufflevectors can only be created for fixed-width vectors.
609 Value *X = ExtElt->getVectorOperand();
610 if (!isa<FixedVectorType>(Val: X->getType()))
611 return nullptr;
612
613 // If the extract can be constant-folded, this code is unsimplified. Defer
614 // to other passes to handle that.
615 Value *C = ExtElt->getIndexOperand();
616 assert(isa<ConstantInt>(C) && "Expected a constant index operand");
617 if (isa<Constant>(Val: X))
618 return nullptr;
619
620 Value *Shuf = createShiftShuffle(Vec: X, OldIndex: cast<ConstantInt>(Val: C)->getZExtValue(),
621 NewIndex, Builder);
622 return Shuf;
623}
624
625/// Try to reduce extract element costs by converting scalar compares to vector
626/// compares followed by extract.
627/// cmp (ext0 V0, ExtIndex), (ext1 V1, ExtIndex)
628Value *VectorCombine::foldExtExtCmp(Value *V0, Value *V1, Value *ExtIndex,
629 Instruction &I) {
630 assert(isa<CmpInst>(&I) && "Expected a compare");
631
632 // cmp Pred (extelt V0, ExtIndex), (extelt V1, ExtIndex)
633 // --> extelt (cmp Pred V0, V1), ExtIndex
634 ++NumVecCmp;
635 CmpInst::Predicate Pred = cast<CmpInst>(Val: &I)->getPredicate();
636 Value *VecCmp = Builder.CreateCmp(Pred, LHS: V0, RHS: V1);
637 return Builder.CreateExtractElement(Vec: VecCmp, Idx: ExtIndex, Name: "foldExtExtCmp");
638}
639
640/// Try to reduce extract element costs by converting scalar binops to vector
641/// binops followed by extract.
642/// bo (ext0 V0, ExtIndex), (ext1 V1, ExtIndex)
643Value *VectorCombine::foldExtExtBinop(Value *V0, Value *V1, Value *ExtIndex,
644 Instruction &I) {
645 assert(isa<BinaryOperator>(&I) && "Expected a binary operator");
646
647 // bo (extelt V0, ExtIndex), (extelt V1, ExtIndex)
648 // --> extelt (bo V0, V1), ExtIndex
649 ++NumVecBO;
650 Value *VecBO = Builder.CreateBinOp(Opc: cast<BinaryOperator>(Val: &I)->getOpcode(), LHS: V0,
651 RHS: V1, Name: "foldExtExtBinop");
652
653 // All IR flags are safe to back-propagate because any potential poison
654 // created in unused vector elements is discarded by the extract.
655 if (auto *VecBOInst = dyn_cast<Instruction>(Val: VecBO))
656 VecBOInst->copyIRFlags(V: &I);
657
658 return Builder.CreateExtractElement(Vec: VecBO, Idx: ExtIndex, Name: "foldExtExtBinop");
659}
660
661/// Match an instruction with extracted vector operands.
662bool VectorCombine::foldExtractExtract(Instruction &I) {
663 // It is not safe to transform things like div, urem, etc. because we may
664 // create undefined behavior when executing those on unknown vector elements.
665 if (!isSafeToSpeculativelyExecute(I: &I))
666 return false;
667
668 Instruction *I0, *I1;
669 CmpPredicate Pred = CmpInst::BAD_ICMP_PREDICATE;
670 if (!match(V: &I, P: m_Cmp(Pred, L: m_Instruction(I&: I0), R: m_Instruction(I&: I1))) &&
671 !match(V: &I, P: m_BinOp(L: m_Instruction(I&: I0), R: m_Instruction(I&: I1))))
672 return false;
673
674 Value *V0, *V1;
675 uint64_t C0, C1;
676 if (!match(V: I0, P: m_ExtractElt(Val: m_Value(V&: V0), Idx: m_ConstantInt(V&: C0))) ||
677 !match(V: I1, P: m_ExtractElt(Val: m_Value(V&: V1), Idx: m_ConstantInt(V&: C1))) ||
678 V0->getType() != V1->getType())
679 return false;
680
681 // For fixed-width vectors, reject out-of-bounds extract indexes
682 if (auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: V0->getType())) {
683 unsigned NumElts = FixedVecTy->getNumElements();
684 if (C0 >= NumElts || C1 >= NumElts)
685 return false;
686 }
687
688 // If the scalar value 'I' is going to be re-inserted into a vector, then try
689 // to create an extract to that same element. The extract/insert can be
690 // reduced to a "select shuffle".
691 // TODO: If we add a larger pattern match that starts from an insert, this
692 // probably becomes unnecessary.
693 auto *Ext0 = cast<ExtractElementInst>(Val: I0);
694 auto *Ext1 = cast<ExtractElementInst>(Val: I1);
695 uint64_t InsertIndex = InvalidIndex;
696 if (I.hasOneUse())
697 match(V: I.user_back(),
698 P: m_InsertElt(Val: m_Value(), Elt: m_Value(), Idx: m_ConstantInt(V&: InsertIndex)));
699
700 ExtractElementInst *ExtractToChange;
701 if (isExtractExtractCheap(Ext0, Ext1, I, ConvertToShuffle&: ExtractToChange, PreferredExtractIndex: InsertIndex))
702 return false;
703
704 Value *ExtOp0 = Ext0->getVectorOperand();
705 Value *ExtOp1 = Ext1->getVectorOperand();
706
707 if (ExtractToChange) {
708 unsigned CheapExtractIdx = ExtractToChange == Ext0 ? C1 : C0;
709 Value *NewExtOp =
710 translateExtract(ExtElt: ExtractToChange, NewIndex: CheapExtractIdx, Builder);
711 if (!NewExtOp)
712 return false;
713 if (ExtractToChange == Ext0)
714 ExtOp0 = NewExtOp;
715 else
716 ExtOp1 = NewExtOp;
717 }
718
719 Value *ExtIndex = ExtractToChange == Ext0 ? Ext1->getIndexOperand()
720 : Ext0->getIndexOperand();
721 Value *NewExt = Pred != CmpInst::BAD_ICMP_PREDICATE
722 ? foldExtExtCmp(V0: ExtOp0, V1: ExtOp1, ExtIndex, I)
723 : foldExtExtBinop(V0: ExtOp0, V1: ExtOp1, ExtIndex, I);
724 Worklist.push(I: Ext0);
725 Worklist.push(I: Ext1);
726 replaceValue(Old&: I, New&: *NewExt);
727 return true;
728}
729
730/// Try to replace an extract + scalar fneg + insert with a vector fneg +
731/// shuffle.
732bool VectorCombine::foldInsExtFNeg(Instruction &I) {
733 // Match an insert (op (extract)) pattern.
734 Value *DstVec;
735 uint64_t ExtIdx, InsIdx;
736 Instruction *FNeg;
737 if (!match(V: &I, P: m_InsertElt(Val: m_Value(V&: DstVec), Elt: m_OneUse(SubPattern: m_Instruction(I&: FNeg)),
738 Idx: m_ConstantInt(V&: InsIdx))))
739 return false;
740
741 // Note: This handles the canonical fneg instruction and "fsub -0.0, X".
742 Value *SrcVec;
743 Instruction *Extract;
744 if (!match(V: FNeg, P: m_FNeg(X: m_CombineAnd(
745 Ps: m_Instruction(I&: Extract),
746 Ps: m_ExtractElt(Val: m_Value(V&: SrcVec), Idx: m_ConstantInt(V&: ExtIdx))))))
747 return false;
748
749 auto *DstVecTy = cast<FixedVectorType>(Val: DstVec->getType());
750 auto *DstVecScalarTy = DstVecTy->getScalarType();
751 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcVec->getType());
752 if (!SrcVecTy || DstVecScalarTy != SrcVecTy->getScalarType())
753 return false;
754
755 // Ignore if insert/extract index is out of bounds or destination vector has
756 // one element
757 unsigned NumDstElts = DstVecTy->getNumElements();
758 unsigned NumSrcElts = SrcVecTy->getNumElements();
759 if (ExtIdx > NumSrcElts || InsIdx >= NumDstElts || NumDstElts == 1)
760 return false;
761
762 // We are inserting the negated element into the same lane that we extracted
763 // from. This is equivalent to a select-shuffle that chooses all but the
764 // negated element from the destination vector.
765 SmallVector<int> Mask(NumDstElts);
766 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
767 Mask[InsIdx] = (ExtIdx % NumDstElts) + NumDstElts;
768 InstructionCost OldCost =
769 TTI.getArithmeticInstrCost(Opcode: Instruction::FNeg, Ty: DstVecScalarTy, CostKind) +
770 TTI.getVectorInstrCost(I, Val: DstVecTy, CostKind, Index: InsIdx);
771
772 // If the extract has one use, it will be eliminated, so count it in the
773 // original cost. If it has more than one use, ignore the cost because it will
774 // be the same before/after.
775 if (Extract->hasOneUse())
776 OldCost += TTI.getVectorInstrCost(I: *Extract, Val: SrcVecTy, CostKind, Index: ExtIdx);
777
778 InstructionCost NewCost =
779 TTI.getArithmeticInstrCost(Opcode: Instruction::FNeg, Ty: SrcVecTy, CostKind) +
780 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: DstVecTy,
781 SrcTy: DstVecTy, CostKind, Mask);
782
783 bool NeedLenChg = SrcVecTy->getNumElements() != NumDstElts;
784 // If the lengths of the two vectors are not equal,
785 // we need to add a length-change vector. Add this cost.
786 SmallVector<int> SrcMask;
787 if (NeedLenChg) {
788 SrcMask.assign(NumElts: NumDstElts, Elt: PoisonMaskElem);
789 SrcMask[ExtIdx % NumDstElts] = ExtIdx;
790 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
791 DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask: SrcMask);
792 }
793
794 LLVM_DEBUG(dbgs() << "Found an insertion of (extract)fneg : " << I
795 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
796 << "\n");
797 if (NewCost > OldCost)
798 return false;
799
800 Value *NewShuf, *LenChgShuf = nullptr;
801 // insertelt DstVec, (fneg (extractelt SrcVec, Index)), Index
802 Value *VecFNeg = Builder.CreateFNegFMF(V: SrcVec, FMFSource: FNeg);
803 if (NeedLenChg) {
804 // shuffle DstVec, (shuffle (fneg SrcVec), poison, SrcMask), Mask
805 LenChgShuf = Builder.CreateShuffleVector(V: VecFNeg, Mask: SrcMask);
806 NewShuf = Builder.CreateShuffleVector(V1: DstVec, V2: LenChgShuf, Mask);
807 Worklist.pushValue(V: LenChgShuf);
808 } else {
809 // shuffle DstVec, (fneg SrcVec), Mask
810 NewShuf = Builder.CreateShuffleVector(V1: DstVec, V2: VecFNeg, Mask);
811 }
812
813 Worklist.pushValue(V: VecFNeg);
814 replaceValue(Old&: I, New&: *NewShuf);
815 return true;
816}
817
818/// Try to fold insert(binop(x,y),binop(a,b),idx)
819/// --> binop(insert(x,a,idx),insert(y,b,idx))
820bool VectorCombine::foldInsExtBinop(Instruction &I) {
821 BinaryOperator *VecBinOp, *SclBinOp;
822 uint64_t Index;
823 if (!match(V: &I,
824 P: m_InsertElt(Val: m_OneUse(SubPattern: m_BinOp(I&: VecBinOp)),
825 Elt: m_OneUse(SubPattern: m_BinOp(I&: SclBinOp)), Idx: m_ConstantInt(V&: Index))))
826 return false;
827
828 // TODO: Add support for addlike etc.
829 Instruction::BinaryOps BinOpcode = VecBinOp->getOpcode();
830 if (BinOpcode != SclBinOp->getOpcode())
831 return false;
832
833 auto *ResultTy = dyn_cast<FixedVectorType>(Val: I.getType());
834 if (!ResultTy)
835 return false;
836
837 // TODO: Attempt to detect m_ExtractElt for scalar operands and convert to
838 // shuffle?
839
840 InstructionCost OldCost = TTI.getInstructionCost(U: &I, CostKind) +
841 TTI.getInstructionCost(U: VecBinOp, CostKind) +
842 TTI.getInstructionCost(U: SclBinOp, CostKind);
843 InstructionCost NewCost =
844 TTI.getArithmeticInstrCost(Opcode: BinOpcode, Ty: ResultTy, CostKind) +
845 TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: ResultTy, CostKind,
846 Index, Op0: VecBinOp->getOperand(i_nocapture: 0),
847 Op1: SclBinOp->getOperand(i_nocapture: 0)) +
848 TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: ResultTy, CostKind,
849 Index, Op0: VecBinOp->getOperand(i_nocapture: 1),
850 Op1: SclBinOp->getOperand(i_nocapture: 1));
851
852 LLVM_DEBUG(dbgs() << "Found an insertion of two binops: " << I
853 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
854 << "\n");
855 if (NewCost > OldCost)
856 return false;
857
858 Value *NewIns0 = Builder.CreateInsertElement(Vec: VecBinOp->getOperand(i_nocapture: 0),
859 NewElt: SclBinOp->getOperand(i_nocapture: 0), Idx: Index);
860 Value *NewIns1 = Builder.CreateInsertElement(Vec: VecBinOp->getOperand(i_nocapture: 1),
861 NewElt: SclBinOp->getOperand(i_nocapture: 1), Idx: Index);
862 Value *NewBO = Builder.CreateBinOp(Opc: BinOpcode, LHS: NewIns0, RHS: NewIns1);
863
864 // Intersect flags from the old binops.
865 if (auto *NewInst = dyn_cast<Instruction>(Val: NewBO)) {
866 NewInst->copyIRFlags(V: VecBinOp);
867 NewInst->andIRFlags(V: SclBinOp);
868 }
869
870 Worklist.pushValue(V: NewIns0);
871 Worklist.pushValue(V: NewIns1);
872 replaceValue(Old&: I, New&: *NewBO);
873 return true;
874}
875
876/// Match: bitop(castop(x), castop(y)) -> castop(bitop(x, y))
877/// Supports: bitcast, trunc, sext, zext
878bool VectorCombine::foldBitOpOfCastops(Instruction &I) {
879 // Check if this is a bitwise logic operation
880 auto *BinOp = dyn_cast<BinaryOperator>(Val: &I);
881 if (!BinOp || !BinOp->isBitwiseLogicOp())
882 return false;
883
884 // Get the cast instructions
885 auto *LHSCast = dyn_cast<CastInst>(Val: BinOp->getOperand(i_nocapture: 0));
886 auto *RHSCast = dyn_cast<CastInst>(Val: BinOp->getOperand(i_nocapture: 1));
887 if (!LHSCast || !RHSCast) {
888 LLVM_DEBUG(dbgs() << " One or both operands are not cast instructions\n");
889 return false;
890 }
891
892 // Both casts must be the same type
893 Instruction::CastOps CastOpcode = LHSCast->getOpcode();
894 if (CastOpcode != RHSCast->getOpcode())
895 return false;
896
897 // Only handle supported cast operations
898 switch (CastOpcode) {
899 case Instruction::BitCast:
900 case Instruction::Trunc:
901 case Instruction::SExt:
902 case Instruction::ZExt:
903 break;
904 default:
905 return false;
906 }
907
908 Value *LHSSrc = LHSCast->getOperand(i_nocapture: 0);
909 Value *RHSSrc = RHSCast->getOperand(i_nocapture: 0);
910
911 // Source types must match
912 if (LHSSrc->getType() != RHSSrc->getType())
913 return false;
914
915 auto *SrcTy = LHSSrc->getType();
916 auto *DstTy = I.getType();
917 // Bitcasts can handle scalar/vector mixes, such as i16 -> <16 x i1>.
918 // Other casts only handle vector types with integer elements.
919 if (CastOpcode != Instruction::BitCast &&
920 (!isa<FixedVectorType>(Val: SrcTy) || !isa<FixedVectorType>(Val: DstTy)))
921 return false;
922
923 // Only integer scalar/vector values are legal for bitwise logic operations.
924 if (!SrcTy->getScalarType()->isIntegerTy() ||
925 !DstTy->getScalarType()->isIntegerTy())
926 return false;
927
928 // Cost Check :
929 // OldCost = bitlogic + 2*casts
930 // NewCost = bitlogic + cast
931
932 // Calculate specific costs for each cast with instruction context
933 InstructionCost LHSCastCost = TTI.getCastInstrCost(
934 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind, I: LHSCast);
935 InstructionCost RHSCastCost = TTI.getCastInstrCost(
936 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind, I: RHSCast);
937
938 InstructionCost OldCost =
939 TTI.getArithmeticInstrCost(Opcode: BinOp->getOpcode(), Ty: DstTy, CostKind) +
940 LHSCastCost + RHSCastCost;
941
942 // For new cost, we can't provide an instruction (it doesn't exist yet)
943 InstructionCost GenericCastCost = TTI.getCastInstrCost(
944 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
945
946 InstructionCost NewCost =
947 TTI.getArithmeticInstrCost(Opcode: BinOp->getOpcode(), Ty: SrcTy, CostKind) +
948 GenericCastCost;
949
950 // Account for multi-use casts using specific costs
951 if (!LHSCast->hasOneUse())
952 NewCost += LHSCastCost;
953 if (!RHSCast->hasOneUse())
954 NewCost += RHSCastCost;
955
956 LLVM_DEBUG(dbgs() << "foldBitOpOfCastops: OldCost=" << OldCost
957 << " NewCost=" << NewCost << "\n");
958
959 if (NewCost > OldCost)
960 return false;
961
962 // Create the operation on the source type
963 Value *NewOp = Builder.CreateBinOp(Opc: BinOp->getOpcode(), LHS: LHSSrc, RHS: RHSSrc,
964 Name: BinOp->getName() + ".inner");
965 if (auto *NewBinOp = dyn_cast<BinaryOperator>(Val: NewOp))
966 NewBinOp->copyIRFlags(V: BinOp);
967
968 Worklist.pushValue(V: NewOp);
969
970 // Create the cast operation directly to ensure we get a new instruction
971 Instruction *NewCast = CastInst::Create(CastOpcode, S: NewOp, Ty: I.getType());
972
973 // Preserve cast instruction flags
974 NewCast->copyIRFlags(V: LHSCast);
975 NewCast->andIRFlags(V: RHSCast);
976
977 // Insert the new instruction
978 Value *Result = Builder.Insert(I: NewCast);
979
980 replaceValue(Old&: I, New&: *Result);
981 return true;
982}
983
984/// Match:
985// bitop(castop(x), C) ->
986// bitop(castop(x), castop(InvC)) ->
987// castop(bitop(x, InvC))
988// Supports: bitcast
989bool VectorCombine::foldBitOpOfCastConstant(Instruction &I) {
990 Instruction *LHS;
991 Constant *C;
992
993 // Check if this is a bitwise logic operation
994 if (!match(V: &I, P: m_c_BitwiseLogic(L: m_Instruction(I&: LHS), R: m_Constant(C))))
995 return false;
996
997 // Get the cast instructions
998 auto *LHSCast = dyn_cast<CastInst>(Val: LHS);
999 if (!LHSCast)
1000 return false;
1001
1002 Instruction::CastOps CastOpcode = LHSCast->getOpcode();
1003
1004 // Only handle supported cast operations
1005 switch (CastOpcode) {
1006 case Instruction::BitCast:
1007 case Instruction::ZExt:
1008 case Instruction::SExt:
1009 case Instruction::Trunc:
1010 break;
1011 default:
1012 return false;
1013 }
1014
1015 Value *LHSSrc = LHSCast->getOperand(i_nocapture: 0);
1016
1017 auto *SrcTy = LHSSrc->getType();
1018 auto *DstTy = I.getType();
1019 // Bitcasts can handle scalar/vector mixes, such as i16 -> <16 x i1>.
1020 // Other casts only handle vector types with integer elements.
1021 if (CastOpcode != Instruction::BitCast &&
1022 (!isa<FixedVectorType>(Val: SrcTy) || !isa<FixedVectorType>(Val: DstTy)))
1023 return false;
1024
1025 // Only integer scalar/vector values are legal for bitwise logic operations.
1026 if (!SrcTy->getScalarType()->isIntegerTy() ||
1027 !DstTy->getScalarType()->isIntegerTy())
1028 return false;
1029
1030 // Find the constant InvC, such that castop(InvC) equals to C.
1031 PreservedCastFlags RHSFlags;
1032 Constant *InvC = getLosslessInvCast(C, InvCastTo: SrcTy, CastOp: CastOpcode, DL: *DL, Flags: &RHSFlags);
1033 if (!InvC)
1034 return false;
1035
1036 // Cost Check :
1037 // OldCost = bitlogic + cast
1038 // NewCost = bitlogic + cast
1039
1040 // Calculate specific costs for each cast with instruction context
1041 InstructionCost LHSCastCost = TTI.getCastInstrCost(
1042 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind, I: LHSCast);
1043
1044 InstructionCost OldCost =
1045 TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: DstTy, CostKind) + LHSCastCost;
1046
1047 // For new cost, we can't provide an instruction (it doesn't exist yet)
1048 InstructionCost GenericCastCost = TTI.getCastInstrCost(
1049 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
1050
1051 InstructionCost NewCost =
1052 TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: SrcTy, CostKind) +
1053 GenericCastCost;
1054
1055 // Account for multi-use casts using specific costs
1056 if (!LHSCast->hasOneUse())
1057 NewCost += LHSCastCost;
1058
1059 LLVM_DEBUG(dbgs() << "foldBitOpOfCastConstant: OldCost=" << OldCost
1060 << " NewCost=" << NewCost << "\n");
1061
1062 if (NewCost > OldCost)
1063 return false;
1064
1065 // Create the operation on the source type
1066 Value *NewOp = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)I.getOpcode(),
1067 LHS: LHSSrc, RHS: InvC, Name: I.getName() + ".inner");
1068 if (auto *NewBinOp = dyn_cast<BinaryOperator>(Val: NewOp))
1069 NewBinOp->copyIRFlags(V: &I);
1070
1071 Worklist.pushValue(V: NewOp);
1072
1073 // Create the cast operation directly to ensure we get a new instruction
1074 Instruction *NewCast = CastInst::Create(CastOpcode, S: NewOp, Ty: I.getType());
1075
1076 // Preserve cast instruction flags
1077 if (RHSFlags.NNeg)
1078 NewCast->setNonNeg();
1079 if (RHSFlags.NUW)
1080 NewCast->setHasNoUnsignedWrap();
1081 if (RHSFlags.NSW)
1082 NewCast->setHasNoSignedWrap();
1083
1084 NewCast->andIRFlags(V: LHSCast);
1085
1086 // Insert the new instruction
1087 Value *Result = Builder.Insert(I: NewCast);
1088
1089 replaceValue(Old&: I, New&: *Result);
1090 return true;
1091}
1092
1093/// If this is a bitcast of a shuffle, try to bitcast the source vector to the
1094/// destination type followed by shuffle. This can enable further transforms by
1095/// moving bitcasts or shuffles together.
1096bool VectorCombine::foldBitcastShuffle(Instruction &I) {
1097 Value *V0, *V1;
1098 ArrayRef<int> Mask;
1099 if (!match(V: &I, P: m_BitCast(Op: m_OneUse(
1100 SubPattern: m_Shuffle(v1: m_Value(V&: V0), v2: m_Value(V&: V1), mask: m_Mask(Mask))))))
1101 return false;
1102
1103 // 1) Do not fold bitcast shuffle for scalable type. First, shuffle cost for
1104 // scalable type is unknown; Second, we cannot reason if the narrowed shuffle
1105 // mask for scalable type is a splat or not.
1106 // 2) Disallow non-vector casts.
1107 // TODO: We could allow any shuffle.
1108 auto *DestTy = dyn_cast<FixedVectorType>(Val: I.getType());
1109 auto *SrcTy = dyn_cast<FixedVectorType>(Val: V0->getType());
1110 if (!DestTy || !SrcTy)
1111 return false;
1112
1113 unsigned DestEltSize = DestTy->getScalarSizeInBits();
1114 unsigned SrcEltSize = SrcTy->getScalarSizeInBits();
1115 if (SrcTy->getPrimitiveSizeInBits() % DestEltSize != 0)
1116 return false;
1117
1118 bool IsUnary = isa<UndefValue>(Val: V1);
1119
1120 // For binary shuffles, only fold bitcast(shuffle(X,Y))
1121 // if it won't increase the number of bitcasts.
1122 if (!IsUnary) {
1123 auto *BCTy0 = dyn_cast<FixedVectorType>(Val: peekThroughBitcasts(V: V0)->getType());
1124 auto *BCTy1 = dyn_cast<FixedVectorType>(Val: peekThroughBitcasts(V: V1)->getType());
1125 if (!(BCTy0 && BCTy0->getElementType() == DestTy->getElementType()) &&
1126 !(BCTy1 && BCTy1->getElementType() == DestTy->getElementType()))
1127 return false;
1128 }
1129
1130 SmallVector<int, 16> NewMask;
1131 if (DestEltSize <= SrcEltSize) {
1132 // The bitcast is from wide to narrow/equal elements. The shuffle mask can
1133 // always be expanded to the equivalent form choosing narrower elements.
1134 if (SrcEltSize % DestEltSize != 0)
1135 return false;
1136 unsigned ScaleFactor = SrcEltSize / DestEltSize;
1137 narrowShuffleMaskElts(Scale: ScaleFactor, Mask, ScaledMask&: NewMask);
1138 } else {
1139 // The bitcast is from narrow elements to wide elements. The shuffle mask
1140 // must choose consecutive elements to allow casting first.
1141 if (DestEltSize % SrcEltSize != 0)
1142 return false;
1143 unsigned ScaleFactor = DestEltSize / SrcEltSize;
1144 if (!widenShuffleMaskElts(Scale: ScaleFactor, Mask, ScaledMask&: NewMask))
1145 return false;
1146 }
1147
1148 // Bitcast the shuffle src - keep its original width but using the destination
1149 // scalar type.
1150 unsigned NumSrcElts = SrcTy->getPrimitiveSizeInBits() / DestEltSize;
1151 auto *NewShuffleTy =
1152 FixedVectorType::get(ElementType: DestTy->getScalarType(), NumElts: NumSrcElts);
1153 auto *OldShuffleTy =
1154 FixedVectorType::get(ElementType: SrcTy->getScalarType(), NumElts: Mask.size());
1155 unsigned NumOps = IsUnary ? 1 : 2;
1156
1157 // The new shuffle must not cost more than the old shuffle.
1158 TargetTransformInfo::ShuffleKind SK =
1159 IsUnary ? TargetTransformInfo::SK_PermuteSingleSrc
1160 : TargetTransformInfo::SK_PermuteTwoSrc;
1161
1162 InstructionCost NewCost =
1163 TTI.getShuffleCost(Kind: SK, DstTy: DestTy, SrcTy: NewShuffleTy, CostKind, Mask: NewMask) +
1164 (NumOps * TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: NewShuffleTy, Src: SrcTy,
1165 CCH: TargetTransformInfo::CastContextHint::None,
1166 CostKind));
1167 InstructionCost OldCost =
1168 TTI.getShuffleCost(Kind: SK, DstTy: OldShuffleTy, SrcTy, CostKind, Mask) +
1169 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: DestTy, Src: OldShuffleTy,
1170 CCH: TargetTransformInfo::CastContextHint::None,
1171 CostKind);
1172
1173 LLVM_DEBUG(dbgs() << "Found a bitcasted shuffle: " << I << "\n OldCost: "
1174 << OldCost << " vs NewCost: " << NewCost << "\n");
1175
1176 if (NewCost > OldCost || !NewCost.isValid())
1177 return false;
1178
1179 // bitcast (shuf V0, V1, MaskC) --> shuf (bitcast V0), (bitcast V1), MaskC'
1180 ++NumShufOfBitcast;
1181 Value *CastV0 = Builder.CreateBitCast(V: peekThroughBitcasts(V: V0), DestTy: NewShuffleTy);
1182 Value *CastV1 = Builder.CreateBitCast(V: peekThroughBitcasts(V: V1), DestTy: NewShuffleTy);
1183 Value *Shuf = Builder.CreateShuffleVector(V1: CastV0, V2: CastV1, Mask: NewMask);
1184 replaceValue(Old&: I, New&: *Shuf);
1185 return true;
1186}
1187
1188/// Match a vector op/compare/intrinsic with at least one
1189/// inserted scalar operand and convert to scalar op/cmp/intrinsic followed
1190/// by insertelement.
1191bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
1192 auto *UO = dyn_cast<UnaryOperator>(Val: &I);
1193 auto *BO = dyn_cast<BinaryOperator>(Val: &I);
1194 auto *CI = dyn_cast<CmpInst>(Val: &I);
1195 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
1196 if (!UO && !BO && !CI && !II)
1197 return false;
1198
1199 // TODO: Allow intrinsics with different argument types
1200 if (II) {
1201 if (!isTriviallyVectorizable(ID: II->getIntrinsicID()))
1202 return false;
1203 for (auto [Idx, Arg] : enumerate(First: II->args()))
1204 if (Arg->getType() != II->getType() &&
1205 !isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(), ScalarOpdIdx: Idx, TTI: &TTI))
1206 return false;
1207 }
1208
1209 // Do not convert the vector condition of a vector select into a scalar
1210 // condition. That may cause problems for codegen because of differences in
1211 // boolean formats and register-file transfers.
1212 // TODO: Can we account for that in the cost model?
1213 if (CI)
1214 for (User *U : I.users())
1215 if (match(V: U, P: m_Select(C: m_Specific(V: &I), L: m_Value(), R: m_Value())))
1216 return false;
1217
1218 // Match constant vectors or scalars being inserted into constant vectors:
1219 // vec_op [VecC0 | (inselt VecC0, V0, Index)], ...
1220 SmallVector<Value *> VecCs, ScalarOps;
1221 std::optional<uint64_t> Index;
1222
1223 auto Ops = II ? II->args() : I.operands();
1224 for (auto [OpNum, Op] : enumerate(First&: Ops)) {
1225 Constant *VecC;
1226 Value *V;
1227 uint64_t InsIdx = 0;
1228 if (match(V: Op.get(), P: m_InsertElt(Val: m_Constant(C&: VecC), Elt: m_Value(V),
1229 Idx: m_ConstantInt(V&: InsIdx)))) {
1230 // Bail if any inserts are out of bounds.
1231 VectorType *OpTy = cast<VectorType>(Val: Op->getType());
1232 if (OpTy->getElementCount().getKnownMinValue() <= InsIdx)
1233 return false;
1234 // All inserts must have the same index.
1235 // TODO: Deal with mismatched index constants and variable indexes?
1236 if (!Index)
1237 Index = InsIdx;
1238 else if (InsIdx != *Index)
1239 return false;
1240 VecCs.push_back(Elt: VecC);
1241 ScalarOps.push_back(Elt: V);
1242 } else if (II && isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(),
1243 ScalarOpdIdx: OpNum, TTI: &TTI)) {
1244 VecCs.push_back(Elt: Op.get());
1245 ScalarOps.push_back(Elt: Op.get());
1246 } else if (match(V: Op.get(), P: m_Constant(C&: VecC))) {
1247 VecCs.push_back(Elt: VecC);
1248 ScalarOps.push_back(Elt: nullptr);
1249 } else {
1250 return false;
1251 }
1252 }
1253
1254 // Bail if all operands are constant.
1255 if (!Index.has_value())
1256 return false;
1257
1258 VectorType *VecTy = cast<VectorType>(Val: I.getType());
1259 Type *ScalarTy = VecTy->getScalarType();
1260 assert(VecTy->isVectorTy() &&
1261 (ScalarTy->isIntegerTy() || ScalarTy->isFloatingPointTy() ||
1262 ScalarTy->isPointerTy()) &&
1263 "Unexpected types for insert element into binop or cmp");
1264
1265 unsigned Opcode = I.getOpcode();
1266 InstructionCost ScalarOpCost, VectorOpCost;
1267 if (CI) {
1268 CmpInst::Predicate Pred = CI->getPredicate();
1269 ScalarOpCost = TTI.getCmpSelInstrCost(
1270 Opcode, ValTy: ScalarTy, CondTy: CmpInst::makeCmpResultType(opnd_type: ScalarTy), VecPred: Pred, CostKind);
1271 VectorOpCost = TTI.getCmpSelInstrCost(
1272 Opcode, ValTy: VecTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VecTy), VecPred: Pred, CostKind);
1273 } else if (UO || BO) {
1274 ScalarOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: ScalarTy, CostKind);
1275 VectorOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1276 } else {
1277 IntrinsicCostAttributes ScalarICA(
1278 II->getIntrinsicID(), ScalarTy,
1279 SmallVector<Type *>(II->arg_size(), ScalarTy));
1280 ScalarOpCost = TTI.getIntrinsicInstrCost(ICA: ScalarICA, CostKind);
1281 IntrinsicCostAttributes VectorICA(
1282 II->getIntrinsicID(), VecTy,
1283 SmallVector<Type *>(II->arg_size(), VecTy));
1284 VectorOpCost = TTI.getIntrinsicInstrCost(ICA: VectorICA, CostKind);
1285 }
1286
1287 // Fold the vector constants in the original vectors into a new base vector to
1288 // get more accurate cost modelling.
1289 Value *NewVecC = nullptr;
1290 if (CI)
1291 NewVecC = simplifyCmpInst(Predicate: CI->getPredicate(), LHS: VecCs[0], RHS: VecCs[1], Q: SQ);
1292 else if (UO)
1293 NewVecC =
1294 simplifyUnOp(Opcode: UO->getOpcode(), Op: VecCs[0], FMF: UO->getFastMathFlags(), Q: SQ);
1295 else if (BO)
1296 NewVecC = simplifyBinOp(Opcode: BO->getOpcode(), LHS: VecCs[0], RHS: VecCs[1], Q: SQ);
1297 else if (II)
1298 NewVecC = simplifyCall(Call: II, Callee: II->getCalledOperand(), Args: VecCs, Q: SQ);
1299
1300 if (!NewVecC)
1301 return false;
1302
1303 // Get cost estimate for the insert element. This cost will factor into
1304 // both sequences.
1305 InstructionCost OldCost = VectorOpCost;
1306 InstructionCost NewCost =
1307 ScalarOpCost + TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: VecTy,
1308 CostKind, Index: *Index, Op0: NewVecC);
1309
1310 for (auto [Idx, Op, VecC, Scalar] : enumerate(First&: Ops, Rest&: VecCs, Rest&: ScalarOps)) {
1311 if (!Scalar || (II && isVectorIntrinsicWithScalarOpAtArg(
1312 ID: II->getIntrinsicID(), ScalarOpdIdx: Idx, TTI: &TTI)))
1313 continue;
1314 InstructionCost InsertCost = TTI.getVectorInstrCost(
1315 Opcode: Instruction::InsertElement, Val: VecTy, CostKind, Index: *Index, Op0: VecC, Op1: Scalar);
1316 OldCost += InsertCost;
1317 NewCost += !Op->hasOneUse() * InsertCost;
1318 }
1319
1320 // We want to scalarize unless the vector variant actually has lower cost.
1321 if (OldCost < NewCost || !NewCost.isValid())
1322 return false;
1323
1324 // vec_op (inselt VecC0, V0, Index), (inselt VecC1, V1, Index) -->
1325 // inselt NewVecC, (scalar_op V0, V1), Index
1326 if (CI)
1327 ++NumScalarCmp;
1328 else if (UO || BO)
1329 ++NumScalarOps;
1330 else
1331 ++NumScalarIntrinsic;
1332
1333 // For constant cases, extract the scalar element, this should constant fold.
1334 for (auto [OpIdx, Scalar, VecC] : enumerate(First&: ScalarOps, Rest&: VecCs))
1335 if (!Scalar)
1336 ScalarOps[OpIdx] = ConstantExpr::getExtractElement(
1337 Vec: cast<Constant>(Val: VecC), Idx: Builder.getInt64(C: *Index));
1338
1339 Value *Scalar;
1340 if (CI)
1341 Scalar = Builder.CreateCmp(Pred: CI->getPredicate(), LHS: ScalarOps[0], RHS: ScalarOps[1]);
1342 else if (UO || BO)
1343 Scalar = Builder.CreateNAryOp(Opc: Opcode, Ops: ScalarOps);
1344 else
1345 Scalar = Builder.CreateIntrinsic(RetTy: ScalarTy, ID: II->getIntrinsicID(), Args: ScalarOps);
1346
1347 Scalar->setName(I.getName() + ".scalar");
1348
1349 // All IR flags are safe to back-propagate. There is no potential for extra
1350 // poison to be created by the scalar instruction.
1351 if (auto *ScalarInst = dyn_cast<Instruction>(Val: Scalar))
1352 ScalarInst->copyIRFlags(V: &I);
1353
1354 Value *Insert = Builder.CreateInsertElement(Vec: NewVecC, NewElt: Scalar, Idx: *Index);
1355 replaceValue(Old&: I, New&: *Insert);
1356 return true;
1357}
1358
1359/// Try to combine a scalar binop + 2 scalar compares of extracted elements of
1360/// a vector into vector operations followed by extract. Note: The SLP pass
1361/// may miss this pattern because of implementation problems.
1362bool VectorCombine::foldExtractedCmps(Instruction &I) {
1363 auto *BI = dyn_cast<BinaryOperator>(Val: &I);
1364
1365 // We are looking for a scalar binop of booleans.
1366 // binop i1 (cmp Pred I0, C0), (cmp Pred I1, C1)
1367 if (!BI || !I.getType()->isIntegerTy(BitWidth: 1))
1368 return false;
1369
1370 // The compare predicates should match, and each compare should have a
1371 // constant operand.
1372 Value *B0 = I.getOperand(i: 0), *B1 = I.getOperand(i: 1);
1373 Instruction *I0, *I1;
1374 Constant *C0, *C1;
1375 CmpPredicate P0, P1;
1376 if (!match(V: B0, P: m_Cmp(Pred&: P0, L: m_Instruction(I&: I0), R: m_Constant(C&: C0))) ||
1377 !match(V: B1, P: m_Cmp(Pred&: P1, L: m_Instruction(I&: I1), R: m_Constant(C&: C1))))
1378 return false;
1379
1380 auto MatchingPred = CmpPredicate::getMatching(A: P0, B: P1);
1381 if (!MatchingPred)
1382 return false;
1383
1384 // The compare operands must be extracts of the same vector with constant
1385 // extract indexes.
1386 Value *X;
1387 uint64_t Index0, Index1;
1388 if (!match(V: I0, P: m_ExtractElt(Val: m_Value(V&: X), Idx: m_ConstantInt(V&: Index0))) ||
1389 !match(V: I1, P: m_ExtractElt(Val: m_Specific(V: X), Idx: m_ConstantInt(V&: Index1))))
1390 return false;
1391
1392 auto *Ext0 = cast<ExtractElementInst>(Val: I0);
1393 auto *Ext1 = cast<ExtractElementInst>(Val: I1);
1394 ExtractElementInst *ConvertToShuf = getShuffleExtract(Ext0, Ext1, PreferredExtractIndex: CostKind);
1395 if (!ConvertToShuf)
1396 return false;
1397 assert((ConvertToShuf == Ext0 || ConvertToShuf == Ext1) &&
1398 "Unknown ExtractElementInst");
1399
1400 // The original scalar pattern is:
1401 // binop i1 (cmp Pred (ext X, Index0), C0), (cmp Pred (ext X, Index1), C1)
1402 CmpInst::Predicate Pred = *MatchingPred;
1403 unsigned CmpOpcode =
1404 CmpInst::isFPPredicate(P: Pred) ? Instruction::FCmp : Instruction::ICmp;
1405 auto *VecTy = dyn_cast<FixedVectorType>(Val: X->getType());
1406 if (!VecTy)
1407 return false;
1408
1409 if (Index0 >= VecTy->getNumElements() || Index1 >= VecTy->getNumElements())
1410 return false;
1411
1412 InstructionCost Ext0Cost =
1413 TTI.getVectorInstrCost(I: *Ext0, Val: VecTy, CostKind, Index: Index0);
1414 InstructionCost Ext1Cost =
1415 TTI.getVectorInstrCost(I: *Ext1, Val: VecTy, CostKind, Index: Index1);
1416 InstructionCost CmpCost = TTI.getCmpSelInstrCost(
1417 Opcode: CmpOpcode, ValTy: I0->getType(), CondTy: CmpInst::makeCmpResultType(opnd_type: I0->getType()), VecPred: Pred,
1418 CostKind);
1419
1420 InstructionCost OldCost =
1421 Ext0Cost + Ext1Cost + CmpCost * 2 +
1422 TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: I.getType(), CostKind);
1423
1424 // The proposed vector pattern is:
1425 // vcmp = cmp Pred X, VecC
1426 // ext (binop vNi1 vcmp, (shuffle vcmp, Index1)), Index0
1427 int CheapIndex = ConvertToShuf == Ext0 ? Index1 : Index0;
1428 int ExpensiveIndex = ConvertToShuf == Ext0 ? Index0 : Index1;
1429 auto *CmpTy = cast<FixedVectorType>(Val: CmpInst::makeCmpResultType(opnd_type: VecTy));
1430 InstructionCost NewCost = TTI.getCmpSelInstrCost(
1431 Opcode: CmpOpcode, ValTy: VecTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VecTy), VecPred: Pred, CostKind);
1432 SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
1433 ShufMask[CheapIndex] = ExpensiveIndex;
1434 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc, DstTy: CmpTy,
1435 SrcTy: CmpTy, CostKind, Mask: ShufMask);
1436 NewCost += TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: CmpTy, CostKind);
1437 NewCost += TTI.getVectorInstrCost(I: *Ext0, Val: CmpTy, CostKind, Index: CheapIndex);
1438 NewCost += Ext0->hasOneUse() ? 0 : Ext0Cost;
1439 NewCost += Ext1->hasOneUse() ? 0 : Ext1Cost;
1440
1441 // Aggressively form vector ops if the cost is equal because the transform
1442 // may enable further optimization.
1443 // Codegen can reverse this transform (scalarize) if it was not profitable.
1444 if (OldCost < NewCost || !NewCost.isValid())
1445 return false;
1446
1447 // Create a vector constant from the 2 scalar constants.
1448 SmallVector<Constant *, 32> CmpC(VecTy->getNumElements(),
1449 PoisonValue::get(T: VecTy->getElementType()));
1450 CmpC[Index0] = C0;
1451 CmpC[Index1] = C1;
1452 Value *VCmp = Builder.CreateCmp(Pred, LHS: X, RHS: ConstantVector::get(V: CmpC));
1453 Value *Shuf = createShiftShuffle(Vec: VCmp, OldIndex: ExpensiveIndex, NewIndex: CheapIndex, Builder);
1454 Value *LHS = ConvertToShuf == Ext0 ? Shuf : VCmp;
1455 Value *RHS = ConvertToShuf == Ext0 ? VCmp : Shuf;
1456 Value *VecLogic = Builder.CreateBinOp(Opc: BI->getOpcode(), LHS, RHS);
1457 Value *NewExt = Builder.CreateExtractElement(Vec: VecLogic, Idx: CheapIndex);
1458 replaceValue(Old&: I, New&: *NewExt);
1459 ++NumVecCmpBO;
1460 return true;
1461}
1462
1463/// Try to fold scalar selects that select between extracted elements and zero
1464/// into extracting from a vector select. This is rooted at the bitcast.
1465///
1466/// This pattern arises when a vector is bitcast to a smaller element type,
1467/// elements are extracted, and then conditionally selected with zero:
1468///
1469/// %bc = bitcast <4 x i32> %src to <16 x i8>
1470/// %e0 = extractelement <16 x i8> %bc, i32 0
1471/// %s0 = select i1 %cond, i8 %e0, i8 0
1472/// %e1 = extractelement <16 x i8> %bc, i32 1
1473/// %s1 = select i1 %cond, i8 %e1, i8 0
1474/// ...
1475///
1476/// Transforms to:
1477/// %sel = select i1 %cond, <4 x i32> %src, <4 x i32> zeroinitializer
1478/// %bc = bitcast <4 x i32> %sel to <16 x i8>
1479/// %e0 = extractelement <16 x i8> %bc, i32 0
1480/// %e1 = extractelement <16 x i8> %bc, i32 1
1481/// ...
1482///
1483/// This is profitable because vector select on wider types produces fewer
1484/// select/cndmask instructions than scalar selects on each element.
1485bool VectorCombine::foldSelectsFromBitcast(Instruction &I) {
1486 auto *BC = dyn_cast<BitCastInst>(Val: &I);
1487 if (!BC)
1488 return false;
1489
1490 FixedVectorType *SrcVecTy = dyn_cast<FixedVectorType>(Val: BC->getSrcTy());
1491 FixedVectorType *DstVecTy = dyn_cast<FixedVectorType>(Val: BC->getDestTy());
1492 if (!SrcVecTy || !DstVecTy)
1493 return false;
1494
1495 // Source must be 32-bit or 64-bit elements, destination must be smaller
1496 // integer elements. Zero in all these types is all-bits-zero.
1497 Type *SrcEltTy = SrcVecTy->getElementType();
1498 Type *DstEltTy = DstVecTy->getElementType();
1499 unsigned SrcEltBits = SrcEltTy->getPrimitiveSizeInBits();
1500 unsigned DstEltBits = DstEltTy->getPrimitiveSizeInBits();
1501
1502 if (SrcEltBits != 32 && SrcEltBits != 64)
1503 return false;
1504
1505 if (!DstEltTy->isIntegerTy() || DstEltBits >= SrcEltBits)
1506 return false;
1507
1508 // Check profitability using TTI before collecting users.
1509 Type *CondTy = CmpInst::makeCmpResultType(opnd_type: DstEltTy);
1510 Type *VecCondTy = CmpInst::makeCmpResultType(opnd_type: SrcVecTy);
1511
1512 InstructionCost ScalarSelCost =
1513 TTI.getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: DstEltTy, CondTy,
1514 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1515 InstructionCost VecSelCost =
1516 TTI.getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: SrcVecTy, CondTy: VecCondTy,
1517 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1518
1519 // We need at least this many selects for vectorization to be profitable.
1520 // VecSelCost < ScalarSelCost * NumSelects => NumSelects > VecSelCost /
1521 // ScalarSelCost
1522 if (!ScalarSelCost.isValid() || ScalarSelCost == 0)
1523 return false;
1524
1525 unsigned MinSelects = (VecSelCost.getValue() / ScalarSelCost.getValue()) + 1;
1526
1527 // Quick check: if bitcast doesn't have enough users, bail early.
1528 if (!BC->hasNUsesOrMore(N: MinSelects))
1529 return false;
1530
1531 // Collect all select users that match the pattern, grouped by condition.
1532 // Pattern: select i1 %cond, (extractelement %bc, idx), 0
1533 DenseMap<Value *, SmallVector<SelectInst *, 8>> CondToSelects;
1534
1535 for (User *U : BC->users()) {
1536 auto *Ext = dyn_cast<ExtractElementInst>(Val: U);
1537 if (!Ext)
1538 continue;
1539
1540 for (User *ExtUser : Ext->users()) {
1541 Value *Cond;
1542 // Match: select i1 %cond, %ext, 0
1543 if (match(V: ExtUser, P: m_Select(C: m_Value(V&: Cond), L: m_Specific(V: Ext), R: m_Zero())) &&
1544 Cond->getType()->isIntegerTy(BitWidth: 1))
1545 CondToSelects[Cond].push_back(Elt: cast<SelectInst>(Val: ExtUser));
1546 }
1547 }
1548
1549 if (CondToSelects.empty())
1550 return false;
1551
1552 bool MadeChange = false;
1553 Value *SrcVec = BC->getOperand(i_nocapture: 0);
1554
1555 // Process each group of selects with the same condition.
1556 for (auto [Cond, Selects] : CondToSelects) {
1557 // Only profitable if vector select cost < total scalar select cost.
1558 if (Selects.size() < MinSelects) {
1559 LLVM_DEBUG(dbgs() << "VectorCombine: foldSelectsFromBitcast not "
1560 << "profitable (VecCost=" << VecSelCost
1561 << ", ScalarCost=" << ScalarSelCost
1562 << ", NumSelects=" << Selects.size() << ")\n");
1563 continue;
1564 }
1565
1566 // Create the vector select and bitcast once for this condition.
1567 auto InsertPt = std::next(x: BC->getIterator());
1568
1569 if (auto *CondInst = dyn_cast<Instruction>(Val: Cond))
1570 if (DT.dominates(Def: BC, User: CondInst))
1571 InsertPt = std::next(x: CondInst->getIterator());
1572
1573 Builder.SetInsertPoint(InsertPt);
1574 Value *VecSel =
1575 Builder.CreateSelect(C: Cond, True: SrcVec, False: Constant::getNullValue(Ty: SrcVecTy));
1576 Value *NewBC = Builder.CreateBitCast(V: VecSel, DestTy: DstVecTy);
1577
1578 // Replace each scalar select with an extract from the new bitcast.
1579 for (SelectInst *Sel : Selects) {
1580 auto *Ext = cast<ExtractElementInst>(Val: Sel->getTrueValue());
1581 Value *Idx = Ext->getIndexOperand();
1582
1583 Builder.SetInsertPoint(Sel);
1584 Value *NewExt = Builder.CreateExtractElement(Vec: NewBC, Idx);
1585 replaceValue(Old&: *Sel, New&: *NewExt);
1586 MadeChange = true;
1587 }
1588
1589 LLVM_DEBUG(dbgs() << "VectorCombine: folded " << Selects.size()
1590 << " selects into vector select\n");
1591 }
1592
1593 return MadeChange;
1594}
1595
1596static void analyzeCostOfVecReduction(const IntrinsicInst &II,
1597 TTI::TargetCostKind CostKind,
1598 const TargetTransformInfo &TTI,
1599 InstructionCost &CostBeforeReduction,
1600 InstructionCost &CostAfterReduction) {
1601 Instruction *Op0, *Op1;
1602 auto *RedOp = dyn_cast<Instruction>(Val: II.getOperand(i_nocapture: 0));
1603 auto *VecRedTy = cast<VectorType>(Val: II.getOperand(i_nocapture: 0)->getType());
1604 unsigned ReductionOpc =
1605 getArithmeticReductionInstruction(RdxID: II.getIntrinsicID());
1606 if (RedOp && match(V: RedOp, P: m_ZExtOrSExt(Op: m_Value()))) {
1607 bool IsUnsigned = isa<ZExtInst>(Val: RedOp);
1608 auto *ExtType = cast<VectorType>(Val: RedOp->getOperand(i: 0)->getType());
1609
1610 CostBeforeReduction =
1611 TTI.getCastInstrCost(Opcode: RedOp->getOpcode(), Dst: VecRedTy, Src: ExtType,
1612 CCH: TTI::CastContextHint::None, CostKind, I: RedOp);
1613 CostAfterReduction =
1614 TTI.getExtendedReductionCost(Opcode: ReductionOpc, IsUnsigned, ResTy: II.getType(),
1615 Ty: ExtType, FMF: FastMathFlags(), CostKind);
1616 return;
1617 }
1618 if (RedOp && II.getIntrinsicID() == Intrinsic::vector_reduce_add &&
1619 match(V: RedOp,
1620 P: m_ZExtOrSExt(Op: m_Mul(L: m_Instruction(I&: Op0), R: m_Instruction(I&: Op1)))) &&
1621 match(V: Op0, P: m_ZExtOrSExt(Op: m_Value())) &&
1622 Op0->getOpcode() == Op1->getOpcode() &&
1623 Op0->getOperand(i: 0)->getType() == Op1->getOperand(i: 0)->getType() &&
1624 (Op0->getOpcode() == RedOp->getOpcode() || Op0 == Op1)) {
1625 // Matched reduce.add(ext(mul(ext(A), ext(B)))
1626 bool IsUnsigned = isa<ZExtInst>(Val: Op0);
1627 auto *ExtType = cast<VectorType>(Val: Op0->getOperand(i: 0)->getType());
1628 VectorType *MulType = VectorType::get(ElementType: Op0->getType(), Other: VecRedTy);
1629
1630 InstructionCost ExtCost =
1631 TTI.getCastInstrCost(Opcode: Op0->getOpcode(), Dst: MulType, Src: ExtType,
1632 CCH: TTI::CastContextHint::None, CostKind, I: Op0);
1633 InstructionCost MulCost =
1634 TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: MulType, CostKind);
1635 InstructionCost Ext2Cost =
1636 TTI.getCastInstrCost(Opcode: RedOp->getOpcode(), Dst: VecRedTy, Src: MulType,
1637 CCH: TTI::CastContextHint::None, CostKind, I: RedOp);
1638
1639 CostBeforeReduction = ExtCost * 2 + MulCost + Ext2Cost;
1640 CostAfterReduction = TTI.getMulAccReductionCost(
1641 IsUnsigned, RedOpcode: ReductionOpc, ResTy: II.getType(), Ty: ExtType, CostKind);
1642 return;
1643 }
1644 CostAfterReduction = TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: VecRedTy,
1645 FMF: std::nullopt, CostKind);
1646}
1647
1648bool VectorCombine::foldBinopOfReductions(Instruction &I) {
1649 Instruction::BinaryOps BinOpOpc = cast<BinaryOperator>(Val: &I)->getOpcode();
1650 Intrinsic::ID ReductionIID = getReductionForBinop(Opc: BinOpOpc);
1651 if (BinOpOpc == Instruction::Sub)
1652 ReductionIID = Intrinsic::vector_reduce_add;
1653 if (ReductionIID == Intrinsic::not_intrinsic)
1654 return false;
1655 // FP reductions have a start-value operand that this fold doesn't handle.
1656 if (ReductionIID == Intrinsic::vector_reduce_fadd ||
1657 ReductionIID == Intrinsic::vector_reduce_fmul)
1658 return false;
1659
1660 auto checkIntrinsicAndGetItsArgument = [](Value *V,
1661 Intrinsic::ID IID) -> Value * {
1662 auto *II = dyn_cast<IntrinsicInst>(Val: V);
1663 if (!II)
1664 return nullptr;
1665 if (II->getIntrinsicID() == IID && II->hasOneUse())
1666 return II->getArgOperand(i: 0);
1667 return nullptr;
1668 };
1669
1670 Value *V0 = checkIntrinsicAndGetItsArgument(I.getOperand(i: 0), ReductionIID);
1671 if (!V0)
1672 return false;
1673 Value *V1 = checkIntrinsicAndGetItsArgument(I.getOperand(i: 1), ReductionIID);
1674 if (!V1)
1675 return false;
1676
1677 auto *VTy = cast<VectorType>(Val: V0->getType());
1678 if (V1->getType() != VTy)
1679 return false;
1680 const auto &II0 = *cast<IntrinsicInst>(Val: I.getOperand(i: 0));
1681 const auto &II1 = *cast<IntrinsicInst>(Val: I.getOperand(i: 1));
1682 unsigned ReductionOpc =
1683 getArithmeticReductionInstruction(RdxID: II0.getIntrinsicID());
1684
1685 InstructionCost OldCost = 0;
1686 InstructionCost NewCost = 0;
1687 InstructionCost CostOfRedOperand0 = 0;
1688 InstructionCost CostOfRed0 = 0;
1689 InstructionCost CostOfRedOperand1 = 0;
1690 InstructionCost CostOfRed1 = 0;
1691 analyzeCostOfVecReduction(II: II0, CostKind, TTI, CostBeforeReduction&: CostOfRedOperand0, CostAfterReduction&: CostOfRed0);
1692 analyzeCostOfVecReduction(II: II1, CostKind, TTI, CostBeforeReduction&: CostOfRedOperand1, CostAfterReduction&: CostOfRed1);
1693 OldCost = CostOfRed0 + CostOfRed1 + TTI.getInstructionCost(U: &I, CostKind);
1694 NewCost =
1695 CostOfRedOperand0 + CostOfRedOperand1 +
1696 TTI.getArithmeticInstrCost(Opcode: BinOpOpc, Ty: VTy, CostKind) +
1697 TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: VTy, FMF: std::nullopt, CostKind);
1698 if (NewCost >= OldCost || !NewCost.isValid())
1699 return false;
1700
1701 LLVM_DEBUG(dbgs() << "Found two mergeable reductions: " << I
1702 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
1703 << "\n");
1704 Value *VectorBO;
1705 if (BinOpOpc == Instruction::Or)
1706 VectorBO = Builder.CreateOr(LHS: V0, RHS: V1, Name: "",
1707 IsDisjoint: cast<PossiblyDisjointInst>(Val&: I).isDisjoint());
1708 else
1709 VectorBO = Builder.CreateBinOp(Opc: BinOpOpc, LHS: V0, RHS: V1);
1710
1711 Value *Rdx = Builder.CreateIntrinsic(ID: ReductionIID, OverloadTypes: {VTy}, Args: {VectorBO});
1712 replaceValue(Old&: I, New&: *Rdx);
1713 return true;
1714}
1715
1716// Check if memory is modified, freed, or synchronized between two instrs in
1717// the same BB.
1718static bool isMemModifiedBetween(BasicBlock::iterator Begin,
1719 BasicBlock::iterator End,
1720 const MemoryLocation &Loc, AAResults &AA) {
1721 unsigned NumScanned = 0;
1722 if (std::any_of(first: Begin, last: End, pred: [&](const Instruction &Instr) {
1723 return isModSet(MRI: AA.getModRefInfo(I: &Instr, OptLoc: Loc)) ||
1724 ++NumScanned > MaxInstrsToScan;
1725 }))
1726 return true;
1727
1728 // willNotFreeBetween expects instructions rather than iterators. An empty
1729 // range cannot free or synchronize, so avoid dereferencing its end.
1730 return Begin != End && !willNotFreeBetween(Assume: &*Begin, CtxI: &*End);
1731}
1732
1733namespace {
1734/// Helper class to indicate whether a vector index can be safely scalarized and
1735/// if a freeze needs to be inserted.
1736class ScalarizationResult {
1737 enum class StatusTy { Unsafe, Safe, SafeWithFreeze };
1738
1739 StatusTy Status;
1740 Value *ToFreeze;
1741
1742 ScalarizationResult(StatusTy Status, Value *ToFreeze = nullptr)
1743 : Status(Status), ToFreeze(ToFreeze) {}
1744
1745public:
1746 ScalarizationResult(const ScalarizationResult &Other) = default;
1747 ~ScalarizationResult() {
1748 assert(!ToFreeze && "freeze() not called with ToFreeze being set");
1749 }
1750
1751 static ScalarizationResult unsafe() { return {StatusTy::Unsafe}; }
1752 static ScalarizationResult safe() { return {StatusTy::Safe}; }
1753 static ScalarizationResult safeWithFreeze(Value *ToFreeze) {
1754 return {StatusTy::SafeWithFreeze, ToFreeze};
1755 }
1756
1757 /// Returns true if the index can be scalarize without requiring a freeze.
1758 bool isSafe() const { return Status == StatusTy::Safe; }
1759 /// Returns true if the index cannot be scalarized.
1760 bool isUnsafe() const { return Status == StatusTy::Unsafe; }
1761 /// Returns true if the index can be scalarize, but requires inserting a
1762 /// freeze.
1763 bool isSafeWithFreeze() const { return Status == StatusTy::SafeWithFreeze; }
1764
1765 /// Reset the state of Unsafe and clear ToFreze if set.
1766 void discard() {
1767 ToFreeze = nullptr;
1768 Status = StatusTy::Unsafe;
1769 }
1770
1771 /// Freeze the ToFreeze and update the use in \p User to use it.
1772 void freeze(IRBuilderBase &Builder, Instruction &UserI) {
1773 assert(isSafeWithFreeze() &&
1774 "should only be used when freezing is required");
1775 assert(is_contained(ToFreeze->users(), &UserI) &&
1776 "UserI must be a user of ToFreeze");
1777 IRBuilder<>::InsertPointGuard Guard(Builder);
1778 Builder.SetInsertPoint(cast<Instruction>(Val: &UserI));
1779 Value *Frozen =
1780 Builder.CreateFreeze(V: ToFreeze, Name: ToFreeze->getName() + ".frozen");
1781 for (Use &U : make_early_inc_range(Range: (UserI.operands())))
1782 if (U.get() == ToFreeze)
1783 U.set(Frozen);
1784
1785 ToFreeze = nullptr;
1786 }
1787};
1788} // namespace
1789
1790/// Check if it is legal to scalarize a memory access to \p VecTy at index \p
1791/// Idx. \p Idx must access a valid vector element.
1792static ScalarizationResult canScalarizeAccess(VectorType *VecTy, Value *Idx,
1793 const SimplifyQuery &SQ) {
1794 // We do checks for both fixed vector types and scalable vector types.
1795 // This is the number of elements of fixed vector types,
1796 // or the minimum number of elements of scalable vector types.
1797 uint64_t NumElements = VecTy->getElementCount().getKnownMinValue();
1798 unsigned IntWidth = Idx->getType()->getScalarSizeInBits();
1799
1800 if (auto *C = dyn_cast<ConstantInt>(Val: Idx)) {
1801 if (C->getValue().ult(RHS: NumElements))
1802 return ScalarizationResult::safe();
1803 return ScalarizationResult::unsafe();
1804 }
1805
1806 // Always unsafe if the index type can't handle all inbound values.
1807 if (!llvm::isUIntN(N: IntWidth, x: NumElements))
1808 return ScalarizationResult::unsafe();
1809
1810 APInt Zero(IntWidth, 0);
1811 APInt MaxElts(IntWidth, NumElements);
1812 ConstantRange ValidIndices(Zero, MaxElts);
1813 ConstantRange IdxRange(IntWidth, true);
1814
1815 if (isGuaranteedNotToBePoison(V: Idx, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT)) {
1816 if (ValidIndices.contains(
1817 CR: computeConstantRange(V: Idx, /*ForSigned=*/false, SQ)))
1818 return ScalarizationResult::safe();
1819 return ScalarizationResult::unsafe();
1820 }
1821
1822 // If the index may be poison, check if we can insert a freeze before the
1823 // range of the index is restricted.
1824 Value *IdxBase;
1825 ConstantInt *CI;
1826 if (match(V: Idx, P: m_And(L: m_Value(V&: IdxBase), R: m_ConstantInt(CI)))) {
1827 IdxRange = IdxRange.binaryAnd(Other: CI->getValue());
1828 } else if (match(V: Idx, P: m_URem(L: m_Value(V&: IdxBase), R: m_ConstantInt(CI)))) {
1829 IdxRange = IdxRange.urem(Other: CI->getValue());
1830 }
1831
1832 if (ValidIndices.contains(CR: IdxRange))
1833 return ScalarizationResult::safeWithFreeze(ToFreeze: IdxBase);
1834 return ScalarizationResult::unsafe();
1835}
1836
1837/// Return the GEP index type if the unsigned vector index \p Idx can be
1838/// represented by an inbounds GEP. A null result means that the maximum byte
1839/// offset cannot be represented by the pointer's signed GEP index type.
1840///
1841/// unsigned lane range
1842/// |
1843/// v
1844/// MaxByteOffset = MaxLane * element store size
1845/// |
1846/// +-- unavailable or outside signed GEP range --> reject
1847/// |
1848/// v
1849/// valid range --> use the pointer's GEP index type
1850static IntegerType *getScalarizedGEPIndexInfo(VectorType *VecTy, Value *Idx,
1851 Type *PtrTy,
1852 const DataLayout &DL) {
1853 auto *GEPIndexTy = cast<IntegerType>(Val: DL.getIndexType(PtrTy));
1854 unsigned GEPBits = GEPIndexTy->getBitWidth();
1855 uint64_t NumElements = VecTy->getElementCount().getKnownMinValue();
1856
1857 uint64_t MaxLane = NumElements - 1;
1858 if (auto *C = dyn_cast<ConstantInt>(Val: Idx)) {
1859 if (C->getValue().uge(RHS: NumElements))
1860 return nullptr;
1861 MaxLane = C->getZExtValue();
1862 }
1863
1864 Type *ElemTy = VecTy->getElementType();
1865 if (!DL.typeSizeEqualsStoreSize(Ty: ElemTy))
1866 return nullptr;
1867
1868 TypeSize ElemStride = DL.getTypeStoreSize(Ty: ElemTy);
1869 if (ElemStride.isScalable())
1870 return nullptr;
1871
1872 // Compare both values in a common width:
1873 //
1874 // MaxLane (uint64_t) * ElemStride (uint64_t) signed_max(GEPBits)
1875 // | |
1876 // v v
1877 // ByteOffset (up to 128 bits) sext to WideBits
1878 // \ /
1879 // +------------ ugt ------------+
1880 // |
1881 // greater -> reject
1882 //
1883 // WideBits = max(GEPBits, 128) prevents the multiplication from wrapping
1884 // and preserves the GEP limit during the comparison.
1885 unsigned WideBits = std::max(a: GEPBits, b: 128u);
1886 APInt MaxLaneValue(WideBits, MaxLane);
1887 APInt ByteOffset = MaxLaneValue;
1888 ByteOffset *= APInt(WideBits, ElemStride.getFixedValue());
1889 APInt MaxGEPOffset = APInt::getSignedMaxValue(numBits: GEPBits).sext(width: WideBits);
1890 // Reject offsets outside the GEP's positive signed range. Compare as
1891 // unsigned because the full 128-bit product may set its sign bit.
1892 if (ByteOffset.ugt(RHS: MaxGEPOffset))
1893 return nullptr;
1894
1895 return GEPIndexTy;
1896}
1897
1898/// Materialize an index for a scalarized GEP after profitability is known.
1899/// Vector element indices are unsigned, but GEP sign-extends narrow integer
1900/// indices. Widen a narrow index explicitly so its unsigned value is retained.
1901static Value *materializeScalarizedGEPIndex(Value *Idx, IntegerType *GEPIndexTy,
1902 IRBuilderBase &Builder) {
1903 unsigned SrcBits = Idx->getType()->getIntegerBitWidth();
1904 unsigned DstBits = GEPIndexTy->getBitWidth();
1905 if (SrcBits >= DstBits)
1906 return Idx;
1907
1908 return Builder.CreateZExt(V: Idx, DestTy: GEPIndexTy, Name: Idx->getName() + ".gepidx");
1909}
1910
1911/// The memory operation on a vector of \p ScalarType had alignment of
1912/// \p VectorAlignment. Compute the maximal, but conservatively correct,
1913/// alignment that will be valid for the memory operation on a single scalar
1914/// element of the same type with index \p Idx.
1915static Align computeAlignmentAfterScalarization(Align VectorAlignment,
1916 Type *ScalarType, Value *Idx,
1917 const DataLayout &DL) {
1918 if (auto *C = dyn_cast<ConstantInt>(Val: Idx))
1919 return commonAlignment(A: VectorAlignment,
1920 Offset: C->getZExtValue() * DL.getTypeStoreSize(Ty: ScalarType));
1921 return commonAlignment(A: VectorAlignment, Offset: DL.getTypeStoreSize(Ty: ScalarType));
1922}
1923
1924/// Fold a vector store fed by a single-use insertelement chain into scalar
1925/// stores.
1926///
1927/// Before:
1928///
1929/// %p --> vector load --> insert %x, lane 1 --> insert %y, lane 3
1930/// |
1931/// v
1932/// vector store to %p
1933///
1934/// Vector lanes: [ 0 ] [ 1 ] [ 2 ] [ 3 ]
1935/// Stored value: [ old | x | old | y ] (one vector store)
1936///
1937/// After:
1938///
1939/// +--> GEP(%p, lane 1) --> store %x
1940/// %p -------------+
1941/// +--> GEP(%p, lane 3) --> store %y
1942///
1943/// Vector lanes: [ 0 ] [ 1 ] [ 2 ] [ 3 ]
1944/// Scalar stores: x y
1945/// store@1 store@3
1946///
1947/// Step 1. Gate:
1948/// target supports vector-element GEP addressing
1949///
1950/// Step 2. Trace:
1951/// vector store <-- insertelement <-- ... <-- insertelement <-- load
1952///
1953/// Steps 3-5. Validate:
1954/// reject unprofitable full overwrites; require simple accesses, a
1955/// common address/block, no memory write in between, and scalarizable
1956/// indices.
1957bool VectorCombine::foldInsertElementsToStores(Instruction &I) {
1958 // Step 1: The target must support addressing a vector element with a GEP.
1959 if (!TTI.allowVectorElementIndexingUsingGEP())
1960 return false;
1961
1962 auto *SI = cast<StoreInst>(Val: &I);
1963 if (!SI->isSimple() || !isa<VectorType>(Val: SI->getValueOperand()->getType()))
1964 return false;
1965
1966 // Step 2: Collect a single-use insertelement chain, starting at the vector
1967 // store and walking back to the candidate load.
1968 Value *Source = SI->getValueOperand();
1969 SmallVector<std::pair<Value *, Value *>, 4> InsertElements;
1970 Value *Base = Source;
1971 while (auto *Insert = dyn_cast<InsertElementInst>(Val: Base)) {
1972 if (!Insert->hasOneUse())
1973 break;
1974 Value *InsertVal = Insert->getOperand(i_nocapture: 1);
1975 Value *Idx = Insert->getOperand(i_nocapture: 2);
1976 InsertElements.push_back(Elt: {InsertVal, Idx});
1977 Base = Insert->getOperand(i_nocapture: 0);
1978 }
1979
1980 if (InsertElements.empty())
1981 return false;
1982
1983 // The backwards walk collected the inserts in reverse program order. Restore
1984 // it now so later scalar stores preserve writes to duplicate/equal indices.
1985 std::reverse(first: InsertElements.begin(), last: InsertElements.end());
1986 auto *Load = dyn_cast<LoadInst>(Val: Base);
1987 if (!Load)
1988 return false;
1989 auto *VecTy = cast<VectorType>(Val: SI->getValueOperand()->getType());
1990
1991 // Step 3: Avoid replacing a complete overwrite with scalar stores when every
1992 // lane receives the same value; keeping the vector operation is preferable.
1993 if (auto *FVT = dyn_cast<FixedVectorType>(Val: VecTy)) {
1994 if (InsertElements.size() == FVT->getNumElements()) {
1995 Value *FirstVal = InsertElements.front().first;
1996 if (all_of(Range&: InsertElements,
1997 P: [FirstVal](const auto &Elt) { return Elt.first == FirstVal; }))
1998 return false;
1999 }
2000 }
2001 Value *SrcAddr = Load->getPointerOperand()->stripPointerCasts();
2002 // Step 4: Establish the load/store update is legal: both accesses are simple,
2003 // have the same base address and block, have scalar elements whose type size
2004 // equals their store size, and no intervening operation modifies the updated
2005 // memory.
2006 if (!Load->isSimple() || Load->getParent() != SI->getParent() ||
2007 !DL->typeSizeEqualsStoreSize(Ty: Load->getType()->getScalarType()) ||
2008 SrcAddr != SI->getPointerOperand()->stripPointerCasts())
2009 return false;
2010
2011 if (isMemModifiedBetween(Begin: Load->getIterator(), End: SI->getIterator(),
2012 Loc: MemoryLocation::get(SI), AA))
2013 return false;
2014
2015 // Step 5: Validate every index before changing IR. A safe-with-freeze result
2016 // is recorded by ScalarizationResult, so discard it until profitability is
2017 // known; otherwise a rejected candidate could leave a freeze behind.
2018 for (auto [InsertVal, Idx] : InsertElements) {
2019 auto ScalarizableIdx =
2020 canScalarizeAccess(VecTy, Idx, SQ: SQ.getWithInstruction(I: &I));
2021 if (ScalarizableIdx.isUnsafe())
2022 return false;
2023
2024 auto GEPIndex =
2025 getScalarizedGEPIndexInfo(VecTy, Idx, PtrTy: SI->getPointerOperandType(), DL: *DL);
2026 if (!GEPIndex) {
2027 ScalarizableIdx.discard();
2028 return false;
2029 }
2030
2031 // We are only checking legality here. Do not mutate IR before the
2032 // profitability check, but also do not leave a pending ToFreeze behind.
2033 ScalarizableIdx.discard();
2034 }
2035
2036 InstructionCost OldCost = TTI.getMemoryOpCost(
2037 Opcode: Instruction::Store, Src: SI->getValueOperand()->getType(), Alignment: SI->getAlign(),
2038 AddressSpace: SI->getPointerAddressSpace(), CostKind);
2039
2040 if (Load->hasOneUse())
2041 OldCost += TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: Load->getType(),
2042 Alignment: Load->getAlign(),
2043 AddressSpace: Load->getPointerAddressSpace(), CostKind);
2044
2045 for (auto [InsertVal, Idx] : InsertElements) {
2046 int Index = -1;
2047 if (auto *CIdx = dyn_cast<ConstantInt>(Val: Idx))
2048 Index = CIdx->getZExtValue();
2049
2050 OldCost += TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: VecTy,
2051 CostKind, Index);
2052 }
2053
2054 InstructionCost NewCost = 0;
2055 // This transform replaces insertelement operations on a single vector with
2056 // GEPs and scalar stores, so assume constant-index GEP offsets stay within
2057 // addressing-mode ranges that getGEPCost considers TCC_Free. Cost only GEPs
2058 // with dynamic indices.
2059 for (auto [InsertVal, Idx] : InsertElements) {
2060 if (isa<ConstantInt>(Val: Idx))
2061 continue;
2062 const Value *GEPIndices[] = {ConstantInt::get(Ty: Idx->getType(), V: 0), Idx};
2063 NewCost += TTI.getGEPCost(PointeeType: VecTy, Ptr: SI->getPointerOperand(), Operands: GEPIndices,
2064 CostKind, AccessType: InsertVal->getType());
2065 }
2066
2067 for (auto [InsertVal, Idx] : InsertElements) {
2068 Align ScalarOpAlignment = computeAlignmentAfterScalarization(
2069 VectorAlignment: std::max(a: SI->getAlign(), b: Load->getAlign()), ScalarType: InsertVal->getType(), Idx,
2070 DL: *DL);
2071
2072 NewCost += TTI.getMemoryOpCost(Opcode: Instruction::Store, Src: InsertVal->getType(),
2073 Alignment: ScalarOpAlignment,
2074 AddressSpace: SI->getPointerAddressSpace(), CostKind);
2075 }
2076
2077 LLVM_DEBUG(dbgs() << "Found an insert-elements vector store scalarization "
2078 "candidate: "
2079 << I << "\n"
2080 << " NumInserts: " << InsertElements.size() << "\n"
2081 << " OldCost: " << OldCost << " vs NewCost: " << NewCost
2082 << "\n");
2083
2084 if (OldCost <= NewCost)
2085 return false;
2086
2087 for (auto [InsertVal, Idx] : InsertElements) {
2088 auto ScalarizableIdx =
2089 canScalarizeAccess(VecTy, Idx, SQ: SQ.getWithInstruction(I: &I));
2090 assert(!ScalarizableIdx.isUnsafe() && "already checked above");
2091
2092 if (ScalarizableIdx.isSafeWithFreeze())
2093 ScalarizableIdx.freeze(Builder, UserI&: *cast<Instruction>(Val: Idx));
2094 }
2095
2096 Worklist.push(I: Load);
2097 StoreInst *LastStore = nullptr;
2098 for (auto [InsertVal, Idx] : InsertElements) {
2099 auto ScalarizableIdx =
2100 canScalarizeAccess(VecTy, Idx, SQ: SQ.getWithInstruction(I: &I));
2101 if (ScalarizableIdx.isUnsafe())
2102 return false;
2103
2104 IntegerType *GEPIndexTy =
2105 getScalarizedGEPIndexInfo(VecTy, Idx, PtrTy: SI->getPointerOperandType(), DL: *DL);
2106
2107 Value *GEPIdx = materializeScalarizedGEPIndex(Idx, GEPIndexTy, Builder);
2108 Value *GEP = Builder.CreateInBoundsGEP(
2109 Ty: SI->getValueOperand()->getType(), Ptr: SI->getPointerOperand(),
2110 IdxList: {ConstantInt::get(Ty: GEPIdx->getType(), V: 0), GEPIdx});
2111
2112 LastStore = Builder.CreateStore(Val: InsertVal, Ptr: GEP);
2113 LastStore->copyMetadata(SrcInst: *SI);
2114
2115 // The new GEP may change the pointer operand, so !invariant.group cannot
2116 // be transferred to the scalar store.
2117 LastStore->setMetadata(KindID: LLVMContext::MD_invariant_group, Node: nullptr);
2118 Align ScalarOpAlignment = computeAlignmentAfterScalarization(
2119 VectorAlignment: std::max(a: SI->getAlign(), b: Load->getAlign()), ScalarType: InsertVal->getType(), Idx,
2120 DL: *DL);
2121 LastStore->setAlignment(ScalarOpAlignment);
2122 }
2123
2124 replaceValue(Old&: I, New&: *LastStore);
2125 eraseInstruction(I);
2126 return true;
2127}
2128
2129/// Try to scalarize vector loads feeding extractelement or bitcast
2130/// instructions.
2131bool VectorCombine::scalarizeLoad(Instruction &I) {
2132 Value *Ptr;
2133 if (!match(V: &I, P: m_Load(Op: m_Value(V&: Ptr))))
2134 return false;
2135
2136 auto *LI = cast<LoadInst>(Val: &I);
2137 auto *VecTy = cast<VectorType>(Val: LI->getType());
2138
2139 // The isSimple() check could be isUnordered(), but for now we cowardly
2140 // refuse to handle even unordered atomics.
2141 if (!LI->isSimple() || !DL->typeSizeEqualsStoreSize(Ty: VecTy->getScalarType()))
2142 return false;
2143
2144 bool AllExtracts = true;
2145 bool AllBitcasts = true;
2146 Instruction *LastCheckedInst = LI;
2147 unsigned NumInstChecked = 0;
2148
2149 // Check what type of users we have (must either all be extracts or
2150 // bitcasts) and ensure no memory modifications between the load and
2151 // its users.
2152 for (User *U : LI->users()) {
2153 auto *UI = dyn_cast<Instruction>(Val: U);
2154 if (!UI || UI->getParent() != LI->getParent())
2155 return false;
2156
2157 // If any user is waiting to be erased, then bail out as this will
2158 // distort the cost calculation and possibly lead to infinite loops.
2159 if (UI->use_empty())
2160 return false;
2161
2162 if (!isa<ExtractElementInst>(Val: UI))
2163 AllExtracts = false;
2164 if (!isa<BitCastInst>(Val: UI))
2165 AllBitcasts = false;
2166
2167 // Check if any instruction between the load and the user may modify memory.
2168 if (LastCheckedInst->comesBefore(Other: UI)) {
2169 for (Instruction &I :
2170 make_range(x: std::next(x: LI->getIterator()), y: UI->getIterator())) {
2171 // Bail out if we reached the check limit or the instruction may write
2172 // to memory.
2173 if (NumInstChecked == MaxInstrsToScan || I.mayWriteToMemory())
2174 return false;
2175 NumInstChecked++;
2176 }
2177 LastCheckedInst = UI;
2178 }
2179 }
2180
2181 if (AllExtracts)
2182 return scalarizeLoadExtract(LI, VecTy, Ptr);
2183 if (AllBitcasts)
2184 return scalarizeLoadBitcast(LI, VecTy, Ptr);
2185 return false;
2186}
2187
2188/// Try to scalarize vector loads feeding extractelement instructions.
2189bool VectorCombine::scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy,
2190 Value *Ptr) {
2191 if (!TTI.allowVectorElementIndexingUsingGEP())
2192 return false;
2193
2194 DenseMap<ExtractElementInst *, ScalarizationResult> NeedFreeze;
2195 DenseMap<ExtractElementInst *, IntegerType *> GEPIndexInfos;
2196 llvm::scope_exit FailureGuard([&]() {
2197 // If the transform is aborted, discard the ScalarizationResults.
2198 for (auto &Pair : NeedFreeze)
2199 Pair.second.discard();
2200 });
2201
2202 InstructionCost OriginalCost =
2203 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: VecTy, Alignment: LI->getAlign(),
2204 AddressSpace: LI->getPointerAddressSpace(), CostKind);
2205 InstructionCost ScalarizedCost = 0;
2206
2207 for (User *U : LI->users()) {
2208 auto *UI = cast<ExtractElementInst>(Val: U);
2209
2210 auto ScalarIdx = canScalarizeAccess(VecTy, Idx: UI->getIndexOperand(),
2211 SQ: SQ.getWithInstruction(I: LI));
2212 if (ScalarIdx.isUnsafe())
2213 return false;
2214
2215 IntegerType *GEPIndex = getScalarizedGEPIndexInfo(
2216 VecTy, Idx: UI->getIndexOperand(), PtrTy: LI->getPointerOperandType(), DL: *DL);
2217 if (!GEPIndex) {
2218 ScalarIdx.discard();
2219 return false;
2220 }
2221
2222 GEPIndexInfos.try_emplace(Key: UI, Args&: GEPIndex);
2223
2224 if (ScalarIdx.isSafeWithFreeze()) {
2225 NeedFreeze.try_emplace(Key: UI, Args&: ScalarIdx);
2226 ScalarIdx.discard();
2227 }
2228
2229 auto *Index = dyn_cast<ConstantInt>(Val: UI->getIndexOperand());
2230 OriginalCost +=
2231 TTI.getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: VecTy, CostKind,
2232 Index: Index ? Index->getZExtValue() : -1);
2233 ScalarizedCost +=
2234 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: VecTy->getElementType(),
2235 Alignment: Align(1), AddressSpace: LI->getPointerAddressSpace(), CostKind);
2236 ScalarizedCost += TTI.getAddressComputationCost(PtrTy: LI->getPointerOperandType(),
2237 SE: nullptr, Ptr: nullptr, CostKind);
2238 if (!Index && UI->getIndexOperand()->getType()->getIntegerBitWidth() <
2239 GEPIndex->getBitWidth())
2240 ScalarizedCost += TTI.getCastInstrCost(
2241 Opcode: Instruction::ZExt, Dst: GEPIndex, Src: UI->getIndexOperand()->getType(),
2242 CCH: TTI::CastContextHint::None, CostKind);
2243 }
2244
2245 LLVM_DEBUG(dbgs() << "Found all extractions of a vector load: " << *LI
2246 << "\n LoadExtractCost: " << OriginalCost
2247 << " vs ScalarizedCost: " << ScalarizedCost << "\n");
2248
2249 if (ScalarizedCost > OriginalCost)
2250 return false;
2251 if (ScalarizedCost == OriginalCost && !LI->hasOneUse())
2252 return false;
2253
2254 // Ensure we add the load back to the worklist BEFORE its users so they can
2255 // erased in the correct order.
2256 Worklist.push(I: LI);
2257
2258 Type *ElemType = VecTy->getElementType();
2259
2260 // Replace extracts with narrow scalar loads.
2261 for (User *U : LI->users()) {
2262 auto *EI = cast<ExtractElementInst>(Val: U);
2263 Value *Idx = EI->getIndexOperand();
2264
2265 // Insert 'freeze' for poison indexes.
2266 if (auto It = NeedFreeze.find(Val: EI); It != NeedFreeze.end())
2267 It->second.freeze(Builder, UserI&: *cast<Instruction>(Val: Idx));
2268
2269 Builder.SetInsertPoint(EI);
2270 auto It = GEPIndexInfos.find(Val: EI);
2271 assert(It != GEPIndexInfos.end() &&
2272 "Missing scalarized GEP index information");
2273 Value *GEPIdx = materializeScalarizedGEPIndex(Idx, GEPIndexTy: It->second, Builder);
2274 Value *GEP = Builder.CreateInBoundsGEP(
2275 Ty: VecTy, Ptr, IdxList: {ConstantInt::get(Ty: GEPIdx->getType(), V: 0), GEPIdx});
2276 auto *NewLoad = cast<LoadInst>(
2277 Val: Builder.CreateLoad(Ty: ElemType, Ptr: GEP, Name: EI->getName() + ".scalar"));
2278
2279 Align ScalarOpAlignment =
2280 computeAlignmentAfterScalarization(VectorAlignment: LI->getAlign(), ScalarType: ElemType, Idx, DL: *DL);
2281 NewLoad->setAlignment(ScalarOpAlignment);
2282
2283 if (auto *ConstIdx = dyn_cast<ConstantInt>(Val: Idx)) {
2284 size_t Offset = ConstIdx->getZExtValue() * DL->getTypeStoreSize(Ty: ElemType);
2285 AAMDNodes OldAAMD = LI->getAAMetadata();
2286 NewLoad->setAAMetadata(OldAAMD.adjustForAccess(Offset, AccessTy: ElemType, DL: *DL));
2287 }
2288
2289 replaceValue(Old&: *EI, New&: *NewLoad, Erase: false);
2290 }
2291
2292 FailureGuard.release();
2293 return true;
2294}
2295
2296/// Try to scalarize vector loads feeding bitcast instructions.
2297bool VectorCombine::scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy,
2298 Value *Ptr) {
2299 InstructionCost OriginalCost =
2300 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: VecTy, Alignment: LI->getAlign(),
2301 AddressSpace: LI->getPointerAddressSpace(), CostKind);
2302
2303 if (!isa<FixedVectorType>(Val: VecTy))
2304 return false;
2305
2306 Type *TargetScalarType = nullptr;
2307 unsigned VecBitWidth = DL->getTypeSizeInBits(Ty: VecTy);
2308
2309 for (User *U : LI->users()) {
2310 auto *BC = cast<BitCastInst>(Val: U);
2311
2312 Type *DestTy = BC->getDestTy();
2313 if (!DestTy->isIntegerTy() && !DestTy->isFloatingPointTy())
2314 return false;
2315
2316 unsigned DestBitWidth = DL->getTypeSizeInBits(Ty: DestTy);
2317 if (DestBitWidth != VecBitWidth)
2318 return false;
2319
2320 // All bitcasts must target the same scalar type.
2321 if (!TargetScalarType)
2322 TargetScalarType = DestTy;
2323 else if (TargetScalarType != DestTy)
2324 return false;
2325
2326 OriginalCost +=
2327 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: TargetScalarType, Src: VecTy,
2328 CCH: TTI.getCastContextHint(I: BC), CostKind, I: BC);
2329 }
2330
2331 if (!TargetScalarType)
2332 return false;
2333
2334 assert(!LI->user_empty() && "Unexpected load without bitcast users");
2335 InstructionCost ScalarizedCost =
2336 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: TargetScalarType, Alignment: LI->getAlign(),
2337 AddressSpace: LI->getPointerAddressSpace(), CostKind);
2338
2339 LLVM_DEBUG(dbgs() << "Found vector load feeding only bitcasts: " << *LI
2340 << "\n OriginalCost: " << OriginalCost
2341 << " vs ScalarizedCost: " << ScalarizedCost << "\n");
2342
2343 if (ScalarizedCost >= OriginalCost)
2344 return false;
2345
2346 // Ensure we add the load back to the worklist BEFORE its users so they can
2347 // erased in the correct order.
2348 Worklist.push(I: LI);
2349
2350 Builder.SetInsertPoint(LI);
2351 auto *ScalarLoad =
2352 Builder.CreateLoad(Ty: TargetScalarType, Ptr, Name: LI->getName() + ".scalar");
2353 ScalarLoad->setAlignment(LI->getAlign());
2354 ScalarLoad->copyMetadata(SrcInst: *LI);
2355
2356 // Replace all bitcast users with the scalar load.
2357 for (User *U : LI->users()) {
2358 auto *BC = cast<BitCastInst>(Val: U);
2359 replaceValue(Old&: *BC, New&: *ScalarLoad, Erase: false);
2360 }
2361
2362 return true;
2363}
2364
2365bool VectorCombine::scalarizeExtExtract(Instruction &I) {
2366 if (!TTI.allowVectorElementIndexingUsingGEP())
2367 return false;
2368 auto *Ext = dyn_cast<ZExtInst>(Val: &I);
2369 if (!Ext)
2370 return false;
2371
2372 // Try to convert a vector zext feeding only extracts to a set of scalar
2373 // (Src << ExtIdx *Size) & (Size -1)
2374 // if profitable .
2375 auto *SrcTy = dyn_cast<FixedVectorType>(Val: Ext->getOperand(i_nocapture: 0)->getType());
2376 if (!SrcTy)
2377 return false;
2378 auto *DstTy = cast<FixedVectorType>(Val: Ext->getType());
2379
2380 Type *ScalarDstTy = DstTy->getElementType();
2381 if (DL->getTypeSizeInBits(Ty: SrcTy) != DL->getTypeSizeInBits(Ty: ScalarDstTy))
2382 return false;
2383
2384 InstructionCost VectorCost =
2385 TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: DstTy, Src: SrcTy,
2386 CCH: TTI::CastContextHint::None, CostKind, I: Ext);
2387 unsigned ExtCnt = 0;
2388 bool ExtLane0 = false;
2389 for (User *U : Ext->users()) {
2390 uint64_t Idx;
2391 if (!match(V: U, P: m_ExtractElt(Val: m_Value(), Idx: m_ConstantInt(V&: Idx))))
2392 return false;
2393 // An out-of-bounds extractelement produces poison; bail out rather
2394 // than computing a shift amount that overflows the packed type.
2395 if (Idx >= SrcTy->getNumElements())
2396 return false;
2397 if (cast<Instruction>(Val: U)->use_empty())
2398 continue;
2399 ExtCnt += 1;
2400 ExtLane0 |= !Idx;
2401 VectorCost += TTI.getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: DstTy,
2402 CostKind, Index: Idx, Op0: U);
2403 }
2404
2405 InstructionCost ScalarCost =
2406 ExtCnt * TTI.getArithmeticInstrCost(
2407 Opcode: Instruction::And, Ty: ScalarDstTy, CostKind,
2408 Opd1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
2409 Opd2Info: {.Kind: TTI::OK_NonUniformConstantValue, .Properties: TTI::OP_None}) +
2410 (ExtCnt - ExtLane0) *
2411 TTI.getArithmeticInstrCost(
2412 Opcode: Instruction::LShr, Ty: ScalarDstTy, CostKind,
2413 Opd1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
2414 Opd2Info: {.Kind: TTI::OK_NonUniformConstantValue, .Properties: TTI::OP_None});
2415 if (ScalarCost > VectorCost)
2416 return false;
2417
2418 Value *ScalarV = Ext->getOperand(i_nocapture: 0);
2419 if (!isGuaranteedNotToBePoison(V: ScalarV, AC: SQ.AC, CtxI: dyn_cast<Instruction>(Val: ScalarV),
2420 DT: SQ.DT)) {
2421 // Check wether all lanes are extracted, all extracts trigger UB
2422 // on poison, and the last extract (and hence all previous ones)
2423 // are guaranteed to execute if Ext executes. If so, we do not
2424 // need to insert a freeze.
2425 SmallDenseSet<ConstantInt *, 8> ExtractedLanes;
2426 bool AllExtractsTriggerUB = true;
2427 ExtractElementInst *LastExtract = nullptr;
2428 BasicBlock *ExtBB = Ext->getParent();
2429 for (User *U : Ext->users()) {
2430 auto *Extract = cast<ExtractElementInst>(Val: U);
2431 if (Extract->getParent() != ExtBB || !programUndefinedIfPoison(Inst: Extract)) {
2432 AllExtractsTriggerUB = false;
2433 break;
2434 }
2435 ExtractedLanes.insert(V: cast<ConstantInt>(Val: Extract->getIndexOperand()));
2436 if (!LastExtract || LastExtract->comesBefore(Other: Extract))
2437 LastExtract = Extract;
2438 }
2439 if (ExtractedLanes.size() != DstTy->getNumElements() ||
2440 !AllExtractsTriggerUB ||
2441 !isGuaranteedToTransferExecutionToSuccessor(Begin: Ext->getIterator(),
2442 End: LastExtract->getIterator()))
2443 ScalarV = Builder.CreateFreeze(V: ScalarV);
2444 }
2445 ScalarV = Builder.CreateBitCast(
2446 V: ScalarV,
2447 DestTy: IntegerType::get(C&: SrcTy->getContext(), NumBits: DL->getTypeSizeInBits(Ty: SrcTy)));
2448 uint64_t SrcEltSizeInBits = DL->getTypeSizeInBits(Ty: SrcTy->getElementType());
2449 uint64_t TotalBits = DL->getTypeSizeInBits(Ty: SrcTy);
2450 APInt EltBitMask = APInt::getLowBitsSet(numBits: TotalBits, loBitsSet: SrcEltSizeInBits);
2451 Type *PackedTy = IntegerType::get(C&: SrcTy->getContext(), NumBits: TotalBits);
2452 Value *Mask = ConstantInt::get(Ty: PackedTy, V: EltBitMask);
2453 for (User *U : Ext->users()) {
2454 auto *Extract = cast<ExtractElementInst>(Val: U);
2455 uint64_t Idx =
2456 cast<ConstantInt>(Val: Extract->getIndexOperand())->getZExtValue();
2457 uint64_t ShiftAmt =
2458 DL->isBigEndian()
2459 ? (TotalBits - SrcEltSizeInBits - Idx * SrcEltSizeInBits)
2460 : (Idx * SrcEltSizeInBits);
2461 Value *LShr = Builder.CreateLShr(LHS: ScalarV, RHS: ShiftAmt);
2462 Value *And = Builder.CreateAnd(LHS: LShr, RHS: Mask);
2463 U->replaceAllUsesWith(V: And);
2464 }
2465 return true;
2466}
2467
2468/// Try to fold "(or (zext (bitcast X)), (shl (zext (bitcast Y)), C))"
2469/// to "(bitcast (concat X, Y))"
2470/// where X/Y are bitcasted from i1 mask vectors.
2471bool VectorCombine::foldConcatOfBoolMasks(Instruction &I) {
2472 Type *Ty = I.getType();
2473 if (!Ty->isIntegerTy())
2474 return false;
2475
2476 // TODO: Add big endian test coverage
2477 if (DL->isBigEndian())
2478 return false;
2479
2480 // Restrict to disjoint cases so the mask vectors aren't overlapping.
2481 Instruction *X, *Y;
2482 if (!match(V: &I, P: m_DisjointOr(L: m_Instruction(I&: X), R: m_Instruction(I&: Y))))
2483 return false;
2484
2485 // Allow both sources to contain shl, to handle more generic pattern:
2486 // "(or (shl (zext (bitcast X)), C1), (shl (zext (bitcast Y)), C2))"
2487 Value *SrcX;
2488 uint64_t ShAmtX = 0;
2489 if (!match(V: X, P: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcX)))))) &&
2490 !match(V: X, P: m_OneUse(
2491 SubPattern: m_Shl(L: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcX))))),
2492 R: m_ConstantInt(V&: ShAmtX)))))
2493 return false;
2494
2495 Value *SrcY;
2496 uint64_t ShAmtY = 0;
2497 if (!match(V: Y, P: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcY)))))) &&
2498 !match(V: Y, P: m_OneUse(
2499 SubPattern: m_Shl(L: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcY))))),
2500 R: m_ConstantInt(V&: ShAmtY)))))
2501 return false;
2502
2503 // Canonicalize larger shift to the RHS.
2504 if (ShAmtX > ShAmtY) {
2505 std::swap(a&: X, b&: Y);
2506 std::swap(a&: SrcX, b&: SrcY);
2507 std::swap(a&: ShAmtX, b&: ShAmtY);
2508 }
2509
2510 // Ensure both sources are matching vXi1 bool mask types, and that the shift
2511 // difference is the mask width so they can be easily concatenated together.
2512 uint64_t ShAmtDiff = ShAmtY - ShAmtX;
2513 unsigned NumSHL = (ShAmtX > 0) + (ShAmtY > 0);
2514 unsigned BitWidth = Ty->getPrimitiveSizeInBits();
2515 auto *MaskTy = dyn_cast<FixedVectorType>(Val: SrcX->getType());
2516 if (!MaskTy || SrcX->getType() != SrcY->getType() ||
2517 !MaskTy->getElementType()->isIntegerTy(BitWidth: 1) ||
2518 MaskTy->getNumElements() != ShAmtDiff ||
2519 MaskTy->getNumElements() > (BitWidth / 2))
2520 return false;
2521
2522 auto *ConcatTy = FixedVectorType::getDoubleElementsVectorType(VTy: MaskTy);
2523 auto *ConcatIntTy =
2524 Type::getIntNTy(C&: Ty->getContext(), N: ConcatTy->getNumElements());
2525 auto *MaskIntTy = Type::getIntNTy(C&: Ty->getContext(), N: ShAmtDiff);
2526
2527 SmallVector<int, 32> ConcatMask(ConcatTy->getNumElements());
2528 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
2529
2530 // TODO: Is it worth supporting multi use cases?
2531 InstructionCost OldCost = 0;
2532 OldCost += TTI.getArithmeticInstrCost(Opcode: Instruction::Or, Ty, CostKind);
2533 OldCost +=
2534 NumSHL * TTI.getArithmeticInstrCost(Opcode: Instruction::Shl, Ty, CostKind);
2535 OldCost += 2 * TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: Ty, Src: MaskIntTy,
2536 CCH: TTI::CastContextHint::None, CostKind);
2537 OldCost += 2 * TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: MaskIntTy, Src: MaskTy,
2538 CCH: TTI::CastContextHint::None, CostKind);
2539
2540 InstructionCost NewCost = 0;
2541 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ConcatTy,
2542 SrcTy: MaskTy, CostKind, Mask: ConcatMask);
2543 NewCost += TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: ConcatIntTy, Src: ConcatTy,
2544 CCH: TTI::CastContextHint::None, CostKind);
2545 if (Ty != ConcatIntTy)
2546 NewCost += TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: Ty, Src: ConcatIntTy,
2547 CCH: TTI::CastContextHint::None, CostKind);
2548 if (ShAmtX > 0)
2549 NewCost += TTI.getArithmeticInstrCost(Opcode: Instruction::Shl, Ty, CostKind);
2550
2551 LLVM_DEBUG(dbgs() << "Found a concatenation of bitcasted bool masks: " << I
2552 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2553 << "\n");
2554
2555 if (NewCost > OldCost)
2556 return false;
2557
2558 // Build bool mask concatenation, bitcast back to scalar integer, and perform
2559 // any residual zero-extension or shifting.
2560 Value *Concat = Builder.CreateShuffleVector(V1: SrcX, V2: SrcY, Mask: ConcatMask);
2561 Worklist.pushValue(V: Concat);
2562
2563 Value *Result = Builder.CreateBitCast(V: Concat, DestTy: ConcatIntTy);
2564
2565 if (Ty != ConcatIntTy) {
2566 Worklist.pushValue(V: Result);
2567 Result = Builder.CreateZExt(V: Result, DestTy: Ty);
2568 }
2569
2570 if (ShAmtX > 0) {
2571 Worklist.pushValue(V: Result);
2572 Result = Builder.CreateShl(LHS: Result, RHS: ShAmtX);
2573 }
2574
2575 replaceValue(Old&: I, New&: *Result);
2576 return true;
2577}
2578
2579/// Try to convert "shuffle (binop (shuffle, shuffle)), undef"
2580/// --> "binop (shuffle), (shuffle)".
2581bool VectorCombine::foldPermuteOfBinops(Instruction &I) {
2582 BinaryOperator *BinOp;
2583 ArrayRef<int> OuterMask;
2584 if (!match(V: &I, P: m_Shuffle(v1: m_BinOp(I&: BinOp), v2: m_Undef(), mask: m_Mask(OuterMask))))
2585 return false;
2586
2587 // Don't introduce poison into div/rem.
2588 if (BinOp->isIntDivRem() && llvm::is_contained(Range&: OuterMask, Element: PoisonMaskElem))
2589 return false;
2590
2591 Value *Op00, *Op01, *Op10, *Op11;
2592 ArrayRef<int> Mask0, Mask1;
2593 bool Match0 = match(V: BinOp->getOperand(i_nocapture: 0),
2594 P: m_Shuffle(v1: m_Value(V&: Op00), v2: m_Value(V&: Op01), mask: m_Mask(Mask0)));
2595 bool Match1 = match(V: BinOp->getOperand(i_nocapture: 1),
2596 P: m_Shuffle(v1: m_Value(V&: Op10), v2: m_Value(V&: Op11), mask: m_Mask(Mask1)));
2597 if (!Match0 && !Match1)
2598 return false;
2599
2600 Op00 = Match0 ? Op00 : BinOp->getOperand(i_nocapture: 0);
2601 Op01 = Match0 ? Op01 : BinOp->getOperand(i_nocapture: 0);
2602 Op10 = Match1 ? Op10 : BinOp->getOperand(i_nocapture: 1);
2603 Op11 = Match1 ? Op11 : BinOp->getOperand(i_nocapture: 1);
2604
2605 Instruction::BinaryOps Opcode = BinOp->getOpcode();
2606 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
2607 auto *BinOpTy = dyn_cast<FixedVectorType>(Val: BinOp->getType());
2608 auto *Op0Ty = dyn_cast<FixedVectorType>(Val: Op00->getType());
2609 auto *Op1Ty = dyn_cast<FixedVectorType>(Val: Op10->getType());
2610 if (!ShuffleDstTy || !BinOpTy || !Op0Ty || !Op1Ty)
2611 return false;
2612
2613 unsigned NumSrcElts = BinOpTy->getNumElements();
2614
2615 // Don't accept shuffles that reference the second operand in
2616 // div/rem or if its an undef arg.
2617 if ((BinOp->isIntDivRem() || !isa<PoisonValue>(Val: I.getOperand(i: 1))) &&
2618 any_of(Range&: OuterMask, P: [NumSrcElts](int M) { return M >= (int)NumSrcElts; }))
2619 return false;
2620
2621 // Merge outer / inner (or identity if no match) shuffles.
2622 SmallVector<int> NewMask0, NewMask1;
2623 for (int M : OuterMask) {
2624 if (M < 0 || M >= (int)NumSrcElts) {
2625 NewMask0.push_back(Elt: PoisonMaskElem);
2626 NewMask1.push_back(Elt: PoisonMaskElem);
2627 } else {
2628 NewMask0.push_back(Elt: Match0 ? Mask0[M] : M);
2629 NewMask1.push_back(Elt: Match1 ? Mask1[M] : M);
2630 }
2631 }
2632
2633 unsigned NumOpElts = Op0Ty->getNumElements();
2634 bool IsIdentity0 = ShuffleDstTy == Op0Ty &&
2635 all_of(Range&: NewMask0, P: [NumOpElts](int M) { return M < (int)NumOpElts; }) &&
2636 ShuffleVectorInst::isIdentityMask(Mask: NewMask0, NumSrcElts: NumOpElts);
2637 bool IsIdentity1 = ShuffleDstTy == Op1Ty &&
2638 all_of(Range&: NewMask1, P: [NumOpElts](int M) { return M < (int)NumOpElts; }) &&
2639 ShuffleVectorInst::isIdentityMask(Mask: NewMask1, NumSrcElts: NumOpElts);
2640
2641 InstructionCost NewCost = 0;
2642 // Try to merge shuffles across the binop if the new shuffles are not costly.
2643 InstructionCost BinOpCost =
2644 TTI.getArithmeticInstrCost(Opcode, Ty: BinOpTy, CostKind);
2645 InstructionCost OldCost =
2646 BinOpCost + TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
2647 DstTy: ShuffleDstTy, SrcTy: BinOpTy, CostKind, Mask: OuterMask,
2648 Index: 0, SubTp: nullptr, Args: {BinOp}, CxtI: &I);
2649 if (!BinOp->hasOneUse())
2650 NewCost += BinOpCost;
2651
2652 if (Match0) {
2653 InstructionCost Shuf0Cost = TTI.getShuffleCost(
2654 Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: BinOpTy, SrcTy: Op0Ty, CostKind, Mask: Mask0,
2655 Index: 0, SubTp: nullptr, Args: {Op00, Op01}, CxtI: cast<Instruction>(Val: BinOp->getOperand(i_nocapture: 0)));
2656 OldCost += Shuf0Cost;
2657 if (!BinOp->hasOneUse() || !BinOp->getOperand(i_nocapture: 0)->hasOneUse())
2658 NewCost += Shuf0Cost;
2659 }
2660 if (Match1) {
2661 InstructionCost Shuf1Cost = TTI.getShuffleCost(
2662 Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: BinOpTy, SrcTy: Op1Ty, CostKind, Mask: Mask1,
2663 Index: 0, SubTp: nullptr, Args: {Op10, Op11}, CxtI: cast<Instruction>(Val: BinOp->getOperand(i_nocapture: 1)));
2664 OldCost += Shuf1Cost;
2665 if (!BinOp->hasOneUse() || !BinOp->getOperand(i_nocapture: 1)->hasOneUse())
2666 NewCost += Shuf1Cost;
2667 }
2668
2669 NewCost += TTI.getArithmeticInstrCost(Opcode, Ty: ShuffleDstTy, CostKind);
2670
2671 if (!IsIdentity0)
2672 NewCost +=
2673 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ShuffleDstTy,
2674 SrcTy: Op0Ty, CostKind, Mask: NewMask0, Index: 0, SubTp: nullptr, Args: {Op00, Op01});
2675 if (!IsIdentity1)
2676 NewCost +=
2677 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ShuffleDstTy,
2678 SrcTy: Op1Ty, CostKind, Mask: NewMask1, Index: 0, SubTp: nullptr, Args: {Op10, Op11});
2679
2680 LLVM_DEBUG(dbgs() << "Found a shuffle feeding a shuffled binop: " << I
2681 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2682 << "\n");
2683
2684 // If costs are equal, still fold as we reduce instruction count.
2685 if (NewCost > OldCost)
2686 return false;
2687
2688 Value *LHS =
2689 IsIdentity0 ? Op00 : Builder.CreateShuffleVector(V1: Op00, V2: Op01, Mask: NewMask0);
2690 Value *RHS =
2691 IsIdentity1 ? Op10 : Builder.CreateShuffleVector(V1: Op10, V2: Op11, Mask: NewMask1);
2692 Value *NewBO = Builder.CreateBinOp(Opc: Opcode, LHS, RHS);
2693
2694 // Intersect flags from the old binops.
2695 if (auto *NewInst = dyn_cast<Instruction>(Val: NewBO))
2696 NewInst->copyIRFlags(V: BinOp);
2697
2698 Worklist.pushValue(V: LHS);
2699 Worklist.pushValue(V: RHS);
2700 replaceValue(Old&: I, New&: *NewBO);
2701 return true;
2702}
2703
2704/// Try to convert "shuffle (binop), (binop)" into "binop (shuffle), (shuffle)".
2705/// Try to convert "shuffle (cmpop), (cmpop)" into "cmpop (shuffle), (shuffle)".
2706bool VectorCombine::foldShuffleOfBinops(Instruction &I) {
2707 ArrayRef<int> OldMask;
2708 Instruction *LHS, *RHS;
2709 if (!match(V: &I, P: m_Shuffle(v1: m_Instruction(I&: LHS), v2: m_Instruction(I&: RHS),
2710 mask: m_Mask(OldMask))))
2711 return false;
2712
2713 // TODO: Add support for addlike etc.
2714 if (LHS->getOpcode() != RHS->getOpcode())
2715 return false;
2716
2717 Value *X, *Y, *Z, *W;
2718 bool IsCommutative = false;
2719 CmpPredicate PredLHS = CmpInst::BAD_ICMP_PREDICATE;
2720 CmpPredicate PredRHS = CmpInst::BAD_ICMP_PREDICATE;
2721 if (match(V: LHS, P: m_BinOp(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
2722 match(V: RHS, P: m_BinOp(L: m_Value(V&: Z), R: m_Value(V&: W)))) {
2723 auto *BO = cast<BinaryOperator>(Val: LHS);
2724 // Don't introduce poison into div/rem.
2725 if (llvm::is_contained(Range&: OldMask, Element: PoisonMaskElem) && BO->isIntDivRem())
2726 return false;
2727 IsCommutative = BinaryOperator::isCommutative(Opcode: BO->getOpcode());
2728 } else if (match(V: LHS, P: m_Cmp(Pred&: PredLHS, L: m_Value(V&: X), R: m_Value(V&: Y))) &&
2729 match(V: RHS, P: m_Cmp(Pred&: PredRHS, L: m_Value(V&: Z), R: m_Value(V&: W))) &&
2730 (CmpInst::Predicate)PredLHS == (CmpInst::Predicate)PredRHS) {
2731 IsCommutative = cast<CmpInst>(Val: LHS)->isCommutative();
2732 } else
2733 return false;
2734
2735 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
2736 auto *BinResTy = dyn_cast<FixedVectorType>(Val: LHS->getType());
2737 auto *BinOpTy = dyn_cast<FixedVectorType>(Val: X->getType());
2738 if (!ShuffleDstTy || !BinResTy || !BinOpTy || X->getType() != Z->getType())
2739 return false;
2740
2741 bool SameBinOp = LHS == RHS;
2742 unsigned NumSrcElts = BinOpTy->getNumElements();
2743
2744 // If we have something like "add X, Y" and "add Z, X", swap ops to match.
2745 if (IsCommutative && X != Z && Y != W && (X == W || Y == Z))
2746 std::swap(a&: X, b&: Y);
2747
2748 auto ConvertToUnary = [NumSrcElts](int &M) {
2749 if (M >= (int)NumSrcElts)
2750 M -= NumSrcElts;
2751 };
2752
2753 SmallVector<int> NewMask0(OldMask);
2754 TargetTransformInfo::ShuffleKind SK0 = TargetTransformInfo::SK_PermuteTwoSrc;
2755 TTI::OperandValueInfo Op0Info = TTI.commonOperandInfo(X, Y: Z);
2756 if (X == Z) {
2757 llvm::for_each(Range&: NewMask0, F: ConvertToUnary);
2758 SK0 = TargetTransformInfo::SK_PermuteSingleSrc;
2759 Z = PoisonValue::get(T: BinOpTy);
2760 }
2761
2762 SmallVector<int> NewMask1(OldMask);
2763 TargetTransformInfo::ShuffleKind SK1 = TargetTransformInfo::SK_PermuteTwoSrc;
2764 TTI::OperandValueInfo Op1Info = TTI.commonOperandInfo(X: Y, Y: W);
2765 if (Y == W) {
2766 llvm::for_each(Range&: NewMask1, F: ConvertToUnary);
2767 SK1 = TargetTransformInfo::SK_PermuteSingleSrc;
2768 W = PoisonValue::get(T: BinOpTy);
2769 }
2770
2771 // Try to replace a binop with a shuffle if the shuffle is not costly.
2772 // When SameBinOp, only count the binop cost once.
2773 InstructionCost LHSCost = TTI.getInstructionCost(U: LHS, CostKind);
2774 InstructionCost RHSCost = TTI.getInstructionCost(U: RHS, CostKind);
2775
2776 InstructionCost OldCost = LHSCost;
2777 if (!SameBinOp) {
2778 OldCost += RHSCost;
2779 }
2780 OldCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc,
2781 DstTy: ShuffleDstTy, SrcTy: BinResTy, CostKind, Mask: OldMask, Index: 0,
2782 SubTp: nullptr, Args: {LHS, RHS}, CxtI: &I);
2783
2784 // Handle shuffle(binop(shuffle(x),y),binop(z,shuffle(w))) style patterns
2785 // where one use shuffles have gotten split across the binop/cmp. These
2786 // often allow a major reduction in total cost that wouldn't happen as
2787 // individual folds.
2788 auto MergeInner = [&](Value *&Op, int Offset, MutableArrayRef<int> Mask,
2789 TTI::TargetCostKind CostKind) -> bool {
2790 Value *InnerOp;
2791 ArrayRef<int> InnerMask;
2792 if (match(V: Op, P: m_OneUse(SubPattern: m_Shuffle(v1: m_Value(V&: InnerOp), v2: m_Undef(),
2793 mask: m_Mask(InnerMask)))) &&
2794 InnerOp->getType() == Op->getType() &&
2795 all_of(Range&: InnerMask,
2796 P: [NumSrcElts](int M) { return M < (int)NumSrcElts; })) {
2797 for (int &M : Mask)
2798 if (Offset <= M && M < (int)(Offset + NumSrcElts)) {
2799 M = InnerMask[M - Offset];
2800 M = 0 <= M ? M + Offset : M;
2801 }
2802 OldCost += TTI.getInstructionCost(U: cast<Instruction>(Val: Op), CostKind);
2803 Op = InnerOp;
2804 return true;
2805 }
2806 return false;
2807 };
2808 bool ReducedInstCount = false;
2809 ReducedInstCount |= MergeInner(X, 0, NewMask0, CostKind);
2810 ReducedInstCount |= MergeInner(Y, 0, NewMask1, CostKind);
2811 ReducedInstCount |= MergeInner(Z, NumSrcElts, NewMask0, CostKind);
2812 ReducedInstCount |= MergeInner(W, NumSrcElts, NewMask1, CostKind);
2813 bool SingleSrcBinOp = (X == Y) && (Z == W) && (NewMask0 == NewMask1);
2814 // SingleSrcBinOp only reduces instruction count if we also eliminate the
2815 // original binop(s). If binops have multiple uses, they won't be eliminated.
2816 ReducedInstCount |= SingleSrcBinOp && LHS->hasOneUser() && RHS->hasOneUser();
2817
2818 // For concat shuffles of i1 vectors where both binops are one-use, the
2819 // transform keeps the same instruction count but canonicalises to a single
2820 // wider binop, enabling downstream folds (e.g. NOT(XOR(concat(a,b),
2821 // concat(c,d))) -> XNOR(concat(a,b),concat(c,d)) on AVX-512 mask regs).
2822 // Restrict to BinaryOperator (not CmpInst) since narrow comparisons may
2823 // be cheaper than wide ones on some targets (e.g. AVX-512 vpcmpeq).
2824 ReducedInstCount |= cast<ShuffleVectorInst>(Val: &I)->isConcat() &&
2825 I.getType()->getScalarType()->isIntegerTy(BitWidth: 1) &&
2826 isa<BinaryOperator>(Val: LHS) && LHS->hasOneUser() &&
2827 RHS->hasOneUser();
2828
2829 auto *ShuffleCmpTy =
2830 FixedVectorType::get(ElementType: BinOpTy->getElementType(), FVTy: ShuffleDstTy);
2831 InstructionCost NewCost = TTI.getShuffleCost(
2832 Kind: SK0, DstTy: ShuffleCmpTy, SrcTy: BinOpTy, CostKind, Mask: NewMask0, Index: 0, SubTp: nullptr, Args: {X, Z});
2833 if (!SingleSrcBinOp)
2834 NewCost += TTI.getShuffleCost(Kind: SK1, DstTy: ShuffleCmpTy, SrcTy: BinOpTy, CostKind,
2835 Mask: NewMask1, Index: 0, SubTp: nullptr, Args: {Y, W});
2836
2837 if (PredLHS == CmpInst::BAD_ICMP_PREDICATE) {
2838 NewCost += TTI.getArithmeticInstrCost(Opcode: LHS->getOpcode(), Ty: ShuffleDstTy,
2839 CostKind, Opd1Info: Op0Info, Opd2Info: Op1Info);
2840 } else {
2841 NewCost +=
2842 TTI.getCmpSelInstrCost(Opcode: LHS->getOpcode(), ValTy: ShuffleCmpTy, CondTy: ShuffleDstTy,
2843 VecPred: PredLHS, CostKind, Op1Info: Op0Info, Op2Info: Op1Info);
2844 }
2845 // If LHS/RHS have other uses, we need to account for the cost of keeping
2846 // the original instructions. When SameBinOp, only add the cost once.
2847 if (!LHS->hasOneUser())
2848 NewCost += LHSCost;
2849 if (!SameBinOp && !RHS->hasOneUser())
2850 NewCost += RHSCost;
2851
2852 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two binops: " << I
2853 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2854 << "\n");
2855
2856 // If either shuffle will constant fold away, then fold for the same cost as
2857 // we will reduce the instruction count.
2858 ReducedInstCount |= (isa<Constant>(Val: X) && isa<Constant>(Val: Z)) ||
2859 (isa<Constant>(Val: Y) && isa<Constant>(Val: W));
2860 if (ReducedInstCount ? (NewCost > OldCost) : (NewCost >= OldCost))
2861 return false;
2862
2863 Value *Shuf0 = Builder.CreateShuffleVector(V1: X, V2: Z, Mask: NewMask0);
2864 Value *Shuf1 =
2865 SingleSrcBinOp ? Shuf0 : Builder.CreateShuffleVector(V1: Y, V2: W, Mask: NewMask1);
2866 Value *NewBO = PredLHS == CmpInst::BAD_ICMP_PREDICATE
2867 ? Builder.CreateBinOp(
2868 Opc: cast<BinaryOperator>(Val: LHS)->getOpcode(), LHS: Shuf0, RHS: Shuf1)
2869 : Builder.CreateCmp(Pred: PredLHS, LHS: Shuf0, RHS: Shuf1);
2870
2871 // Intersect flags from the old binops.
2872 if (auto *NewInst = dyn_cast<Instruction>(Val: NewBO)) {
2873 NewInst->copyIRFlags(V: LHS);
2874 NewInst->andIRFlags(V: RHS);
2875 }
2876
2877 Worklist.pushValue(V: Shuf0);
2878 Worklist.pushValue(V: Shuf1);
2879 replaceValue(Old&: I, New&: *NewBO);
2880 return true;
2881}
2882
2883/// Try to convert,
2884/// (shuffle(select(c1,t1,f1)), (select(c2,t2,f2)), m) into
2885/// (select (shuffle c1,c2,m), (shuffle t1,t2,m), (shuffle f1,f2,m))
2886bool VectorCombine::foldShuffleOfSelects(Instruction &I) {
2887 ArrayRef<int> Mask;
2888 Value *C1, *T1, *F1, *C2, *T2, *F2;
2889 if (!match(V: &I, P: m_Shuffle(v1: m_Select(C: m_Value(V&: C1), L: m_Value(V&: T1), R: m_Value(V&: F1)),
2890 v2: m_Select(C: m_Value(V&: C2), L: m_Value(V&: T2), R: m_Value(V&: F2)),
2891 mask: m_Mask(Mask))))
2892 return false;
2893
2894 auto *Sel1 = cast<Instruction>(Val: I.getOperand(i: 0));
2895 auto *Sel2 = cast<Instruction>(Val: I.getOperand(i: 1));
2896
2897 auto *C1VecTy = dyn_cast<FixedVectorType>(Val: C1->getType());
2898 auto *C2VecTy = dyn_cast<FixedVectorType>(Val: C2->getType());
2899 if (!C1VecTy || !C2VecTy || C1VecTy != C2VecTy)
2900 return false;
2901
2902 auto *SI0FOp = dyn_cast<FPMathOperator>(Val: I.getOperand(i: 0));
2903 auto *SI1FOp = dyn_cast<FPMathOperator>(Val: I.getOperand(i: 1));
2904 // SelectInsts must have the same FMF.
2905 if (((SI0FOp == nullptr) != (SI1FOp == nullptr)) ||
2906 ((SI0FOp != nullptr) &&
2907 (SI0FOp->getFastMathFlags() != SI1FOp->getFastMathFlags())))
2908 return false;
2909
2910 auto *SrcVecTy = cast<FixedVectorType>(Val: T1->getType());
2911 auto *DstVecTy = cast<FixedVectorType>(Val: I.getType());
2912 auto SK = TargetTransformInfo::SK_PermuteTwoSrc;
2913 auto SelOp = Instruction::Select;
2914
2915 InstructionCost CostSel1 = TTI.getCmpSelInstrCost(
2916 Opcode: SelOp, ValTy: SrcVecTy, CondTy: C1VecTy, VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
2917 InstructionCost CostSel2 = TTI.getCmpSelInstrCost(
2918 Opcode: SelOp, ValTy: SrcVecTy, CondTy: C2VecTy, VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
2919
2920 InstructionCost OldCost =
2921 CostSel1 + CostSel2 +
2922 TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask, Index: 0, SubTp: nullptr,
2923 Args: {I.getOperand(i: 0), I.getOperand(i: 1)}, CxtI: &I);
2924
2925 InstructionCost NewCost = TTI.getShuffleCost(
2926 Kind: SK, DstTy: FixedVectorType::get(ElementType: C1VecTy->getScalarType(), NumElts: Mask.size()), SrcTy: C1VecTy,
2927 CostKind, Mask, Index: 0, SubTp: nullptr, Args: {C1, C2});
2928 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask, Index: 0,
2929 SubTp: nullptr, Args: {T1, T2});
2930 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask, Index: 0,
2931 SubTp: nullptr, Args: {F1, F2});
2932 auto *C1C2ShuffledVecTy = FixedVectorType::get(
2933 ElementType: Type::getInt1Ty(C&: I.getContext()), NumElts: DstVecTy->getNumElements());
2934 NewCost += TTI.getCmpSelInstrCost(Opcode: SelOp, ValTy: DstVecTy, CondTy: C1C2ShuffledVecTy,
2935 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
2936
2937 if (!Sel1->hasOneUse())
2938 NewCost += CostSel1;
2939 if (!Sel2->hasOneUse())
2940 NewCost += CostSel2;
2941
2942 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two selects: " << I
2943 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2944 << "\n");
2945 if (NewCost > OldCost)
2946 return false;
2947
2948 Value *ShuffleCmp = Builder.CreateShuffleVector(V1: C1, V2: C2, Mask);
2949 Value *ShuffleTrue = Builder.CreateShuffleVector(V1: T1, V2: T2, Mask);
2950 Value *ShuffleFalse = Builder.CreateShuffleVector(V1: F1, V2: F2, Mask);
2951 Value *NewSel;
2952 // We presuppose that the SelectInsts have the same FMF.
2953 if (SI0FOp)
2954 NewSel = Builder.CreateSelectFMF(C: ShuffleCmp, True: ShuffleTrue, False: ShuffleFalse,
2955 FMFSource: SI0FOp->getFastMathFlags());
2956 else
2957 NewSel = Builder.CreateSelect(C: ShuffleCmp, True: ShuffleTrue, False: ShuffleFalse);
2958
2959 Worklist.pushValue(V: ShuffleCmp);
2960 Worklist.pushValue(V: ShuffleTrue);
2961 Worklist.pushValue(V: ShuffleFalse);
2962 replaceValue(Old&: I, New&: *NewSel);
2963 return true;
2964}
2965
2966/// Try to convert "shuffle (castop), (castop)" with a shared castop operand
2967/// into "castop (shuffle)".
2968bool VectorCombine::foldShuffleOfCastops(Instruction &I) {
2969 Value *V0, *V1;
2970 ArrayRef<int> OldMask;
2971 if (!match(V: &I, P: m_Shuffle(v1: m_Value(V&: V0), v2: m_Value(V&: V1), mask: m_Mask(OldMask))))
2972 return false;
2973
2974 // Check whether this is a binary shuffle.
2975 bool IsBinaryShuffle = !isa<UndefValue>(Val: V1);
2976
2977 auto *C0 = dyn_cast<CastInst>(Val: V0);
2978 auto *C1 = dyn_cast<CastInst>(Val: V1);
2979 if (!C0 || (IsBinaryShuffle && !C1))
2980 return false;
2981
2982 Instruction::CastOps Opcode = C0->getOpcode();
2983
2984 // If this is allowed, foldShuffleOfCastops can get stuck in a loop
2985 // with foldBitcastOfShuffle. Reject in favor of foldBitcastOfShuffle.
2986 if (!IsBinaryShuffle && Opcode == Instruction::BitCast)
2987 return false;
2988
2989 if (IsBinaryShuffle) {
2990 if (C0->getSrcTy() != C1->getSrcTy())
2991 return false;
2992 // Handle shuffle(zext_nneg(x), sext(y)) -> sext(shuffle(x,y)) folds.
2993 if (Opcode != C1->getOpcode()) {
2994 if (match(V: C0, P: m_SExtLike(Op: m_Value())) && match(V: C1, P: m_SExtLike(Op: m_Value())))
2995 Opcode = Instruction::SExt;
2996 else
2997 return false;
2998 }
2999 }
3000
3001 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3002 auto *CastDstTy = dyn_cast<FixedVectorType>(Val: C0->getDestTy());
3003 auto *CastSrcTy = dyn_cast<FixedVectorType>(Val: C0->getSrcTy());
3004 if (!ShuffleDstTy || !CastDstTy || !CastSrcTy)
3005 return false;
3006
3007 unsigned NumSrcElts = CastSrcTy->getNumElements();
3008 unsigned NumDstElts = CastDstTy->getNumElements();
3009 assert((NumDstElts == NumSrcElts || Opcode == Instruction::BitCast) &&
3010 "Only bitcasts expected to alter src/dst element counts");
3011
3012 // Check for bitcasting of unscalable vector types.
3013 // e.g. <32 x i40> -> <40 x i32>
3014 if (NumDstElts != NumSrcElts && (NumSrcElts % NumDstElts) != 0 &&
3015 (NumDstElts % NumSrcElts) != 0)
3016 return false;
3017
3018 SmallVector<int, 16> NewMask;
3019 if (NumSrcElts >= NumDstElts) {
3020 // The bitcast is from wide to narrow/equal elements. The shuffle mask can
3021 // always be expanded to the equivalent form choosing narrower elements.
3022 assert(NumSrcElts % NumDstElts == 0 && "Unexpected shuffle mask");
3023 unsigned ScaleFactor = NumSrcElts / NumDstElts;
3024 narrowShuffleMaskElts(Scale: ScaleFactor, Mask: OldMask, ScaledMask&: NewMask);
3025 } else {
3026 // The bitcast is from narrow elements to wide elements. The shuffle mask
3027 // must choose consecutive elements to allow casting first.
3028 assert(NumDstElts % NumSrcElts == 0 && "Unexpected shuffle mask");
3029 unsigned ScaleFactor = NumDstElts / NumSrcElts;
3030 if (!widenShuffleMaskElts(Scale: ScaleFactor, Mask: OldMask, ScaledMask&: NewMask))
3031 return false;
3032 }
3033
3034 auto *NewShuffleDstTy =
3035 FixedVectorType::get(ElementType: CastSrcTy->getScalarType(), NumElts: NewMask.size());
3036
3037 // Try to replace a castop with a shuffle if the shuffle is not costly.
3038 InstructionCost CostC0 =
3039 TTI.getCastInstrCost(Opcode: C0->getOpcode(), Dst: CastDstTy, Src: CastSrcTy,
3040 CCH: TTI::CastContextHint::None, CostKind, I: C0);
3041
3042 TargetTransformInfo::ShuffleKind ShuffleKind;
3043 if (IsBinaryShuffle)
3044 ShuffleKind = TargetTransformInfo::SK_PermuteTwoSrc;
3045 else
3046 ShuffleKind = TargetTransformInfo::SK_PermuteSingleSrc;
3047
3048 InstructionCost OldCost = CostC0;
3049 OldCost += TTI.getShuffleCost(Kind: ShuffleKind, DstTy: ShuffleDstTy, SrcTy: CastDstTy, CostKind,
3050 Mask: OldMask, Index: 0, SubTp: nullptr, Args: {}, CxtI: &I);
3051
3052 InstructionCost NewCost = TTI.getShuffleCost(Kind: ShuffleKind, DstTy: NewShuffleDstTy,
3053 SrcTy: CastSrcTy, CostKind, Mask: NewMask);
3054 NewCost += TTI.getCastInstrCost(Opcode, Dst: ShuffleDstTy, Src: NewShuffleDstTy,
3055 CCH: TTI::CastContextHint::None, CostKind);
3056 if (!C0->hasOneUse())
3057 NewCost += CostC0;
3058 if (IsBinaryShuffle) {
3059 InstructionCost CostC1 =
3060 TTI.getCastInstrCost(Opcode: C1->getOpcode(), Dst: CastDstTy, Src: CastSrcTy,
3061 CCH: TTI::CastContextHint::None, CostKind, I: C1);
3062 OldCost += CostC1;
3063 if (!C1->hasOneUse())
3064 NewCost += CostC1;
3065 }
3066
3067 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two casts: " << I
3068 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
3069 << "\n");
3070 if (NewCost > OldCost)
3071 return false;
3072
3073 Value *Shuf;
3074 if (IsBinaryShuffle)
3075 Shuf = Builder.CreateShuffleVector(V1: C0->getOperand(i_nocapture: 0), V2: C1->getOperand(i_nocapture: 0),
3076 Mask: NewMask);
3077 else
3078 Shuf = Builder.CreateShuffleVector(V: C0->getOperand(i_nocapture: 0), Mask: NewMask);
3079
3080 Value *Cast = Builder.CreateCast(Op: Opcode, V: Shuf, DestTy: ShuffleDstTy);
3081
3082 // Intersect flags from the old casts.
3083 if (auto *NewInst = dyn_cast<Instruction>(Val: Cast)) {
3084 NewInst->copyIRFlags(V: C0);
3085 if (IsBinaryShuffle)
3086 NewInst->andIRFlags(V: C1);
3087 }
3088
3089 Worklist.pushValue(V: Shuf);
3090 replaceValue(Old&: I, New&: *Cast);
3091 return true;
3092}
3093
3094/// Try to convert any of:
3095/// "shuffle (shuffle x, y), (shuffle y, x)"
3096/// "shuffle (shuffle x, undef), (shuffle y, undef)"
3097/// "shuffle (shuffle x, undef), y"
3098/// "shuffle x, (shuffle y, undef)"
3099/// into "shuffle x, y".
3100bool VectorCombine::foldShuffleOfShuffles(Instruction &I) {
3101 ArrayRef<int> OuterMask;
3102 Value *OuterV0, *OuterV1;
3103 if (!match(V: &I,
3104 P: m_Shuffle(v1: m_Value(V&: OuterV0), v2: m_Value(V&: OuterV1), mask: m_Mask(OuterMask))))
3105 return false;
3106
3107 ArrayRef<int> InnerMask0, InnerMask1;
3108 Value *X0, *X1, *Y0, *Y1;
3109 bool Match0 =
3110 match(V: OuterV0, P: m_Shuffle(v1: m_Value(V&: X0), v2: m_Value(V&: Y0), mask: m_Mask(InnerMask0)));
3111 bool Match1 =
3112 match(V: OuterV1, P: m_Shuffle(v1: m_Value(V&: X1), v2: m_Value(V&: Y1), mask: m_Mask(InnerMask1)));
3113 if (!Match0 && !Match1)
3114 return false;
3115
3116 // If the outer shuffle is a permute, then create a fake inner all-poison
3117 // shuffle. This is easier than accounting for length-changing shuffles below.
3118 SmallVector<int, 16> PoisonMask1;
3119 if (!Match1 && isa<PoisonValue>(Val: OuterV1)) {
3120 X1 = X0;
3121 Y1 = Y0;
3122 PoisonMask1.append(NumInputs: InnerMask0.size(), Elt: PoisonMaskElem);
3123 InnerMask1 = PoisonMask1;
3124 Match1 = true; // fake match
3125 }
3126
3127 X0 = Match0 ? X0 : OuterV0;
3128 Y0 = Match0 ? Y0 : OuterV0;
3129 X1 = Match1 ? X1 : OuterV1;
3130 Y1 = Match1 ? Y1 : OuterV1;
3131 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3132 auto *ShuffleSrcTy = dyn_cast<FixedVectorType>(Val: X0->getType());
3133 auto *ShuffleImmTy = dyn_cast<FixedVectorType>(Val: OuterV0->getType());
3134 if (!ShuffleDstTy || !ShuffleSrcTy || !ShuffleImmTy ||
3135 X0->getType() != X1->getType())
3136 return false;
3137
3138 unsigned NumSrcElts = ShuffleSrcTy->getNumElements();
3139 unsigned NumImmElts = ShuffleImmTy->getNumElements();
3140
3141 // Attempt to merge shuffles, matching upto 2 source operands.
3142 // Replace index to a poison arg with PoisonMaskElem.
3143 // Bail if either inner masks reference an undef arg.
3144 SmallVector<int, 16> NewMask(OuterMask);
3145 Value *NewX = nullptr, *NewY = nullptr;
3146 for (int &M : NewMask) {
3147 Value *Src = nullptr;
3148 if (0 <= M && M < (int)NumImmElts) {
3149 Src = OuterV0;
3150 if (Match0) {
3151 M = InnerMask0[M];
3152 Src = M >= (int)NumSrcElts ? Y0 : X0;
3153 M = M >= (int)NumSrcElts ? (M - NumSrcElts) : M;
3154 }
3155 } else if (M >= (int)NumImmElts) {
3156 Src = OuterV1;
3157 M -= NumImmElts;
3158 if (Match1) {
3159 M = InnerMask1[M];
3160 Src = M >= (int)NumSrcElts ? Y1 : X1;
3161 M = M >= (int)NumSrcElts ? (M - NumSrcElts) : M;
3162 }
3163 }
3164 if (Src && M != PoisonMaskElem) {
3165 assert(0 <= M && M < (int)NumSrcElts && "Unexpected shuffle mask index");
3166 if (isa<UndefValue>(Val: Src)) {
3167 // We've referenced an undef element - if its poison, update the shuffle
3168 // mask, else bail.
3169 if (!isa<PoisonValue>(Val: Src))
3170 return false;
3171 M = PoisonMaskElem;
3172 continue;
3173 }
3174 if (!NewX || NewX == Src) {
3175 NewX = Src;
3176 continue;
3177 }
3178 if (!NewY || NewY == Src) {
3179 M += NumSrcElts;
3180 NewY = Src;
3181 continue;
3182 }
3183 return false;
3184 }
3185 }
3186
3187 if (!NewX) {
3188 replaceValue(Old&: I, New&: *PoisonValue::get(T: ShuffleDstTy));
3189 return true;
3190 }
3191
3192 if (!NewY)
3193 NewY = PoisonValue::get(T: ShuffleSrcTy);
3194
3195 // Have we folded to an Identity shuffle?
3196 if (ShuffleVectorInst::isIdentityMask(Mask: NewMask, NumSrcElts)) {
3197 replaceValue(Old&: I, New&: *NewX);
3198 return true;
3199 }
3200
3201 // Try to merge the shuffles if the new shuffle is not costly.
3202 InstructionCost InnerCost0 = 0;
3203 if (Match0)
3204 InnerCost0 = TTI.getInstructionCost(U: cast<User>(Val: OuterV0), CostKind);
3205
3206 InstructionCost InnerCost1 = 0;
3207 if (Match1)
3208 InnerCost1 = TTI.getInstructionCost(U: cast<User>(Val: OuterV1), CostKind);
3209
3210 InstructionCost OuterCost = TTI.getInstructionCost(U: &I, CostKind);
3211
3212 InstructionCost OldCost = InnerCost0 + InnerCost1 + OuterCost;
3213
3214 bool IsUnary = all_of(Range&: NewMask, P: [&](int M) { return M < (int)NumSrcElts; });
3215 TargetTransformInfo::ShuffleKind SK =
3216 IsUnary ? TargetTransformInfo::SK_PermuteSingleSrc
3217 : TargetTransformInfo::SK_PermuteTwoSrc;
3218 InstructionCost NewCost =
3219 TTI.getShuffleCost(Kind: SK, DstTy: ShuffleDstTy, SrcTy: ShuffleSrcTy, CostKind, Mask: NewMask, Index: 0,
3220 SubTp: nullptr, Args: {NewX, NewY});
3221 if (!OuterV0->hasOneUse())
3222 NewCost += InnerCost0;
3223 if (!OuterV1->hasOneUse())
3224 NewCost += InnerCost1;
3225
3226 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two shuffles: " << I
3227 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
3228 << "\n");
3229 if (NewCost > OldCost)
3230 return false;
3231
3232 Value *Shuf = Builder.CreateShuffleVector(V1: NewX, V2: NewY, Mask: NewMask);
3233 replaceValue(Old&: I, New&: *Shuf);
3234 return true;
3235}
3236
3237/// Try to convert a chain of length-preserving shuffles that are fed by
3238/// length-changing shuffles from the same source, e.g. a chain of length 3:
3239///
3240/// "shuffle (shuffle (shuffle x, (shuffle y, undef)),
3241/// (shuffle y, undef)),
3242// (shuffle y, undef)"
3243///
3244/// into a single shuffle fed by a length-changing shuffle:
3245///
3246/// "shuffle x, (shuffle y, undef)"
3247///
3248/// Such chains arise e.g. from folding extract/insert sequences.
3249bool VectorCombine::foldShufflesOfLengthChangingShuffles(Instruction &I) {
3250 FixedVectorType *TrunkType = dyn_cast<FixedVectorType>(Val: I.getType());
3251 if (!TrunkType)
3252 return false;
3253
3254 unsigned ChainLength = 0;
3255 SmallVector<int> Mask;
3256 SmallVector<int> YMask;
3257 InstructionCost OldCost = 0;
3258 InstructionCost NewCost = 0;
3259 Value *Trunk = &I;
3260 unsigned NumTrunkElts = TrunkType->getNumElements();
3261 Value *Y = nullptr;
3262
3263 for (;;) {
3264 // Match the current trunk against (commutations of) the pattern
3265 // "shuffle trunk', (shuffle y, undef)"
3266 ArrayRef<int> OuterMask;
3267 Value *OuterV0, *OuterV1;
3268 if (ChainLength != 0 && !Trunk->hasOneUse())
3269 break;
3270 if (!match(V: Trunk, P: m_Shuffle(v1: m_Value(V&: OuterV0), v2: m_Value(V&: OuterV1),
3271 mask: m_Mask(OuterMask))))
3272 break;
3273 if (OuterV0->getType() != TrunkType) {
3274 // This shuffle is not length-preserving, so it cannot be part of the
3275 // chain.
3276 break;
3277 }
3278
3279 ArrayRef<int> InnerMask0, InnerMask1;
3280 Value *A0, *A1, *B0, *B1;
3281 bool Match0 =
3282 match(V: OuterV0, P: m_Shuffle(v1: m_Value(V&: A0), v2: m_Value(V&: B0), mask: m_Mask(InnerMask0)));
3283 bool Match1 =
3284 match(V: OuterV1, P: m_Shuffle(v1: m_Value(V&: A1), v2: m_Value(V&: B1), mask: m_Mask(InnerMask1)));
3285 bool Match0Leaf = Match0 && A0->getType() != I.getType();
3286 bool Match1Leaf = Match1 && A1->getType() != I.getType();
3287 if (Match0Leaf == Match1Leaf) {
3288 // Only handle the case of exactly one leaf in each step. The "two leaves"
3289 // case is handled by foldShuffleOfShuffles.
3290 break;
3291 }
3292
3293 SmallVector<int> CommutedOuterMask;
3294 if (Match0Leaf) {
3295 std::swap(a&: OuterV0, b&: OuterV1);
3296 std::swap(a&: InnerMask0, b&: InnerMask1);
3297 std::swap(a&: A0, b&: A1);
3298 std::swap(a&: B0, b&: B1);
3299 llvm::append_range(C&: CommutedOuterMask, R&: OuterMask);
3300 for (int &M : CommutedOuterMask) {
3301 if (M == PoisonMaskElem)
3302 continue;
3303 if (M < (int)NumTrunkElts)
3304 M += NumTrunkElts;
3305 else
3306 M -= NumTrunkElts;
3307 }
3308 OuterMask = CommutedOuterMask;
3309 }
3310 if (!OuterV1->hasOneUse())
3311 break;
3312
3313 if (!isa<UndefValue>(Val: A1)) {
3314 if (!Y)
3315 Y = A1;
3316 else if (Y != A1)
3317 break;
3318 }
3319 if (!isa<UndefValue>(Val: B1)) {
3320 if (!Y)
3321 Y = B1;
3322 else if (Y != B1)
3323 break;
3324 }
3325
3326 auto *YType = cast<FixedVectorType>(Val: A1->getType());
3327 int NumLeafElts = YType->getNumElements();
3328 SmallVector<int> LocalYMask(InnerMask1);
3329 for (int &M : LocalYMask) {
3330 if (M >= NumLeafElts)
3331 M -= NumLeafElts;
3332 }
3333
3334 InstructionCost LocalOldCost =
3335 TTI.getInstructionCost(U: cast<User>(Val: Trunk), CostKind) +
3336 TTI.getInstructionCost(U: cast<User>(Val: OuterV1), CostKind);
3337
3338 // Handle the initial (start of chain) case.
3339 if (!ChainLength) {
3340 Mask.assign(AR: OuterMask);
3341 YMask.assign(RHS: LocalYMask);
3342 OldCost = NewCost = LocalOldCost;
3343 Trunk = OuterV0;
3344 ChainLength++;
3345 continue;
3346 }
3347
3348 // For the non-root case, first attempt to combine masks.
3349 SmallVector<int> NewYMask(YMask);
3350 bool Valid = true;
3351 for (auto [CombinedM, LeafM] : llvm::zip(t&: NewYMask, u&: LocalYMask)) {
3352 if (LeafM == -1 || CombinedM == LeafM)
3353 continue;
3354 if (CombinedM == -1) {
3355 CombinedM = LeafM;
3356 } else {
3357 Valid = false;
3358 break;
3359 }
3360 }
3361 if (!Valid)
3362 break;
3363
3364 SmallVector<int> NewMask;
3365 NewMask.reserve(N: NumTrunkElts);
3366 for (int M : Mask) {
3367 if (M < 0 || M >= static_cast<int>(NumTrunkElts))
3368 NewMask.push_back(Elt: M);
3369 else
3370 NewMask.push_back(Elt: OuterMask[M]);
3371 }
3372
3373 // Break the chain if adding this new step complicates the shuffles such
3374 // that it would increase the new cost by more than the old cost of this
3375 // step.
3376 InstructionCost LocalNewCost =
3377 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc, DstTy: TrunkType,
3378 SrcTy: YType, CostKind, Mask: NewYMask) +
3379 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: TrunkType,
3380 SrcTy: TrunkType, CostKind, Mask: NewMask);
3381
3382 if (LocalNewCost >= NewCost && LocalOldCost < LocalNewCost - NewCost)
3383 break;
3384
3385 LLVM_DEBUG({
3386 if (ChainLength == 1) {
3387 dbgs() << "Found chain of shuffles fed by length-changing shuffles: "
3388 << I << '\n';
3389 }
3390 dbgs() << " next chain link: " << *Trunk << '\n'
3391 << " old cost: " << (OldCost + LocalOldCost)
3392 << " new cost: " << LocalNewCost << '\n';
3393 });
3394
3395 Mask = NewMask;
3396 YMask = NewYMask;
3397 OldCost += LocalOldCost;
3398 NewCost = LocalNewCost;
3399 Trunk = OuterV0;
3400 ChainLength++;
3401 }
3402 if (ChainLength <= 1)
3403 return false;
3404
3405 // Bail out if all leaves were poison.
3406 if (!Y)
3407 return false;
3408
3409 if (llvm::all_of(Range&: Mask, P: [&](int M) {
3410 return M < 0 || M >= static_cast<int>(NumTrunkElts);
3411 })) {
3412 // Produce a canonical simplified form if all elements are sourced from Y.
3413 for (int &M : Mask) {
3414 if (M >= static_cast<int>(NumTrunkElts))
3415 M = YMask[M - NumTrunkElts];
3416 }
3417 Value *Root =
3418 Builder.CreateShuffleVector(V1: Y, V2: PoisonValue::get(T: Y->getType()), Mask);
3419 replaceValue(Old&: I, New&: *Root);
3420 return true;
3421 }
3422
3423 Value *Leaf =
3424 Builder.CreateShuffleVector(V1: Y, V2: PoisonValue::get(T: Y->getType()), Mask: YMask);
3425 Value *Root = Builder.CreateShuffleVector(V1: Trunk, V2: Leaf, Mask);
3426 replaceValue(Old&: I, New&: *Root);
3427 return true;
3428}
3429
3430/// Try to convert
3431/// "shuffle (intrinsic), (intrinsic)" into "intrinsic (shuffle), (shuffle)".
3432bool VectorCombine::foldShuffleOfIntrinsics(Instruction &I) {
3433 Value *V0, *V1;
3434 ArrayRef<int> OldMask;
3435 if (!match(V: &I, P: m_Shuffle(v1: m_Value(V&: V0), v2: m_Value(V&: V1), mask: m_Mask(OldMask))))
3436 return false;
3437
3438 auto *II0 = dyn_cast<IntrinsicInst>(Val: V0);
3439 auto *II1 = dyn_cast<IntrinsicInst>(Val: V1);
3440 if (!II0 || !II1)
3441 return false;
3442
3443 Intrinsic::ID IID = II0->getIntrinsicID();
3444 if (IID != II1->getIntrinsicID())
3445 return false;
3446 InstructionCost CostII0 =
3447 TTI.getIntrinsicInstrCost(ICA: IntrinsicCostAttributes(IID, *II0), CostKind);
3448 InstructionCost CostII1 =
3449 TTI.getIntrinsicInstrCost(ICA: IntrinsicCostAttributes(IID, *II1), CostKind);
3450
3451 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3452 auto *II0Ty = dyn_cast<FixedVectorType>(Val: II0->getType());
3453 if (!ShuffleDstTy || !II0Ty)
3454 return false;
3455
3456 if (!isTriviallyVectorizable(ID: IID))
3457 return false;
3458
3459 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I) {
3460 Value *Arg0 = II0->getArgOperand(i: I);
3461 Value *Arg1 = II1->getArgOperand(i: I);
3462 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3463 // Scalar operands must be identical.
3464 if (Arg0 != Arg1)
3465 return false;
3466 } else if (Arg0->getType() != Arg1->getType()) {
3467 // The corresponding vector operands are shuffled together, so they must
3468 // share the same type. For intrinsics overloaded on their operand type
3469 // (e.g. llvm.fptosi.sat), two calls can produce the same result type
3470 // from different operand types; shuffling those would be invalid.
3471 return false;
3472 }
3473 }
3474
3475 InstructionCost OldCost =
3476 CostII0 + CostII1 +
3477 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ShuffleDstTy,
3478 SrcTy: II0Ty, CostKind, Mask: OldMask, Index: 0, SubTp: nullptr, Args: {II0, II1}, CxtI: &I);
3479
3480 SmallVector<Type *> NewArgsTy;
3481 InstructionCost NewCost = 0;
3482 SmallDenseSet<std::pair<Value *, Value *>> SeenOperandPairs;
3483 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I) {
3484 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3485 NewArgsTy.push_back(Elt: II0->getArgOperand(i: I)->getType());
3486 } else {
3487 auto *VecTy = cast<FixedVectorType>(Val: II0->getArgOperand(i: I)->getType());
3488 auto *ArgTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
3489 NumElts: ShuffleDstTy->getNumElements());
3490 NewArgsTy.push_back(Elt: ArgTy);
3491 std::pair<Value *, Value *> OperandPair =
3492 std::make_pair(x: II0->getArgOperand(i: I), y: II1->getArgOperand(i: I));
3493 if (!SeenOperandPairs.insert(V: OperandPair).second) {
3494 // We've already computed the cost for this operand pair.
3495 continue;
3496 }
3497 NewCost += TTI.getShuffleCost(
3498 Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ArgTy, SrcTy: VecTy, CostKind,
3499 Mask: OldMask, Index: 0, SubTp: nullptr, Args: {II0->getArgOperand(i: I), II1->getArgOperand(i: I)});
3500 }
3501 }
3502 IntrinsicCostAttributes NewAttr(IID, ShuffleDstTy, NewArgsTy);
3503
3504 NewCost += TTI.getIntrinsicInstrCost(ICA: NewAttr, CostKind);
3505 if (!II0->hasOneUse())
3506 NewCost += CostII0;
3507 if (II1 != II0 && !II1->hasOneUse())
3508 NewCost += CostII1;
3509
3510 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two intrinsics: " << I
3511 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
3512 << "\n");
3513
3514 if (NewCost > OldCost)
3515 return false;
3516
3517 SmallVector<Value *> NewArgs;
3518 SmallDenseMap<std::pair<Value *, Value *>, Value *> ShuffleCache;
3519 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I)
3520 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3521 NewArgs.push_back(Elt: II0->getArgOperand(i: I));
3522 } else {
3523 std::pair<Value *, Value *> OperandPair =
3524 std::make_pair(x: II0->getArgOperand(i: I), y: II1->getArgOperand(i: I));
3525 auto It = ShuffleCache.find(Val: OperandPair);
3526 if (It != ShuffleCache.end()) {
3527 // Reuse previously created shuffle for this operand pair.
3528 NewArgs.push_back(Elt: It->second);
3529 continue;
3530 }
3531 Value *Shuf = Builder.CreateShuffleVector(V1: II0->getArgOperand(i: I),
3532 V2: II1->getArgOperand(i: I), Mask: OldMask);
3533 ShuffleCache[OperandPair] = Shuf;
3534 NewArgs.push_back(Elt: Shuf);
3535 Worklist.pushValue(V: Shuf);
3536 }
3537 Value *NewIntrinsic = Builder.CreateIntrinsic(RetTy: ShuffleDstTy, ID: IID, Args: NewArgs);
3538
3539 // Intersect flags from the old intrinsics.
3540 if (auto *NewInst = dyn_cast<Instruction>(Val: NewIntrinsic)) {
3541 NewInst->copyIRFlags(V: II0);
3542 NewInst->andIRFlags(V: II1);
3543 }
3544
3545 replaceValue(Old&: I, New&: *NewIntrinsic);
3546 return true;
3547}
3548
3549/// Try to convert
3550/// "shuffle (intrinsic), (poison/undef)" into "intrinsic (shuffle)".
3551bool VectorCombine::foldPermuteOfIntrinsic(Instruction &I) {
3552 Value *V0;
3553 ArrayRef<int> Mask;
3554 if (!match(V: &I, P: m_Shuffle(v1: m_Value(V&: V0), v2: m_Undef(), mask: m_Mask(Mask))))
3555 return false;
3556
3557 auto *II0 = dyn_cast<IntrinsicInst>(Val: V0);
3558 if (!II0)
3559 return false;
3560
3561 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3562 auto *IntrinsicSrcTy = dyn_cast<FixedVectorType>(Val: II0->getType());
3563 if (!ShuffleDstTy || !IntrinsicSrcTy)
3564 return false;
3565
3566 // Validate it's a pure permute, mask should only reference the first vector
3567 unsigned NumSrcElts = IntrinsicSrcTy->getNumElements();
3568 if (any_of(Range&: Mask, P: [NumSrcElts](int M) { return M >= (int)NumSrcElts; }))
3569 return false;
3570
3571 Intrinsic::ID IID = II0->getIntrinsicID();
3572 if (!isTriviallyVectorizable(ID: IID))
3573 return false;
3574
3575 // Cost analysis
3576 InstructionCost IntrinsicCost =
3577 TTI.getIntrinsicInstrCost(ICA: IntrinsicCostAttributes(IID, *II0), CostKind);
3578 InstructionCost OldCost =
3579 IntrinsicCost +
3580 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc, DstTy: ShuffleDstTy,
3581 SrcTy: IntrinsicSrcTy, CostKind, Mask, Index: 0, SubTp: nullptr, Args: {V0}, CxtI: &I);
3582
3583 SmallVector<Type *> NewArgsTy;
3584 InstructionCost NewCost = 0;
3585 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I) {
3586 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3587 NewArgsTy.push_back(Elt: II0->getArgOperand(i: I)->getType());
3588 } else {
3589 auto *VecTy = cast<FixedVectorType>(Val: II0->getArgOperand(i: I)->getType());
3590 auto *ArgTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
3591 NumElts: ShuffleDstTy->getNumElements());
3592 NewArgsTy.push_back(Elt: ArgTy);
3593 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
3594 DstTy: ArgTy, SrcTy: VecTy, CostKind, Mask, Index: 0, SubTp: nullptr,
3595 Args: {II0->getArgOperand(i: I)});
3596 }
3597 }
3598 IntrinsicCostAttributes NewAttr(IID, ShuffleDstTy, NewArgsTy);
3599 NewCost += TTI.getIntrinsicInstrCost(ICA: NewAttr, CostKind);
3600
3601 // If the intrinsic has multiple uses, we need to account for the cost of
3602 // keeping the original intrinsic around.
3603 if (!II0->hasOneUse())
3604 NewCost += IntrinsicCost;
3605
3606 LLVM_DEBUG(dbgs() << "Found a permute of intrinsic: " << I << "\n OldCost: "
3607 << OldCost << " vs NewCost: " << NewCost << "\n");
3608
3609 if (NewCost > OldCost)
3610 return false;
3611
3612 // Transform
3613 SmallVector<Value *> NewArgs;
3614 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I) {
3615 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3616 NewArgs.push_back(Elt: II0->getArgOperand(i: I));
3617 } else {
3618 Value *Shuf = Builder.CreateShuffleVector(V: II0->getArgOperand(i: I), Mask);
3619 NewArgs.push_back(Elt: Shuf);
3620 Worklist.pushValue(V: Shuf);
3621 }
3622 }
3623
3624 Value *NewIntrinsic = Builder.CreateIntrinsic(RetTy: ShuffleDstTy, ID: IID, Args: NewArgs);
3625
3626 if (auto *NewInst = dyn_cast<Instruction>(Val: NewIntrinsic))
3627 NewInst->copyIRFlags(V: II0);
3628
3629 replaceValue(Old&: I, New&: *NewIntrinsic);
3630 return true;
3631}
3632
3633using InstLane = std::pair<Value *, int>;
3634
3635static InstLane lookThroughShuffles(Value *V, int Lane) {
3636 while (auto *SV = dyn_cast<ShuffleVectorInst>(Val: V)) {
3637 unsigned NumElts =
3638 cast<FixedVectorType>(Val: SV->getOperand(i_nocapture: 0)->getType())->getNumElements();
3639 int M = SV->getMaskValue(Elt: Lane);
3640 if (M < 0)
3641 return {nullptr, PoisonMaskElem};
3642 if (static_cast<unsigned>(M) < NumElts) {
3643 V = SV->getOperand(i_nocapture: 0);
3644 Lane = M;
3645 } else {
3646 V = SV->getOperand(i_nocapture: 1);
3647 Lane = M - NumElts;
3648 }
3649 }
3650 return InstLane{V, Lane};
3651}
3652
3653static SmallVector<InstLane>
3654generateInstLaneVectorFromOperand(ArrayRef<InstLane> Item, int Op) {
3655 SmallVector<InstLane> NItem;
3656 for (InstLane IL : Item) {
3657 auto [U, Lane] = IL;
3658 InstLane OpLane =
3659 U ? lookThroughShuffles(V: cast<Instruction>(Val: U)->getOperand(i: Op), Lane)
3660 : InstLane{nullptr, PoisonMaskElem};
3661 NItem.emplace_back(Args&: OpLane);
3662 }
3663 return NItem;
3664}
3665
3666/// Detect concat of multiple values into a vector
3667static bool isFreeConcat(ArrayRef<InstLane> Item, TTI::TargetCostKind CostKind,
3668 const TargetTransformInfo &TTI) {
3669 auto *Ty = cast<FixedVectorType>(Val: Item.front().first->getType());
3670 unsigned NumElts = Ty->getNumElements();
3671 if (Item.size() == NumElts || NumElts == 1 || Item.size() % NumElts != 0)
3672 return false;
3673
3674 // Check that the concat is free, usually meaning that the type will be split
3675 // during legalization.
3676 SmallVector<int, 16> ConcatMask(NumElts * 2);
3677 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
3678 if (TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc,
3679 DstTy: FixedVectorType::get(ElementType: Ty->getScalarType(), NumElts: NumElts * 2),
3680 SrcTy: Ty, CostKind, Mask: ConcatMask) != 0)
3681 return false;
3682
3683 unsigned NumSlices = Item.size() / NumElts;
3684 // Currently we generate a tree of shuffles for the concats, which limits us
3685 // to a power2.
3686 if (!isPowerOf2_32(Value: NumSlices))
3687 return false;
3688 for (unsigned Slice = 0; Slice < NumSlices; ++Slice) {
3689 Value *SliceV = Item[Slice * NumElts].first;
3690 if (!SliceV || SliceV->getType() != Ty)
3691 return false;
3692 for (unsigned Elt = 0; Elt < NumElts; ++Elt) {
3693 auto [V, Lane] = Item[Slice * NumElts + Elt];
3694 if (Lane != static_cast<int>(Elt) || SliceV != V)
3695 return false;
3696 }
3697 }
3698 return true;
3699}
3700
3701static Value *
3702generateNewInstTree(ArrayRef<InstLane> Item, Use *From,
3703 const DenseSet<std::pair<Value *, Use *>> &IdentityLeafs,
3704 const DenseSet<std::pair<Value *, Use *>> &SplatLeafs,
3705 const DenseSet<std::pair<Value *, Use *>> &ConcatLeafs,
3706 IRBuilderBase &Builder, InstructionWorklist &WorkList,
3707 const TargetTransformInfo *TTI) {
3708 auto [FrontV, FrontLane] = Item.front();
3709
3710 if (IdentityLeafs.contains(V: std::make_pair(x&: FrontV, y&: From))) {
3711 return FrontV;
3712 }
3713 if (SplatLeafs.contains(V: std::make_pair(x&: FrontV, y&: From))) {
3714 SmallVector<int, 16> Mask(Item.size(), FrontLane);
3715 return Builder.CreateShuffleVector(V: FrontV, Mask);
3716 }
3717 if (ConcatLeafs.contains(V: std::make_pair(x&: FrontV, y&: From))) {
3718 unsigned NumElts =
3719 cast<FixedVectorType>(Val: FrontV->getType())->getNumElements();
3720 SmallVector<Value *> Values(Item.size() / NumElts, nullptr);
3721 for (unsigned S = 0; S < Values.size(); ++S)
3722 Values[S] = Item[S * NumElts].first;
3723
3724 while (Values.size() > 1) {
3725 NumElts *= 2;
3726 SmallVector<int, 16> Mask(NumElts, 0);
3727 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
3728 SmallVector<Value *> NewValues(Values.size() / 2, nullptr);
3729 for (unsigned S = 0; S < NewValues.size(); ++S)
3730 NewValues[S] =
3731 Builder.CreateShuffleVector(V1: Values[S * 2], V2: Values[S * 2 + 1], Mask);
3732 Values = NewValues;
3733 }
3734 return Values[0];
3735 }
3736
3737 auto *I = cast<Instruction>(Val: FrontV);
3738
3739 // Handle vector bitcasts that change element count. We cannot use
3740 // generateInstLaneVectorFromOperand for these because the lane indices
3741 // don't map 1:1 through the bitcast.
3742 if (auto *BitCast = dyn_cast<BitCastInst>(Val: I)) {
3743 auto *BCDstTy = dyn_cast<FixedVectorType>(Val: BitCast->getDestTy());
3744 auto *BCSrcTy = dyn_cast<FixedVectorType>(Val: BitCast->getSrcTy());
3745 if (BCDstTy && BCSrcTy &&
3746 BCDstTy->getElementCount() != BCSrcTy->getElementCount()) {
3747 unsigned DstElts = BCDstTy->getNumElements();
3748 unsigned SrcElts = BCSrcTy->getNumElements();
3749 SmallVector<InstLane> NewItem;
3750 if (DstElts > SrcElts) {
3751 // Widening: compress operand Item.
3752 unsigned R = DstElts / SrcElts;
3753 if (Item.size() % R != 0)
3754 return nullptr;
3755 for (unsigned Idx = 0, E = Item.size(); Idx < E; Idx += R) {
3756 auto [V, Lane] = Item[Idx];
3757 if (!V) {
3758 NewItem.push_back(Elt: {nullptr, PoisonMaskElem});
3759 continue;
3760 }
3761 NewItem.push_back(
3762 Elt: lookThroughShuffles(V: cast<Operator>(Val: V)->getOperand(i: 0), Lane: Lane / R));
3763 }
3764 } else {
3765 // Narrowing: expand operand Item.
3766 unsigned R = SrcElts / DstElts;
3767 for (auto [V, Lane] : Item) {
3768 if (!V) {
3769 NewItem.append(NumInputs: R, Elt: {nullptr, PoisonMaskElem});
3770 continue;
3771 }
3772 Value *Op = cast<Operator>(Val: V)->getOperand(i: 0);
3773 for (unsigned J = 0; J < R; ++J)
3774 NewItem.push_back(Elt: lookThroughShuffles(V: Op, Lane: Lane * R + J));
3775 }
3776 }
3777 Value *Op = generateNewInstTree(Item: NewItem, From: &BitCast->getOperandUse(i: 0),
3778 IdentityLeafs, SplatLeafs, ConcatLeafs,
3779 Builder, WorkList, TTI);
3780 WorkList.pushValue(V: Op);
3781 return Builder.CreateBitCast(
3782 V: Op, DestTy: FixedVectorType::get(ElementType: BCDstTy->getScalarType(), NumElts: Item.size()));
3783 }
3784 }
3785 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3786 unsigned NumOps = I->getNumOperands() - (II ? 1 : 0);
3787 SmallVector<Value *> Ops(NumOps);
3788 for (unsigned Idx = 0; Idx < NumOps; Idx++) {
3789 if (II &&
3790 isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(), ScalarOpdIdx: Idx, TTI)) {
3791 Ops[Idx] = II->getOperand(i_nocapture: Idx);
3792 continue;
3793 }
3794 Ops[Idx] = generateNewInstTree(
3795 Item: generateInstLaneVectorFromOperand(Item, Op: Idx), From: &I->getOperandUse(i: Idx),
3796 IdentityLeafs, SplatLeafs, ConcatLeafs, Builder, WorkList, TTI);
3797 // Don't re-queue the operand of a bitcast we just regenerated. Doing so
3798 // lets foldBitcastShuffle sink the bitcast back into a shuffle(bitcast),
3799 // which foldShuffleToIdentity then re-matches as the same superfluous
3800 // identity - an infinite loop between the two folds.
3801 if (!isa<BitCastInst>(Val: I))
3802 WorkList.pushValue(V: Ops[Idx]);
3803 }
3804
3805 SmallVector<Value *, 8> ValueList;
3806 for (const auto &Lane : Item)
3807 if (Lane.first)
3808 ValueList.push_back(Elt: Lane.first);
3809
3810 Type *DstTy =
3811 FixedVectorType::get(ElementType: I->getType()->getScalarType(), NumElts: Item.size());
3812 if (auto *BI = dyn_cast<BinaryOperator>(Val: I)) {
3813 auto *Value = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)BI->getOpcode(),
3814 LHS: Ops[0], RHS: Ops[1]);
3815 propagateIRFlags(I: Value, VL: ValueList);
3816 return Value;
3817 }
3818 if (auto *CI = dyn_cast<CmpInst>(Val: I)) {
3819 auto *Value = Builder.CreateCmp(Pred: CI->getPredicate(), LHS: Ops[0], RHS: Ops[1]);
3820 propagateIRFlags(I: Value, VL: ValueList);
3821 return Value;
3822 }
3823 if (auto *SI = dyn_cast<SelectInst>(Val: I)) {
3824 auto *Value = Builder.CreateSelect(C: Ops[0], True: Ops[1], False: Ops[2], Name: "", MDFrom: SI);
3825 propagateIRFlags(I: Value, VL: ValueList);
3826 return Value;
3827 }
3828 if (auto *CI = dyn_cast<CastInst>(Val: I)) {
3829 auto *Value = Builder.CreateCast(Op: CI->getOpcode(), V: Ops[0], DestTy: DstTy);
3830 propagateIRFlags(I: Value, VL: ValueList);
3831 return Value;
3832 }
3833 if (II) {
3834 auto *Value = Builder.CreateIntrinsic(RetTy: DstTy, ID: II->getIntrinsicID(), Args: Ops);
3835 propagateIRFlags(I: Value, VL: ValueList);
3836 return Value;
3837 }
3838 assert(isa<UnaryInstruction>(I) && "Unexpected instruction type in Generate");
3839 auto *Value =
3840 Builder.CreateUnOp(Opc: (Instruction::UnaryOps)I->getOpcode(), V: Ops[0]);
3841 propagateIRFlags(I: Value, VL: ValueList);
3842 return Value;
3843}
3844
3845// Starting from a shuffle, look up through operands tracking the shuffled index
3846// of each lane. If we can simplify away the shuffles to identities then
3847// do so.
3848bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
3849 auto *Ty = dyn_cast<FixedVectorType>(Val: I.getType());
3850 if (!Ty || I.use_empty())
3851 return false;
3852
3853 SmallVector<InstLane> Start(Ty->getNumElements());
3854 for (unsigned M = 0, E = Ty->getNumElements(); M < E; ++M)
3855 Start[M] = lookThroughShuffles(V: &I, Lane: M);
3856
3857 SmallVector<std::pair<SmallVector<InstLane>, Use *>> Candidates;
3858 Candidates.push_back(Elt: std::make_pair(x&: Start, y: &*I.use_begin()));
3859 DenseSet<std::pair<Value *, Use *>> IdentityLeafs, SplatLeafs, ConcatLeafs;
3860 unsigned NumVisited = 0;
3861 bool TraversedElCountChangingBitcast = false;
3862
3863 while (!Candidates.empty()) {
3864 if (++NumVisited > MaxInstrsToScan)
3865 return false;
3866
3867 auto ItemFrom = Candidates.pop_back_val();
3868 auto Item = ItemFrom.first;
3869 auto From = ItemFrom.second;
3870 auto [FrontV, FrontLane] = Item.front();
3871
3872 // If we found an undef first lane then bail out to keep things simple.
3873 if (!FrontV)
3874 return false;
3875
3876 // Look for an identity value.
3877 if (FrontLane == 0 &&
3878 cast<FixedVectorType>(Val: FrontV->getType())->getNumElements() ==
3879 Item.size() &&
3880 all_of(Range: drop_begin(RangeOrContainer: enumerate(First&: Item)), P: [Item](const auto &E) {
3881 Value *FrontV = Item.front().first;
3882 return !E.value().first || (isEquivBitcast(E.value().first, FrontV) &&
3883 E.value().second == (int)E.index());
3884 })) {
3885 IdentityLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3886 continue;
3887 }
3888 // Look for constants, for the moment only supporting constant splats.
3889 if (auto *C = dyn_cast<Constant>(Val: FrontV);
3890 C && C->getSplatValue() &&
3891 all_of(Range: drop_begin(RangeOrContainer&: Item), P: [Item](InstLane &IL) {
3892 Value *FrontV = Item.front().first;
3893 Value *V = IL.first;
3894 return !V || (isa<Constant>(Val: V) &&
3895 cast<Constant>(Val: V)->getSplatValue() ==
3896 cast<Constant>(Val: FrontV)->getSplatValue());
3897 })) {
3898 SplatLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3899 continue;
3900 }
3901 // Look for a splat value.
3902 if (all_of(Range: drop_begin(RangeOrContainer&: Item), P: [Item](InstLane &IL) {
3903 auto [FrontV, FrontLane] = Item.front();
3904 auto [V, Lane] = IL;
3905 return !V || (V == FrontV && Lane == FrontLane);
3906 })) {
3907 SplatLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3908 continue;
3909 }
3910
3911 // We need each element to be the same type of value, and check that each
3912 // element has a single use.
3913 auto CheckLaneIsEquivalentToFirst = [Item](InstLane IL) {
3914 Value *FrontV = Item.front().first;
3915 if (!IL.first)
3916 return true;
3917 Value *V = IL.first;
3918 if (auto *I = dyn_cast<Instruction>(Val: V); I && !I->hasOneUser())
3919 return false;
3920 if (V->getValueID() != FrontV->getValueID())
3921 return false;
3922 if (auto *CI = dyn_cast<CmpInst>(Val: V))
3923 if (CI->getPredicate() != cast<CmpInst>(Val: FrontV)->getPredicate())
3924 return false;
3925 if (auto *CI = dyn_cast<CastInst>(Val: V))
3926 if (CI->getSrcTy()->getScalarType() !=
3927 cast<CastInst>(Val: FrontV)->getSrcTy()->getScalarType())
3928 return false;
3929 if (auto *SI = dyn_cast<SelectInst>(Val: V))
3930 if (!isa<VectorType>(Val: SI->getOperand(i_nocapture: 0)->getType()) ||
3931 SI->getOperand(i_nocapture: 0)->getType() !=
3932 cast<SelectInst>(Val: FrontV)->getOperand(i_nocapture: 0)->getType())
3933 return false;
3934 if (isa<CallInst>(Val: V) && !isa<IntrinsicInst>(Val: V))
3935 return false;
3936 auto *II = dyn_cast<IntrinsicInst>(Val: V);
3937 return !II || (isa<IntrinsicInst>(Val: FrontV) &&
3938 II->getIntrinsicID() ==
3939 cast<IntrinsicInst>(Val: FrontV)->getIntrinsicID() &&
3940 !II->hasOperandBundles());
3941 };
3942 if (all_of(Range: drop_begin(RangeOrContainer&: Item), P: CheckLaneIsEquivalentToFirst)) {
3943 // Check the operator is one that we support.
3944 if (isa<BinaryOperator, CmpInst>(Val: FrontV)) {
3945 // We exclude div/rem in case they hit UB from poison lanes.
3946 if (auto *BO = dyn_cast<BinaryOperator>(Val: FrontV);
3947 BO && BO->isIntDivRem())
3948 return false;
3949 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3950 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: 0));
3951 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 1),
3952 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: 1));
3953 continue;
3954 } else if (isa<UnaryOperator, TruncInst, ZExtInst, SExtInst, FPToSIInst,
3955 FPToUIInst, SIToFPInst, UIToFPInst>(Val: FrontV)) {
3956 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3957 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: 0));
3958 continue;
3959 } else if (auto *BitCast = dyn_cast<BitCastInst>(Val: FrontV)) {
3960 auto *BCDstTy = dyn_cast<FixedVectorType>(Val: BitCast->getDestTy());
3961 auto *BCSrcTy = dyn_cast<FixedVectorType>(Val: BitCast->getSrcTy());
3962 if (BCDstTy && BCSrcTy) {
3963 ElementCount DstEC = BCDstTy->getElementCount();
3964 ElementCount SrcEC = BCSrcTy->getElementCount();
3965 if (DstEC == SrcEC) {
3966 // Same element count - simple pass-through.
3967 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3968 Args: &BitCast->getOperandUse(i: 0));
3969 continue;
3970 }
3971 unsigned DstElts = DstEC.getFixedValue();
3972 unsigned SrcElts = SrcEC.getFixedValue();
3973 if (DstElts > SrcElts && DstElts % SrcElts == 0) {
3974 // Widening bitcast (e.g. <2 x i32> -> <4 x i16>). Compress
3975 // consecutive groups of R destination lanes into one source
3976 // lane.
3977 unsigned R = DstElts / SrcElts;
3978 SmallVector<InstLane> NItem;
3979 bool Valid = Item.size() % R == 0;
3980 for (unsigned Idx = 0, E = Item.size(); Valid && Idx < E;
3981 Idx += R) {
3982 auto [V0, L0] = Item[Idx];
3983 if (!V0) {
3984 if (any_of(Range: ArrayRef(Item).slice(N: Idx + 1, M: R - 1),
3985 P: [](InstLane IL) { return IL.first != nullptr; })) {
3986 Valid = false;
3987 break;
3988 }
3989 NItem.push_back(Elt: {nullptr, PoisonMaskElem});
3990 continue;
3991 }
3992 if (L0 % R != 0) {
3993 Valid = false;
3994 break;
3995 }
3996 for (unsigned J = 1; J < R; ++J) {
3997 auto [VJ, LJ] = Item[Idx + J];
3998 if (!VJ || VJ != V0 || LJ != L0 + (int)J) {
3999 Valid = false;
4000 break;
4001 }
4002 }
4003 if (!Valid)
4004 break;
4005 NItem.push_back(Elt: lookThroughShuffles(
4006 V: cast<Operator>(Val: V0)->getOperand(i: 0), Lane: L0 / R));
4007 }
4008 if (Valid) {
4009 TraversedElCountChangingBitcast = true;
4010 Candidates.emplace_back(Args&: NItem, Args: &BitCast->getOperandUse(i: 0));
4011 continue;
4012 }
4013 } else if (SrcElts > DstElts && SrcElts % DstElts == 0) {
4014 // Narrowing bitcast (e.g. <4 x i16> -> <2 x i32>). Expand
4015 // each destination lane into R source lanes.
4016 unsigned R = SrcElts / DstElts;
4017 SmallVector<InstLane> NItem;
4018 for (auto [V, Lane] : Item) {
4019 if (!V) {
4020 NItem.append(NumInputs: R, Elt: {nullptr, PoisonMaskElem});
4021 continue;
4022 }
4023 Value *Op = cast<Operator>(Val: V)->getOperand(i: 0);
4024 for (unsigned J = 0; J < R; ++J)
4025 NItem.push_back(Elt: lookThroughShuffles(V: Op, Lane: Lane * R + J));
4026 }
4027 TraversedElCountChangingBitcast = true;
4028 Candidates.emplace_back(Args&: NItem, Args: &BitCast->getOperandUse(i: 0));
4029 continue;
4030 }
4031 }
4032 } else if (auto *Sel = dyn_cast<SelectInst>(Val: FrontV)) {
4033 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
4034 Args: &Sel->getOperandUse(i: 0));
4035 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 1),
4036 Args: &Sel->getOperandUse(i: 1));
4037 Candidates.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 2),
4038 Args: &Sel->getOperandUse(i: 2));
4039 continue;
4040 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: FrontV);
4041 II && isTriviallyVectorizable(ID: II->getIntrinsicID()) &&
4042 !II->hasOperandBundles()) {
4043 for (unsigned Op = 0, E = II->getNumOperands() - 1; Op < E; Op++) {
4044 if (isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(), ScalarOpdIdx: Op,
4045 TTI: &TTI)) {
4046 if (!all_of(Range: drop_begin(RangeOrContainer&: Item), P: [Item, Op](InstLane &IL) {
4047 Value *FrontV = Item.front().first;
4048 Value *V = IL.first;
4049 return !V || (cast<Instruction>(Val: V)->getOperand(i: Op) ==
4050 cast<Instruction>(Val: FrontV)->getOperand(i: Op));
4051 }))
4052 return false;
4053 continue;
4054 }
4055 Candidates.emplace_back(
4056 Args: generateInstLaneVectorFromOperand(Item, Op),
4057 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: Op));
4058 }
4059 continue;
4060 }
4061 }
4062
4063 if (isFreeConcat(Item, CostKind, TTI)) {
4064 ConcatLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
4065 continue;
4066 }
4067
4068 return false;
4069 }
4070
4071 if (NumVisited <= 1)
4072 return false;
4073
4074 // If the only non-leaf node traversed was a single bitcast that changes
4075 // element count, the fold would just commute the bitcast and shuffle.
4076 // foldBitcastShuffle does the reverse transform, causing an infinite loop.
4077 if (NumVisited == 2 && TraversedElCountChangingBitcast)
4078 return false;
4079
4080 LLVM_DEBUG(dbgs() << "Found a superfluous identity shuffle: " << I << "\n");
4081
4082 // If we got this far, we know the shuffles are superfluous and can be
4083 // removed. Scan through again and generate the new tree of instructions.
4084 Builder.SetInsertPoint(&I);
4085 Value *V =
4086 generateNewInstTree(Item: Start, From: &*I.use_begin(), IdentityLeafs, SplatLeafs,
4087 ConcatLeafs, Builder, WorkList&: Worklist, TTI: &TTI);
4088 replaceValue(Old&: I, New&: *V);
4089 return true;
4090}
4091
4092/// Given a commutative reduction, the order of the input lanes does not alter
4093/// the results. We can use this to remove certain shuffles feeding the
4094/// reduction, removing the need to shuffle at all.
4095bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
4096 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
4097 if (!II)
4098 return false;
4099 switch (II->getIntrinsicID()) {
4100 case Intrinsic::vector_reduce_add:
4101 case Intrinsic::vector_reduce_mul:
4102 case Intrinsic::vector_reduce_and:
4103 case Intrinsic::vector_reduce_or:
4104 case Intrinsic::vector_reduce_xor:
4105 case Intrinsic::vector_reduce_smin:
4106 case Intrinsic::vector_reduce_smax:
4107 case Intrinsic::vector_reduce_umin:
4108 case Intrinsic::vector_reduce_umax:
4109 break;
4110 default:
4111 return false;
4112 }
4113
4114 // Find all the inputs when looking through operations that do not alter the
4115 // lane order (binops, for example). Currently we look for a single shuffle,
4116 // and can ignore splat values.
4117 std::queue<Value *> Worklist;
4118 SmallPtrSet<Value *, 4> Visited;
4119 ShuffleVectorInst *Shuffle = nullptr;
4120 if (auto *Op = dyn_cast<Instruction>(Val: I.getOperand(i: 0)))
4121 Worklist.push(x: Op);
4122
4123 while (!Worklist.empty()) {
4124 Value *CV = Worklist.front();
4125 Worklist.pop();
4126 if (Visited.contains(Ptr: CV))
4127 continue;
4128
4129 // Splats don't change the order, so can be safely ignored.
4130 if (isSplatValue(V: CV))
4131 continue;
4132
4133 Visited.insert(Ptr: CV);
4134
4135 if (auto *CI = dyn_cast<Instruction>(Val: CV)) {
4136 if (CI->isBinaryOp()) {
4137 for (auto *Op : CI->operand_values())
4138 Worklist.push(x: Op);
4139 continue;
4140 } else if (auto *SV = dyn_cast<ShuffleVectorInst>(Val: CI)) {
4141 if (Shuffle && Shuffle != SV)
4142 return false;
4143 Shuffle = SV;
4144 continue;
4145 }
4146 }
4147
4148 // Anything else is currently an unknown node.
4149 return false;
4150 }
4151
4152 if (!Shuffle)
4153 return false;
4154
4155 // Check all uses of the binary ops and shuffles are also included in the
4156 // lane-invariant operations (Visited should be the list of lanewise
4157 // instructions, including the shuffle that we found).
4158 for (auto *V : Visited)
4159 for (auto *U : V->users())
4160 if (!Visited.contains(Ptr: U) && U != &I)
4161 return false;
4162
4163 FixedVectorType *VecType =
4164 dyn_cast<FixedVectorType>(Val: II->getOperand(i_nocapture: 0)->getType());
4165 if (!VecType)
4166 return false;
4167 FixedVectorType *ShuffleInputType =
4168 dyn_cast<FixedVectorType>(Val: Shuffle->getOperand(i_nocapture: 0)->getType());
4169 if (!ShuffleInputType)
4170 return false;
4171 unsigned NumInputElts = ShuffleInputType->getNumElements();
4172
4173 // Find the mask from sorting the lanes into order. This is most likely to
4174 // become a identity or concat mask. Undef elements are pushed to the end.
4175 SmallVector<int> ConcatMask;
4176 Shuffle->getShuffleMask(Result&: ConcatMask);
4177 sort(C&: ConcatMask, Comp: [](int X, int Y) { return (unsigned)X < (unsigned)Y; });
4178 bool UsesSecondVec =
4179 any_of(Range&: ConcatMask, P: [&](int M) { return M >= (int)NumInputElts; });
4180
4181 InstructionCost OldCost = TTI.getShuffleCost(
4182 Kind: UsesSecondVec ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc, DstTy: VecType,
4183 SrcTy: ShuffleInputType, CostKind, Mask: Shuffle->getShuffleMask());
4184 InstructionCost NewCost = TTI.getShuffleCost(
4185 Kind: UsesSecondVec ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc, DstTy: VecType,
4186 SrcTy: ShuffleInputType, CostKind, Mask: ConcatMask);
4187
4188 LLVM_DEBUG(dbgs() << "Found a reduction feeding from a shuffle: " << *Shuffle
4189 << "\n");
4190 LLVM_DEBUG(dbgs() << " OldCost: " << OldCost << " vs NewCost: " << NewCost
4191 << "\n");
4192 bool MadeChanges = false;
4193 if (NewCost < OldCost) {
4194 Builder.SetInsertPoint(Shuffle);
4195 Value *NewShuffle = Builder.CreateShuffleVector(
4196 V1: Shuffle->getOperand(i_nocapture: 0), V2: Shuffle->getOperand(i_nocapture: 1), Mask: ConcatMask);
4197 LLVM_DEBUG(dbgs() << "Created new shuffle: " << *NewShuffle << "\n");
4198 replaceValue(Old&: *Shuffle, New&: *NewShuffle);
4199 return true;
4200 }
4201
4202 // See if we can re-use foldSelectShuffle, getting it to reduce the size of
4203 // the shuffle into a nicer order, as it can ignore the order of the shuffles.
4204 MadeChanges |= foldSelectShuffle(I&: *Shuffle, FromReduction: true);
4205 return MadeChanges;
4206}
4207
4208/// Try to fold a chain of shuffles and ops feeding extractelement(..., 0)
4209/// into llvm.vector.reduce.*, by tracking which lanes contribute to the
4210/// extracted lane and reducing the widest vector whose lanes each contribute
4211/// once.
4212///
4213/// For example:
4214///
4215/// %lo = shufflevector <4 x i32> %a, poison, <2 x i32> <i32 0, i32 1>
4216/// %hi = shufflevector <4 x i32> %a, poison, <2 x i32> <i32 2, i32 3>
4217/// %s = add <2 x i32> %lo, %hi
4218/// %sh = shufflevector <2 x i32> %s, poison, <2 x i32> <i32 1, i32 poison>
4219/// %r = add <2 x i32> %s, %sh
4220/// %e = extractelement <2 x i32> %r, i64 0
4221///
4222/// transforms to:
4223///
4224/// %e = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
4225bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
4226 Value *VecOpEE;
4227 if (!match(V: &I, P: m_ExtractElt(Val: m_Value(V&: VecOpEE), Idx: m_Zero())))
4228 return false;
4229
4230 auto *FVT = dyn_cast<FixedVectorType>(Val: VecOpEE->getType());
4231 if (!FVT)
4232 return false;
4233
4234 if (FVT->getNumElements() < 2)
4235 return false;
4236
4237 std::optional<Instruction::BinaryOps> CommonBinOp;
4238 std::optional<Intrinsic::ID> CommonCallOp;
4239
4240 if (auto *BO = dyn_cast<BinaryOperator>(Val: VecOpEE)) {
4241 if (!getReductionForBinop(Opc: BO->getOpcode()))
4242 return false;
4243 CommonBinOp = BO->getOpcode();
4244 } else if (auto *MMI = dyn_cast<MinMaxIntrinsic>(Val: VecOpEE)) {
4245 CommonCallOp = MMI->getIntrinsicID();
4246 } else {
4247 return false;
4248 }
4249
4250 // For floating-point reductions, track FMF intersection across all binops.
4251 FastMathFlags CommonFMF;
4252 bool IsFloatReduction = false;
4253
4254 // A chain node is one we walk through, either a matching-opcode binop/min-max
4255 // or a single-source shuffle. Anything else is a leaf source.
4256 auto IsChainNode = [&](Value *V) {
4257 if (auto *BO = dyn_cast<BinaryOperator>(Val: V))
4258 return CommonBinOp && BO->getOpcode() == *CommonBinOp;
4259 if (auto *MMI = dyn_cast<MinMaxIntrinsic>(Val: V))
4260 return CommonCallOp && MMI->getIntrinsicID() == *CommonCallOp;
4261 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V))
4262 return isa<PoisonValue>(Val: SVI->getOperand(i_nocapture: 1));
4263 return false;
4264 };
4265
4266 // Collect the chain, building Nodes in postorder. Bail if the chain is empty
4267 // or exceeds MaxChainNodes.
4268 constexpr unsigned MaxChainNodes = 32;
4269 SmallSetVector<Value *, 16> Nodes;
4270 SmallSetVector<Value *, 4> Sources;
4271 unsigned NumVisited = 0;
4272 auto AddSource = [&](Value *V) {
4273 if (!isa<FixedVectorType>(Val: V->getType()))
4274 return false;
4275 Sources.insert(X: V);
4276 return true;
4277 };
4278 auto Walk = [&](Value *V, auto &&Walk) -> bool {
4279 if (Nodes.contains(key: V) || Sources.contains(key: V))
4280 return true;
4281 if (++NumVisited > MaxChainNodes)
4282 return false;
4283 if (!IsChainNode(V))
4284 return AddSource(V);
4285 // Chain shuffles always have poison as op1, so only op0 matters.
4286 auto *U = cast<Instruction>(Val: V);
4287 unsigned NumOps = isa<ShuffleVectorInst>(Val: U) ? 1 : 2;
4288 for (unsigned I = 0; I != NumOps; ++I)
4289 if (!Walk(U->getOperand(i: I), Walk))
4290 return false;
4291 if (isa<ShuffleVectorInst>(Val: U) || Nodes.contains(key: U->getOperand(i: 0)) ||
4292 Nodes.contains(key: U->getOperand(i: 1))) {
4293 Nodes.insert(X: V);
4294 return true;
4295 }
4296 // Both operands are leaves so treat this binop as a source rather than
4297 // walking into it.
4298 return AddSource(V);
4299 };
4300 if (!Walk(VecOpEE, Walk) || Nodes.empty())
4301 return false;
4302
4303 bool IsIdempotent =
4304 CommonCallOp || (CommonBinOp && Instruction::isIdempotent(Opcode: *CommonBinOp));
4305
4306 // For FP reductions, require reassoc on every binop and collect FMF.
4307 for (Value *V : Nodes) {
4308 auto *BinOp = dyn_cast<BinaryOperator>(Val: V);
4309 if (!BinOp || !BinOp->getType()->isFPOrFPVectorTy())
4310 continue;
4311 if (!BinOp->hasAllowReassoc())
4312 return false;
4313 if (!IsFloatReduction) {
4314 CommonFMF = BinOp->getFastMathFlags();
4315 IsFloatReduction = true;
4316 } else {
4317 CommonFMF &= BinOp->getFastMathFlags();
4318 }
4319 }
4320
4321 // Top-down demanded elements. For each chain value, track which lanes feed
4322 // the extracted lane 0 and which feed it more than once. Reverse postorder
4323 // visits every use before its value. A binop forwards its demand to both
4324 // operands and a shuffle follows its mask back to the source lane.
4325 struct Demand {
4326 APInt Lanes;
4327 APInt Duplicates;
4328 };
4329 DenseMap<Value *, Demand> Demands;
4330 auto DemandOf = [&](Value *V) -> Demand & {
4331 unsigned N = cast<FixedVectorType>(Val: V->getType())->getNumElements();
4332 Demand &D = Demands[V];
4333 if (D.Lanes.getBitWidth() != N)
4334 D.Lanes = D.Duplicates = APInt::getZero(numBits: N);
4335 return D;
4336 };
4337 DemandOf(VecOpEE).Lanes.setBit(0);
4338 for (Value *V : reverse(C&: Nodes)) {
4339 Demand DV = Demands.lookup(Val: V);
4340 if (DV.Lanes.isZero())
4341 continue;
4342 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V)) {
4343 ArrayRef<int> Mask = SVI->getShuffleMask();
4344 Demand &DS = DemandOf(SVI->getOperand(i_nocapture: 0));
4345 for (unsigned I = 0, E = Mask.size(); I != E; ++I) {
4346 // Skip lanes that are undemanded or map to poison.
4347 if (!DV.Lanes[I] || Mask[I] < 0 ||
4348 (unsigned)Mask[I] >= DS.Lanes.getBitWidth())
4349 continue;
4350 if (DS.Lanes[Mask[I]] || DV.Duplicates[I])
4351 DS.Duplicates.setBit(Mask[I]);
4352 DS.Lanes.setBit(Mask[I]);
4353 }
4354 } else {
4355 auto *U = cast<User>(Val: V);
4356 for (Value *Op : {U->getOperand(i: 0), U->getOperand(i: 1)}) {
4357 Demand &DOp = DemandOf(Op);
4358 // Lanes demanded through more than one path accumulate in Duplicates.
4359 DOp.Duplicates |= DV.Duplicates | (DOp.Lanes & DV.Lanes);
4360 DOp.Lanes |= DV.Lanes;
4361 }
4362 }
4363 }
4364
4365 // Reducing V replaces the entire chain, so every contribution to the result
4366 // must flow through V. Reject if anything above V reads outside the chain.
4367 auto CoversChain = [&](Value *V) {
4368 SmallVector<Value *, 8> Worklist(1, VecOpEE);
4369 SmallPtrSet<Value *, 8> Seen;
4370 Seen.insert(Ptr: VecOpEE);
4371 while (!Worklist.empty()) {
4372 auto *U = cast<Instruction>(Val: Worklist.pop_back_val());
4373 unsigned NumOps = isa<ShuffleVectorInst>(Val: U) ? 1 : 2;
4374 for (unsigned I = 0; I != NumOps; ++I) {
4375 Value *Op = U->getOperand(i: I);
4376 if (Op == V || !Seen.insert(Ptr: Op).second)
4377 continue;
4378 if (!Nodes.contains(key: Op))
4379 return false;
4380 Worklist.push_back(Elt: Op);
4381 }
4382 }
4383 return true;
4384 };
4385
4386 // Reduce a single cleanly demanded source if there is one, otherwise the
4387 // deepest intermediate that covers the chain.
4388 struct ReductionCut {
4389 Value *Src;
4390 APInt Elts;
4391 };
4392 std::optional<ReductionCut> Cut;
4393 for (Value *S : Sources) {
4394 auto It = Demands.find(Val: S);
4395 if (It == Demands.end() || It->second.Lanes.isZero())
4396 continue;
4397 if (!IsIdempotent && !It->second.Duplicates.isZero()) {
4398 Cut.reset();
4399 break;
4400 }
4401 if (!Cut) {
4402 Cut = ReductionCut{.Src: S, .Elts: It->second.Lanes};
4403 continue;
4404 }
4405 if (!isEquivBitcast(X: Cut->Src, Y: S)) {
4406 Cut.reset();
4407 break;
4408 }
4409 if (!IsIdempotent && !(Cut->Elts & It->second.Lanes).isZero()) {
4410 Cut.reset();
4411 break;
4412 }
4413 Cut->Elts |= It->second.Lanes;
4414 }
4415 if (!Cut) {
4416 for (Value *V : Nodes) {
4417 if (!isa<BinaryOperator>(Val: V) && !isa<MinMaxIntrinsic>(Val: V))
4418 continue;
4419 auto It = Demands.find(Val: V);
4420 if (It == Demands.end() || !It->second.Lanes.isAllOnes())
4421 continue;
4422 if (!IsIdempotent && !It->second.Duplicates.isZero())
4423 continue;
4424 if (!CoversChain(V))
4425 continue;
4426 Cut = ReductionCut{.Src: V, .Elts: It->second.Lanes};
4427 break;
4428 }
4429 }
4430 // Reducing one lane is just an extract and can refold forever.
4431 if (!Cut || Cut->Elts.popcount() < 2)
4432 return false;
4433
4434 Intrinsic::ID ReducedOp =
4435 (CommonCallOp ? getMinMaxReductionIntrinsicID(IID: *CommonCallOp)
4436 : getReductionForBinop(Opc: *CommonBinOp));
4437 if (!ReducedOp)
4438 return false;
4439
4440 InstructionCost OrigCost = 0;
4441 for (Value *V : Nodes)
4442 OrigCost += TTI.getInstructionCost(U: cast<Instruction>(Val: V), CostKind);
4443
4444 auto *SrcVT = cast<FixedVectorType>(Val: Cut->Src->getType());
4445 bool IsPartialReduction = !Cut->Elts.isAllOnes();
4446 FixedVectorType *ReduceVecTy =
4447 IsPartialReduction
4448 ? FixedVectorType::get(ElementType: FVT->getElementType(), NumElts: Cut->Elts.popcount())
4449 : SrcVT;
4450
4451 SmallVector<int> ExtractMask;
4452 InstructionCost NewCost = 0;
4453 if (IsPartialReduction) {
4454 for (unsigned I = 0, E = Cut->Elts.getBitWidth(); I != E; ++I)
4455 if (Cut->Elts[I])
4456 ExtractMask.push_back(Elt: I);
4457 unsigned SubIdx = 0, SubLen;
4458 auto SK = Cut->Elts.isShiftedMask(MaskIdx&: SubIdx, MaskLen&: SubLen)
4459 ? TargetTransformInfo::SK_ExtractSubvector
4460 : TargetTransformInfo::SK_PermuteSingleSrc;
4461 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: ReduceVecTy, SrcTy: SrcVT, CostKind, Mask: ExtractMask,
4462 Index: SubIdx, SubTp: ReduceVecTy);
4463 }
4464
4465 IntrinsicCostAttributes ICA(
4466 ReducedOp, ReduceVecTy->getElementType(),
4467 IsFloatReduction
4468 ? SmallVector<Type *, 2>{ReduceVecTy->getElementType(), ReduceVecTy}
4469 : SmallVector<Type *, 2>{ReduceVecTy},
4470 IsFloatReduction ? CommonFMF : FastMathFlags());
4471 NewCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
4472
4473 LLVM_DEBUG(dbgs() << "Found reduction shuffle chain: " << I << "\n OldCost : "
4474 << OrigCost << " vs NewCost: " << NewCost << "\n");
4475
4476 if (!OrigCost.isValid() || !NewCost.isValid())
4477 return false;
4478
4479 if (VecOpEE->hasOneUse() ? (NewCost > OrigCost) : (NewCost >= OrigCost))
4480 return false;
4481
4482 Value *ReduceInput = Cut->Src;
4483 if (IsPartialReduction)
4484 ReduceInput = Builder.CreateShuffleVector(V: Cut->Src, Mask: ExtractMask);
4485
4486 Value *ReducedResult;
4487 if (IsFloatReduction) {
4488 Value *Identity = ConstantExpr::getBinOpIdentity(
4489 Opcode: *CommonBinOp, Ty: ReduceVecTy->getElementType(), /*AllowRHSConstant=*/false,
4490 NSZ: CommonFMF.noSignedZeros());
4491 ReducedResult = Builder.CreateIntrinsic(ID: ReducedOp, OverloadTypes: {ReduceVecTy},
4492 Args: {Identity, ReduceInput}, FMFSource: CommonFMF);
4493 } else {
4494 ReducedResult =
4495 Builder.CreateIntrinsic(ID: ReducedOp, OverloadTypes: {ReduceVecTy}, Args: {ReduceInput});
4496 }
4497 replaceValue(Old&: I, New&: *ReducedResult);
4498
4499 return true;
4500}
4501
4502/// Determine if its more efficient to fold:
4503/// reduce(trunc(x)) -> trunc(reduce(x)).
4504/// reduce(sext(x)) -> sext(reduce(x)).
4505/// reduce(zext(x)) -> zext(reduce(x)).
4506bool VectorCombine::foldCastFromReductions(Instruction &I) {
4507 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
4508 if (!II)
4509 return false;
4510
4511 bool TruncOnly = false;
4512 Intrinsic::ID IID = II->getIntrinsicID();
4513 switch (IID) {
4514 case Intrinsic::vector_reduce_add:
4515 case Intrinsic::vector_reduce_mul:
4516 TruncOnly = true;
4517 break;
4518 case Intrinsic::vector_reduce_and:
4519 case Intrinsic::vector_reduce_or:
4520 case Intrinsic::vector_reduce_xor:
4521 break;
4522 default:
4523 return false;
4524 }
4525
4526 unsigned ReductionOpc = getArithmeticReductionInstruction(RdxID: IID);
4527 Value *ReductionSrc = I.getOperand(i: 0);
4528
4529 Value *Src;
4530 if (!match(V: ReductionSrc, P: m_OneUse(SubPattern: m_Trunc(Op: m_Value(V&: Src)))) &&
4531 (TruncOnly || !match(V: ReductionSrc, P: m_OneUse(SubPattern: m_ZExtOrSExt(Op: m_Value(V&: Src))))))
4532 return false;
4533
4534 auto CastOpc =
4535 (Instruction::CastOps)cast<Instruction>(Val: ReductionSrc)->getOpcode();
4536
4537 auto *SrcTy = cast<VectorType>(Val: Src->getType());
4538 auto *ReductionSrcTy = cast<VectorType>(Val: ReductionSrc->getType());
4539 Type *ResultTy = I.getType();
4540
4541 InstructionCost OldCost = TTI.getArithmeticReductionCost(
4542 Opcode: ReductionOpc, Ty: ReductionSrcTy, FMF: std::nullopt, CostKind);
4543 OldCost += TTI.getCastInstrCost(Opcode: CastOpc, Dst: ReductionSrcTy, Src: SrcTy,
4544 CCH: TTI::CastContextHint::None, CostKind,
4545 I: cast<CastInst>(Val: ReductionSrc));
4546 InstructionCost NewCost =
4547 TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: SrcTy, FMF: std::nullopt,
4548 CostKind) +
4549 TTI.getCastInstrCost(Opcode: CastOpc, Dst: ResultTy, Src: ReductionSrcTy->getScalarType(),
4550 CCH: TTI::CastContextHint::None, CostKind);
4551
4552 if (OldCost <= NewCost || !NewCost.isValid())
4553 return false;
4554
4555 Value *NewReduction = Builder.CreateIntrinsic(RetTy: SrcTy->getScalarType(),
4556 ID: II->getIntrinsicID(), Args: {Src});
4557 Value *NewCast = Builder.CreateCast(Op: CastOpc, V: NewReduction, DestTy: ResultTy);
4558 replaceValue(Old&: I, New&: *NewCast);
4559 return true;
4560}
4561
4562/// Fold:
4563/// icmp pred (reduce.{add,or,and,umax,umin}(signbit_extract(x))), C
4564/// into:
4565/// icmp sgt/slt (reduce.{or,umax,and,umin}(x)), -1/0
4566///
4567/// Sign-bit reductions produce values with known semantics:
4568/// - reduce.{or,umax}: 0 if no element is negative, 1 if any is
4569/// - reduce.{and,umin}: 1 if all elements are negative, 0 if any isn't
4570/// - reduce.add: count of negative elements (0 to NumElts)
4571///
4572/// Both lshr and ashr are supported:
4573/// - lshr produces 0 or 1, so reduce.add range is [0, N]
4574/// - ashr produces 0 or -1, so reduce.add range is [-N, 0]
4575///
4576/// The fold generalizes to multiple source vectors combined with the same
4577/// operation as the reduction. For example:
4578/// reduce.or(or(shr A, shr B)) conceptually extends the vector
4579/// For reduce.add, this changes the count to M*N where M is the number of
4580/// source vectors.
4581///
4582/// We transform to a direct sign check on the original vector using
4583/// reduce.{or,umax} or reduce.{and,umin}.
4584///
4585/// In spirit, it's similar to foldSignBitCheck in InstCombine.
4586bool VectorCombine::foldSignBitReductionCmp(Instruction &I) {
4587 CmpPredicate Pred;
4588 IntrinsicInst *ReduceOp;
4589 const APInt *CmpVal;
4590 if (!match(V: &I,
4591 P: m_ICmp(Pred, L: m_OneUse(SubPattern: m_AnyIntrinsic(I&: ReduceOp)), R: m_APInt(Res&: CmpVal))))
4592 return false;
4593
4594 Intrinsic::ID OrigIID = ReduceOp->getIntrinsicID();
4595 switch (OrigIID) {
4596 case Intrinsic::vector_reduce_or:
4597 case Intrinsic::vector_reduce_umax:
4598 case Intrinsic::vector_reduce_and:
4599 case Intrinsic::vector_reduce_umin:
4600 case Intrinsic::vector_reduce_add:
4601 break;
4602 default:
4603 return false;
4604 }
4605
4606 Value *ReductionSrc = ReduceOp->getArgOperand(i: 0);
4607 auto *VecTy = dyn_cast<FixedVectorType>(Val: ReductionSrc->getType());
4608 if (!VecTy)
4609 return false;
4610
4611 unsigned BitWidth = VecTy->getScalarSizeInBits();
4612 if (BitWidth == 1)
4613 return false;
4614
4615 unsigned NumElts = VecTy->getNumElements();
4616
4617 // Determine the expected tree opcode for multi-vector patterns.
4618 // The tree opcode must match the reduction's underlying operation.
4619 //
4620 // TODO: for pairs of equivalent operators, we should match both,
4621 // not only the most common.
4622 Instruction::BinaryOps TreeOpcode;
4623 switch (OrigIID) {
4624 case Intrinsic::vector_reduce_or:
4625 case Intrinsic::vector_reduce_umax:
4626 TreeOpcode = Instruction::Or;
4627 break;
4628 case Intrinsic::vector_reduce_and:
4629 case Intrinsic::vector_reduce_umin:
4630 TreeOpcode = Instruction::And;
4631 break;
4632 case Intrinsic::vector_reduce_add:
4633 TreeOpcode = Instruction::Add;
4634 break;
4635 default:
4636 llvm_unreachable("Unexpected intrinsic");
4637 }
4638
4639 // Collect sign-bit extraction leaves from an associative tree of TreeOpcode.
4640 // The tree conceptually extends the vector being reduced.
4641 SmallVector<Value *, 8> Worklist;
4642 SmallVector<Value *, 8> Sources; // Original vectors (X in shr X, BW-1)
4643 Worklist.push_back(Elt: ReductionSrc);
4644 std::optional<bool> IsAShr;
4645 constexpr unsigned MaxSources = 8;
4646
4647 // Calculate old cost: all shifts + tree ops + reduction
4648 InstructionCost OldCost = TTI.getInstructionCost(U: ReduceOp, CostKind);
4649
4650 while (!Worklist.empty() && Worklist.size() <= MaxSources &&
4651 Sources.size() <= MaxSources) {
4652 Value *V = Worklist.pop_back_val();
4653
4654 // Try to match sign-bit extraction: shr X, (bitwidth-1)
4655 Value *X;
4656 if (match(V, P: m_OneUse(SubPattern: m_Shr(L: m_Value(V&: X), R: m_SpecificInt(V: BitWidth - 1))))) {
4657 auto *Shr = cast<Instruction>(Val: V);
4658
4659 // All shifts must be the same type (all lshr or all ashr)
4660 bool ThisIsAShr = Shr->getOpcode() == Instruction::AShr;
4661 if (!IsAShr)
4662 IsAShr = ThisIsAShr;
4663 else if (*IsAShr != ThisIsAShr)
4664 return false;
4665
4666 Sources.push_back(Elt: X);
4667
4668 // As part of the fold, we remove all of the shifts, so we need to keep
4669 // track of their costs.
4670 OldCost += TTI.getInstructionCost(U: Shr, CostKind);
4671
4672 continue;
4673 }
4674
4675 // Try to extend through a tree node of the expected opcode
4676 Value *A, *B;
4677 if (!match(V, P: m_OneUse(SubPattern: m_BinOp(Opcode: TreeOpcode, L: m_Value(V&: A), R: m_Value(V&: B)))))
4678 return false;
4679
4680 // We are potentially replacing these operations as well, so we add them
4681 // to the costs.
4682 OldCost += TTI.getInstructionCost(U: cast<Instruction>(Val: V), CostKind);
4683
4684 Worklist.push_back(Elt: A);
4685 Worklist.push_back(Elt: B);
4686 }
4687
4688 // Must have at least one source and not exceed limit
4689 if (Sources.empty() || Sources.size() > MaxSources ||
4690 Worklist.size() > MaxSources || !IsAShr)
4691 return false;
4692
4693 unsigned NumSources = Sources.size();
4694
4695 // For reduce.add, the total count must fit as a signed integer.
4696 // Range is [0, M*N] for lshr or [-M*N, 0] for ashr.
4697 if (OrigIID == Intrinsic::vector_reduce_add &&
4698 !isIntN(N: BitWidth, x: NumSources * NumElts))
4699 return false;
4700
4701 // Compute the boundary value when all elements are negative:
4702 // - Per-element contribution: 1 for lshr, -1 for ashr
4703 // - For add: M*N (total elements across all sources); for others: just 1
4704 unsigned Count =
4705 (OrigIID == Intrinsic::vector_reduce_add) ? NumSources * NumElts : 1;
4706 APInt NegativeVal(CmpVal->getBitWidth(), Count);
4707 if (*IsAShr)
4708 NegativeVal.negate();
4709
4710 // Range is [min(0, AllNegVal), max(0, AllNegVal)]
4711 APInt Zero = APInt::getZero(numBits: CmpVal->getBitWidth());
4712 APInt RangeLow = APIntOps::smin(A: Zero, B: NegativeVal);
4713 APInt RangeHigh = APIntOps::smax(A: Zero, B: NegativeVal);
4714
4715 // Determine comparison semantics:
4716 // - IsEq: true for equality test, false for inequality
4717 // - TestsNegative: true if testing against AllNegVal, false for zero
4718 //
4719 // In addition to EQ/NE against 0 or AllNegVal, we support inequalities
4720 // that fold to boundary tests given the narrow value range:
4721 // < RangeHigh -> != RangeHigh
4722 // > RangeHigh-1 -> == RangeHigh
4723 // > RangeLow -> != RangeLow
4724 // < RangeLow+1 -> == RangeLow
4725 //
4726 // For inequalities, we work with signed predicates only. Unsigned predicates
4727 // are canonicalized to signed when the range is non-negative (where they are
4728 // equivalent). When the range includes negative values, unsigned predicates
4729 // would have different semantics due to wrap-around, so we reject them.
4730 if (!ICmpInst::isEquality(P: Pred) && !ICmpInst::isSigned(Pred)) {
4731 if (RangeLow.isNegative())
4732 return false;
4733 Pred = ICmpInst::getSignedPredicate(Pred);
4734 }
4735
4736 bool IsEq;
4737 bool TestsNegative;
4738 if (ICmpInst::isEquality(P: Pred)) {
4739 if (CmpVal->isZero()) {
4740 TestsNegative = false;
4741 } else if (*CmpVal == NegativeVal) {
4742 TestsNegative = true;
4743 } else {
4744 return false;
4745 }
4746 IsEq = Pred == ICmpInst::ICMP_EQ;
4747 } else if (Pred == ICmpInst::ICMP_SLT && *CmpVal == RangeHigh) {
4748 IsEq = false;
4749 TestsNegative = (RangeHigh == NegativeVal);
4750 } else if (Pred == ICmpInst::ICMP_SGT && *CmpVal == RangeHigh - 1) {
4751 IsEq = true;
4752 TestsNegative = (RangeHigh == NegativeVal);
4753 } else if (Pred == ICmpInst::ICMP_SGT && *CmpVal == RangeLow) {
4754 IsEq = false;
4755 TestsNegative = (RangeLow == NegativeVal);
4756 } else if (Pred == ICmpInst::ICMP_SLT && *CmpVal == RangeLow + 1) {
4757 IsEq = true;
4758 TestsNegative = (RangeLow == NegativeVal);
4759 } else {
4760 return false;
4761 }
4762
4763 // For this fold we support four types of checks:
4764 //
4765 // 1. All lanes are negative - AllNeg
4766 // 2. All lanes are non-negative - AllNonNeg
4767 // 3. At least one negative lane - AnyNeg
4768 // 4. At least one non-negative lane - AnyNonNeg
4769 //
4770 // For each case, we can generate the following code:
4771 //
4772 // 1. AllNeg - reduce.and/umin(X) < 0
4773 // 2. AllNonNeg - reduce.or/umax(X) > -1
4774 // 3. AnyNeg - reduce.or/umax(X) < 0
4775 // 4. AnyNonNeg - reduce.and/umin(X) > -1
4776 //
4777 // The table below shows the aggregation of all supported cases
4778 // using these four cases.
4779 //
4780 // Reduction | == 0 | != 0 | == MAX | != MAX
4781 // ------------+-----------+-----------+-----------+-----------
4782 // or/umax | AllNonNeg | AnyNeg | AnyNeg | AllNonNeg
4783 // and/umin | AnyNonNeg | AllNeg | AllNeg | AnyNonNeg
4784 // add | AllNonNeg | AnyNeg | AllNeg | AnyNonNeg
4785 //
4786 // NOTE: MAX = 1 for or/and/umax/umin, and the vector size N for add
4787 //
4788 // For easier codegen and check inversion, we use the following encoding:
4789 //
4790 // 1. Bit-3 === requires or/umax (1) or and/umin (0) check
4791 // 2. Bit-2 === checks < 0 (1) or > -1 (0)
4792 // 3. Bit-1 === universal (1) or existential (0) check
4793 //
4794 // AnyNeg = 0b110: uses or/umax, checks negative, any-check
4795 // AllNonNeg = 0b101: uses or/umax, checks non-neg, all-check
4796 // AnyNonNeg = 0b000: uses and/umin, checks non-neg, any-check
4797 // AllNeg = 0b011: uses and/umin, checks negative, all-check
4798 //
4799 // XOR with 0b011 inverts the check (swaps all/any and neg/non-neg).
4800 //
4801 enum CheckKind : unsigned {
4802 AnyNonNeg = 0b000,
4803 AllNeg = 0b011,
4804 AllNonNeg = 0b101,
4805 AnyNeg = 0b110,
4806 };
4807 // Return true if we fold this check into or/umax and false for and/umin
4808 auto RequiresOr = [](CheckKind C) -> bool { return C & 0b100; };
4809 // Return true if we should check if result is negative and false otherwise
4810 auto IsNegativeCheck = [](CheckKind C) -> bool { return C & 0b010; };
4811 // Logically invert the check
4812 auto Invert = [](CheckKind C) { return CheckKind(C ^ 0b011); };
4813
4814 CheckKind Base;
4815 switch (OrigIID) {
4816 case Intrinsic::vector_reduce_or:
4817 case Intrinsic::vector_reduce_umax:
4818 Base = TestsNegative ? AnyNeg : AllNonNeg;
4819 break;
4820 case Intrinsic::vector_reduce_and:
4821 case Intrinsic::vector_reduce_umin:
4822 Base = TestsNegative ? AllNeg : AnyNonNeg;
4823 break;
4824 case Intrinsic::vector_reduce_add:
4825 Base = TestsNegative ? AllNeg : AllNonNeg;
4826 break;
4827 default:
4828 llvm_unreachable("Unexpected intrinsic");
4829 }
4830
4831 CheckKind Check = IsEq ? Base : Invert(Base);
4832
4833 auto PickCheaper = [&](Intrinsic::ID Arith, Intrinsic::ID MinMax) {
4834 InstructionCost ArithCost =
4835 TTI.getArithmeticReductionCost(Opcode: getArithmeticReductionInstruction(RdxID: Arith),
4836 Ty: VecTy, FMF: std::nullopt, CostKind);
4837 InstructionCost MinMaxCost =
4838 TTI.getMinMaxReductionCost(IID: getMinMaxReductionIntrinsicOp(RdxID: MinMax), Ty: VecTy,
4839 FMF: FastMathFlags(), CostKind);
4840 return ArithCost <= MinMaxCost ? std::make_pair(x&: Arith, y&: ArithCost)
4841 : std::make_pair(x&: MinMax, y&: MinMaxCost);
4842 };
4843
4844 // Choose output reduction based on encoding's MSB
4845 auto [NewIID, NewCost] = RequiresOr(Check)
4846 ? PickCheaper(Intrinsic::vector_reduce_or,
4847 Intrinsic::vector_reduce_umax)
4848 : PickCheaper(Intrinsic::vector_reduce_and,
4849 Intrinsic::vector_reduce_umin);
4850
4851 // Add cost of combining multiple sources with or/and
4852 if (NumSources > 1) {
4853 unsigned CombineOpc =
4854 RequiresOr(Check) ? Instruction::Or : Instruction::And;
4855 NewCost += TTI.getArithmeticInstrCost(Opcode: CombineOpc, Ty: VecTy, CostKind) *
4856 (NumSources - 1);
4857 }
4858
4859 LLVM_DEBUG(dbgs() << "Found sign-bit reduction cmp: " << I << "\n OldCost: "
4860 << OldCost << " vs NewCost: " << NewCost << "\n");
4861
4862 if (NewCost > OldCost)
4863 return false;
4864
4865 // Generate the combined input and reduction
4866 Builder.SetInsertPoint(&I);
4867 Type *ScalarTy = VecTy->getScalarType();
4868
4869 Value *Input;
4870 if (NumSources == 1) {
4871 Input = Sources[0];
4872 } else {
4873 // Combine sources with or/and based on check type
4874 Input = RequiresOr(Check) ? Builder.CreateOr(Ops: Sources)
4875 : Builder.CreateAnd(Ops: Sources);
4876 }
4877
4878 Value *NewReduce = Builder.CreateIntrinsic(RetTy: ScalarTy, ID: NewIID, Args: {Input});
4879 Value *NewCmp = IsNegativeCheck(Check) ? Builder.CreateIsNeg(Arg: NewReduce)
4880 : Builder.CreateIsNotNeg(Arg: NewReduce);
4881 replaceValue(Old&: I, New&: *NewCmp);
4882 return true;
4883}
4884
4885/// Fold a zero test of reduce.or or reduce.umax into a boolean reduction.
4886///
4887/// Vectorization may produce IR that compares the result of a scalar reduction
4888/// with zero. Depending on the target, lowering a reduction and a scalar
4889/// comparison separately can cost more than reducing lane-wise comparison
4890/// results. This fold creates the latter form only when it is not costlier.
4891///
4892/// Before:
4893/// %r = call iT @llvm.vector.reduce.or.vNiT(<N x iT> %x)
4894/// %cmp = icmp ne iT %r, 0
4895///
4896/// After:
4897/// %lane.cmp = icmp ne <N x iT> %x, zeroinitializer
4898/// %cmp = call i1 @llvm.vector.reduce.or.vNi1(<N x i1> %lane.cmp)
4899///
4900/// `reduce.or` and `reduce.umax` are non-zero when at least one lane is
4901/// non-zero. Therefore, `icmp ne` uses the existential `reduce.or` test.
4902/// Conversely, `icmp eq` must check that every lane is zero, so it uses the
4903/// universal `reduce.and` test.
4904///
4905/// Before:
4906/// %r = call iT @llvm.vector.reduce.umax.vNiT(<N x iT> %x)
4907/// %cmp = icmp eq iT %r, 0
4908///
4909/// After:
4910/// %lane.cmp = icmp eq <N x iT> %x, zeroinitializer
4911/// %cmp = call i1 @llvm.vector.reduce.and.vNi1(<N x i1> %lane.cmp)
4912bool VectorCombine::foldReductionZeroTest(Instruction &I) {
4913 CmpPredicate Pred;
4914 Value *Op;
4915
4916 if (!match(V: &I, P: m_c_ICmp(Pred, L: m_Value(V&: Op), R: m_Zero())) ||
4917 !ICmpInst::isEquality(P: Pred))
4918 return false;
4919
4920 auto *II = dyn_cast<IntrinsicInst>(Val: Op);
4921 if (!II || !II->hasOneUse())
4922 return false;
4923
4924 auto ReduceID = II->getIntrinsicID();
4925 if (ReduceID != Intrinsic::vector_reduce_or &&
4926 ReduceID != Intrinsic::vector_reduce_umax)
4927 return false;
4928
4929 Value *Vec = II->getArgOperand(i: 0);
4930 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
4931 if (!VecTy || !VecTy->getElementType()->isIntegerTy())
4932 return false;
4933
4934 // Map the scalar zero test to an any-lane or all-lane boolean reduction.
4935 Intrinsic::ID NewIID = (Pred == ICmpInst::ICMP_NE)
4936 ? Intrinsic::vector_reduce_or
4937 : Intrinsic::vector_reduce_and;
4938
4939 // This is not an unconditional canonicalization: compare the cost of the
4940 // original scalar reduction and compare with the vector compare and i1
4941 // reduction replacement for both reduce.or and reduce.umax.
4942 InstructionCost OldCost = TTI.getInstructionCost(U: II, CostKind) +
4943 TTI.getInstructionCost(U: &I, CostKind);
4944
4945 auto *CmpTy = cast<VectorType>(Val: CmpInst::makeCmpResultType(opnd_type: VecTy));
4946 InstructionCost NewCost =
4947 TTI.getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: VecTy, CondTy: CmpTy, VecPred: Pred, CostKind);
4948 NewCost += TTI.getArithmeticReductionCost(
4949 Opcode: getArithmeticReductionInstruction(RdxID: NewIID), Ty: CmpTy, FMF: std::nullopt, CostKind);
4950
4951 LLVM_DEBUG(dbgs() << "Found a reduction zero test: " << I << "\n OldCost: "
4952 << OldCost << " vs NewCost: " << NewCost << "\n");
4953
4954 if (!OldCost.isValid() || !NewCost.isValid() || NewCost > OldCost)
4955 return false;
4956
4957 Builder.SetInsertPoint(&I);
4958 Value *NewCmp = Builder.CreateICmp(P: Pred, LHS: Vec, RHS: Constant::getNullValue(Ty: VecTy));
4959 Value *NewReduce = Builder.CreateIntrinsic(ID: NewIID, OverloadTypes: {CmpTy}, Args: {NewCmp});
4960 replaceValue(Old&: I, New&: *NewReduce);
4961 return true;
4962}
4963
4964/// vector.reduce.OP f(X_i) == 0 -> vector.reduce.OP X_i == 0
4965///
4966/// We can prove it for cases when:
4967///
4968/// 1. OP X_i == 0 <=> \forall i \in [1, N] X_i == 0
4969/// 1'. OP X_i == 0 <=> \exists j \in [1, N] X_j == 0
4970/// 2. f(x) == 0 <=> x == 0
4971///
4972/// From 1 and 2 (or 1' and 2), we can infer that
4973///
4974/// OP f(X_i) == 0 <=> OP X_i == 0.
4975///
4976/// (1)
4977/// OP f(X_i) == 0 <=> \forall i \in [1, N] f(X_i) == 0
4978/// (2)
4979/// <=> \forall i \in [1, N] X_i == 0
4980/// (1)
4981/// <=> OP(X_i) == 0
4982///
4983/// For some of the OP's and f's, we need to have domain constraints on X
4984/// to ensure properties 1 (or 1') and 2.
4985bool VectorCombine::foldICmpEqZeroVectorReduce(Instruction &I) {
4986 CmpPredicate Pred;
4987 Value *Op;
4988 if (!match(V: &I, P: m_ICmp(Pred, L: m_Value(V&: Op), R: m_Zero())) ||
4989 !ICmpInst::isEquality(P: Pred))
4990 return false;
4991
4992 auto *II = dyn_cast<IntrinsicInst>(Val: Op);
4993 if (!II)
4994 return false;
4995
4996 switch (II->getIntrinsicID()) {
4997 case Intrinsic::vector_reduce_add:
4998 case Intrinsic::vector_reduce_or:
4999 case Intrinsic::vector_reduce_umin:
5000 case Intrinsic::vector_reduce_umax:
5001 case Intrinsic::vector_reduce_smin:
5002 case Intrinsic::vector_reduce_smax:
5003 break;
5004 default:
5005 return false;
5006 }
5007
5008 Value *InnerOp = II->getArgOperand(i: 0);
5009
5010 // TODO: fixed vector type might be too restrictive
5011 if (!II->hasOneUse() || !isa<FixedVectorType>(Val: InnerOp->getType()))
5012 return false;
5013
5014 Value *X = nullptr;
5015
5016 // Check for zero-preserving operations where f(x) = 0 <=> x = 0
5017 //
5018 // 1. f(x) = shl nuw x, y for arbitrary y
5019 // 2. f(x) = mul nuw x, c for defined c != 0
5020 // 3. f(x) = zext x
5021 // 4. f(x) = sext x
5022 // 5. f(x) = neg x
5023 //
5024 if (!(match(V: InnerOp, P: m_NUWShl(L: m_Value(V&: X), R: m_Value())) || // Case 1
5025 match(V: InnerOp, P: m_NUWMul(L: m_Value(V&: X), R: m_NonZeroInt())) || // Case 2
5026 match(V: InnerOp, P: m_ZExt(Op: m_Value(V&: X))) || // Case 3
5027 match(V: InnerOp, P: m_SExt(Op: m_Value(V&: X))) || // Case 4
5028 match(V: InnerOp, P: m_Neg(V: m_Value(V&: X))) // Case 5
5029 ))
5030 return false;
5031
5032 SimplifyQuery S = SQ.getWithInstruction(I: &I);
5033 auto *XTy = cast<FixedVectorType>(Val: X->getType());
5034
5035 // Check for domain constraints for all supported reductions.
5036 //
5037 // a. OR X_i - has property 1 for every X
5038 // b. UMAX X_i - has property 1 for every X
5039 // c. UMIN X_i - has property 1' for every X
5040 // d. SMAX X_i - has property 1 for X >= 0
5041 // e. SMIN X_i - has property 1' for X >= 0
5042 // f. ADD X_i - has property 1 for X >= 0 && ADD X_i doesn't sign wrap
5043 //
5044 // In order for the proof to work, we need 1 (or 1') to be true for both
5045 // OP f(X_i) and OP X_i and that's why below we check constraints twice.
5046 //
5047 // NOTE: ADD X_i holds property 1 for a mirror case as well, i.e. when
5048 // X <= 0 && ADD X_i doesn't sign wrap. However, due to the nature
5049 // of known bits, we can't reasonably hold knowledge of "either 0
5050 // or negative".
5051 switch (II->getIntrinsicID()) {
5052 case Intrinsic::vector_reduce_add: {
5053 // We need to check that both X_i and f(X_i) have enough leading
5054 // zeros to not overflow.
5055 KnownBits KnownX = computeKnownBits(V: X, Q: S);
5056 KnownBits KnownFX = computeKnownBits(V: InnerOp, Q: S);
5057 unsigned NumElems = XTy->getNumElements();
5058 // Adding N elements loses at most ceil(log2(N)) leading bits.
5059 unsigned LostBits = Log2_32_Ceil(Value: NumElems);
5060 unsigned LeadingZerosX = KnownX.countMinLeadingZeros();
5061 unsigned LeadingZerosFX = KnownFX.countMinLeadingZeros();
5062 // Need at least one leading zero left after summation to ensure no overflow
5063 if (LeadingZerosX <= LostBits || LeadingZerosFX <= LostBits)
5064 return false;
5065
5066 // We are not checking whether X or f(X) are positive explicitly because
5067 // we implicitly checked for it when we checked if both cases have enough
5068 // leading zeros to not wrap addition.
5069 break;
5070 }
5071 case Intrinsic::vector_reduce_smin:
5072 case Intrinsic::vector_reduce_smax:
5073 // Check whether X >= 0 and f(X) >= 0
5074 if (!isKnownNonNegative(V: InnerOp, SQ: S) || !isKnownNonNegative(V: X, SQ: S))
5075 return false;
5076
5077 break;
5078 default:
5079 break;
5080 };
5081
5082 LLVM_DEBUG(dbgs() << "Found a reduction to 0 comparison with removable op: "
5083 << *II << "\n");
5084
5085 // For zext/sext, check if the transform is profitable using cost model.
5086 // For other operations (shl, mul, neg), we're removing an instruction
5087 // while keeping the same reduction type, so it's always profitable.
5088 if (isa<ZExtInst>(Val: InnerOp) || isa<SExtInst>(Val: InnerOp)) {
5089 auto *FXTy = cast<FixedVectorType>(Val: InnerOp->getType());
5090 Intrinsic::ID IID = II->getIntrinsicID();
5091
5092 InstructionCost ExtCost = TTI.getCastInstrCost(
5093 Opcode: cast<CastInst>(Val: InnerOp)->getOpcode(), Dst: FXTy, Src: XTy,
5094 CCH: TTI::CastContextHint::None, CostKind, I: cast<CastInst>(Val: InnerOp));
5095
5096 InstructionCost OldReduceCost, NewReduceCost;
5097 switch (IID) {
5098 case Intrinsic::vector_reduce_add:
5099 case Intrinsic::vector_reduce_or:
5100 OldReduceCost = TTI.getArithmeticReductionCost(
5101 Opcode: getArithmeticReductionInstruction(RdxID: IID), Ty: FXTy, FMF: std::nullopt, CostKind);
5102 NewReduceCost = TTI.getArithmeticReductionCost(
5103 Opcode: getArithmeticReductionInstruction(RdxID: IID), Ty: XTy, FMF: std::nullopt, CostKind);
5104 break;
5105 case Intrinsic::vector_reduce_umin:
5106 case Intrinsic::vector_reduce_umax:
5107 case Intrinsic::vector_reduce_smin:
5108 case Intrinsic::vector_reduce_smax:
5109 OldReduceCost = TTI.getMinMaxReductionCost(
5110 IID: getMinMaxReductionIntrinsicOp(RdxID: IID), Ty: FXTy, FMF: FastMathFlags(), CostKind);
5111 NewReduceCost = TTI.getMinMaxReductionCost(
5112 IID: getMinMaxReductionIntrinsicOp(RdxID: IID), Ty: XTy, FMF: FastMathFlags(), CostKind);
5113 break;
5114 default:
5115 llvm_unreachable("Unexpected reduction");
5116 }
5117
5118 InstructionCost OldCost = OldReduceCost + ExtCost;
5119 InstructionCost NewCost =
5120 NewReduceCost + (InnerOp->hasOneUse() ? 0 : ExtCost);
5121
5122 LLVM_DEBUG(dbgs() << "Found a removable extension before reduction: "
5123 << *InnerOp << "\n OldCost: " << OldCost
5124 << " vs NewCost: " << NewCost << "\n");
5125
5126 // We consider transformation to still be potentially beneficial even
5127 // when the costs are the same because we might remove a use from f(X)
5128 // and unlock other optimizations. Equal costs would just mean that we
5129 // didn't make it worse in the worst case.
5130 if (NewCost > OldCost)
5131 return false;
5132 }
5133
5134 // Since we support zext and sext as f, we might change the scalar type
5135 // of the intrinsic.
5136 Type *Ty = XTy->getScalarType();
5137 Value *NewReduce = Builder.CreateIntrinsic(RetTy: Ty, ID: II->getIntrinsicID(), Args: {X});
5138 Value *NewCmp =
5139 Builder.CreateICmp(P: Pred, LHS: NewReduce, RHS: ConstantInt::getNullValue(Ty));
5140 replaceValue(Old&: I, New&: *NewCmp);
5141 return true;
5142}
5143
5144/// Fold comparisons of reduce.or/reduce.and with reduce.umax/reduce.umin
5145/// based on cost, preserving the comparison semantics.
5146///
5147/// We use two fundamental properties for each pair:
5148///
5149/// 1. or(X) == 0 <=> umax(X) == 0
5150/// 2. or(X) == 1 <=> umax(X) == 1
5151/// 3. sign(or(X)) == sign(umax(X))
5152///
5153/// 1. and(X) == -1 <=> umin(X) == -1
5154/// 2. and(X) == -2 <=> umin(X) == -2
5155/// 3. sign(and(X)) == sign(umin(X))
5156///
5157/// From these we can infer the following transformations:
5158/// a. or(X) ==/!= 0 <-> umax(X) ==/!= 0
5159/// b. or(X) s< 0 <-> umax(X) s< 0
5160/// c. or(X) s> -1 <-> umax(X) s> -1
5161/// d. or(X) s< 1 <-> umax(X) s< 1
5162/// e. or(X) ==/!= 1 <-> umax(X) ==/!= 1
5163/// f. or(X) s< 2 <-> umax(X) s< 2
5164/// g. and(X) ==/!= -1 <-> umin(X) ==/!= -1
5165/// h. and(X) s< 0 <-> umin(X) s< 0
5166/// i. and(X) s> -1 <-> umin(X) s> -1
5167/// j. and(X) s> -2 <-> umin(X) s> -2
5168/// k. and(X) ==/!= -2 <-> umin(X) ==/!= -2
5169/// l. and(X) s> -3 <-> umin(X) s> -3
5170///
5171bool VectorCombine::foldEquivalentReductionCmp(Instruction &I) {
5172 CmpPredicate Pred;
5173 Value *ReduceOp;
5174 const APInt *CmpVal;
5175 if (!match(V: &I, P: m_ICmp(Pred, L: m_Value(V&: ReduceOp), R: m_APInt(Res&: CmpVal))))
5176 return false;
5177
5178 auto *II = dyn_cast<IntrinsicInst>(Val: ReduceOp);
5179 if (!II || !II->hasOneUse())
5180 return false;
5181
5182 const auto IsValidOrUmaxCmp = [&]() {
5183 // or === umax for i1
5184 if (CmpVal->getBitWidth() == 1)
5185 return true;
5186
5187 // Cases a and e
5188 bool IsEquality =
5189 (CmpVal->isZero() || CmpVal->isOne()) && ICmpInst::isEquality(P: Pred);
5190 // Case c
5191 bool IsPositive = CmpVal->isAllOnes() && Pred == ICmpInst::ICMP_SGT;
5192 // Cases b, d, and f
5193 bool IsNegative = (CmpVal->isZero() || CmpVal->isOne() || *CmpVal == 2) &&
5194 Pred == ICmpInst::ICMP_SLT;
5195 return IsEquality || IsPositive || IsNegative;
5196 };
5197
5198 const auto IsValidAndUminCmp = [&]() {
5199 // and === umin for i1
5200 if (CmpVal->getBitWidth() == 1)
5201 return true;
5202
5203 const auto LeadingOnes = CmpVal->countl_one();
5204
5205 // Cases g and k
5206 bool IsEquality =
5207 (CmpVal->isAllOnes() || LeadingOnes + 1 == CmpVal->getBitWidth()) &&
5208 ICmpInst::isEquality(P: Pred);
5209 // Case h
5210 bool IsNegative = CmpVal->isZero() && Pred == ICmpInst::ICMP_SLT;
5211 // Cases i, j, and l
5212 bool IsPositive =
5213 // if the number has at least N - 2 leading ones
5214 // and the two LSBs are:
5215 // - 1 x 1 -> -1
5216 // - 1 x 0 -> -2
5217 // - 0 x 1 -> -3
5218 LeadingOnes + 2 >= CmpVal->getBitWidth() &&
5219 ((*CmpVal)[0] || (*CmpVal)[1]) && Pred == ICmpInst::ICMP_SGT;
5220 return IsEquality || IsNegative || IsPositive;
5221 };
5222
5223 Intrinsic::ID OriginalIID = II->getIntrinsicID();
5224 Intrinsic::ID AlternativeIID;
5225
5226 // Check if this is a valid comparison pattern and determine the alternate
5227 // reduction intrinsic.
5228 switch (OriginalIID) {
5229 case Intrinsic::vector_reduce_or:
5230 if (!IsValidOrUmaxCmp())
5231 return false;
5232 AlternativeIID = Intrinsic::vector_reduce_umax;
5233 break;
5234 case Intrinsic::vector_reduce_umax:
5235 if (!IsValidOrUmaxCmp())
5236 return false;
5237 AlternativeIID = Intrinsic::vector_reduce_or;
5238 break;
5239 case Intrinsic::vector_reduce_and:
5240 if (!IsValidAndUminCmp())
5241 return false;
5242 AlternativeIID = Intrinsic::vector_reduce_umin;
5243 break;
5244 case Intrinsic::vector_reduce_umin:
5245 if (!IsValidAndUminCmp())
5246 return false;
5247 AlternativeIID = Intrinsic::vector_reduce_and;
5248 break;
5249 default:
5250 return false;
5251 }
5252
5253 Value *X = II->getArgOperand(i: 0);
5254 auto *VecTy = dyn_cast<FixedVectorType>(Val: X->getType());
5255 if (!VecTy)
5256 return false;
5257
5258 const auto GetReductionCost = [&](Intrinsic::ID IID) -> InstructionCost {
5259 unsigned ReductionOpc = getArithmeticReductionInstruction(RdxID: IID);
5260 if (ReductionOpc != Instruction::ICmp)
5261 return TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: VecTy, FMF: std::nullopt,
5262 CostKind);
5263 return TTI.getMinMaxReductionCost(IID: getMinMaxReductionIntrinsicOp(RdxID: IID), Ty: VecTy,
5264 FMF: FastMathFlags(), CostKind);
5265 };
5266
5267 InstructionCost OrigCost = GetReductionCost(OriginalIID);
5268 InstructionCost AltCost = GetReductionCost(AlternativeIID);
5269
5270 LLVM_DEBUG(dbgs() << "Found equivalent reduction cmp: " << I
5271 << "\n OrigCost: " << OrigCost
5272 << " vs AltCost: " << AltCost << "\n");
5273
5274 if (AltCost >= OrigCost)
5275 return false;
5276
5277 Builder.SetInsertPoint(&I);
5278 Type *ScalarTy = VecTy->getScalarType();
5279 Value *NewReduce = Builder.CreateIntrinsic(RetTy: ScalarTy, ID: AlternativeIID, Args: {X});
5280 Value *NewCmp =
5281 Builder.CreateICmp(P: Pred, LHS: NewReduce, RHS: ConstantInt::get(Ty: ScalarTy, V: *CmpVal));
5282
5283 replaceValue(Old&: I, New&: *NewCmp);
5284 return true;
5285}
5286
5287/// Used by foldReduceAddCmpZero to check if we can prove that a value is
5288/// non-positive.
5289/// KnownBits cannot see sext <? x i1> as non-positive: each top bit equals a
5290/// single unknown input bit, which a per-bit lattice cannot track. The fold's
5291/// target shape is popcount-style sums of <N x i1> valid/invalid masks (e.g.
5292/// ray-intersection hits) tested for any-hit.
5293/// Previous attempts to approximate the known bits of such expressions were
5294/// using a fully recursive value tracking approach to infer a constant range
5295/// but ultimately turned to be too expensive in compile time.
5296static bool isKnownNonPositive(const Value *V, const SimplifyQuery &SQ,
5297 unsigned Depth = 0) {
5298 constexpr unsigned MaxLocalDepth = 2;
5299 if (Depth > MaxLocalDepth)
5300 return false;
5301
5302 auto NumSignBits = [&](const Value *X) {
5303 return ComputeNumSignBits(Op: X, DL: SQ.DL, AC: SQ.AC, CxtI: SQ.CxtI, DT: SQ.DT);
5304 };
5305 if (NumSignBits(V) == V->getType()->getScalarSizeInBits())
5306 return true;
5307
5308 Value *A, *B;
5309 if (match(V, P: m_Add(L: m_Value(V&: A), R: m_Value(V&: B))))
5310 return NumSignBits(A) >= 2 && NumSignBits(B) >= 2 &&
5311 isKnownNonPositive(V: A, SQ, Depth: Depth + 1) &&
5312 isKnownNonPositive(V: B, SQ, Depth: Depth + 1);
5313
5314 return computeKnownBits(V, Q: SQ).isNonPositive();
5315}
5316
5317/// Fold (icmp pred (reduce.add X), 0) to (icmp pred' (reduce.or X), 0) when X
5318/// has lanes known to all be non-negative or all non-positive, so that
5319/// sum == 0 iff every lane is 0. Falls back to reduce.umax if reduce.or is
5320/// more expensive on the target.
5321bool VectorCombine::foldReduceAddCmpZero(Instruction &I) {
5322 CmpPredicate Pred;
5323 Value *Vec;
5324 if (!match(V: &I, P: m_ICmp(Pred,
5325 L: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::vector_reduce_add>(
5326 Ops: m_Value(V&: Vec))),
5327 R: m_Zero())))
5328 return false;
5329
5330 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
5331 if (!VecTy || VecTy->getNumElements() < 2)
5332 return false;
5333
5334 SimplifyQuery Q = SQ.getWithInstruction(I: &I);
5335 bool IsNonNegative = isKnownNonNegative(V: Vec, SQ: Q);
5336 bool IsNonPositive = !IsNonNegative && isKnownNonPositive(V: Vec, SQ: Q);
5337 if (!IsNonNegative && !IsNonPositive)
5338 return false;
5339
5340 // Summing NumElts lanes can consume up to log2(NumElts) sign bits. Require
5341 // strictly more headroom than that so the sum cannot wrap to zero.
5342 unsigned NumElts = VecTy->getNumElements();
5343 unsigned NumSignBits = ComputeNumSignBits(Op: Vec, DL: *DL, AC: SQ.AC, CxtI: &I, DT: &DT);
5344 if (Log2_32(Value: NumElts) >= NumSignBits)
5345 return false;
5346
5347 ICmpInst::Predicate NewPred;
5348 switch (Pred) {
5349 case ICmpInst::ICMP_EQ:
5350 case ICmpInst::ICMP_ULE:
5351 case ICmpInst::ICMP_SLE:
5352 case ICmpInst::ICMP_SGE:
5353 NewPred = ICmpInst::ICMP_EQ;
5354 break;
5355 case ICmpInst::ICMP_NE:
5356 case ICmpInst::ICMP_UGT:
5357 case ICmpInst::ICMP_SGT:
5358 case ICmpInst::ICMP_SLT:
5359 NewPred = ICmpInst::ICMP_NE;
5360 break;
5361 default:
5362 return false;
5363 }
5364
5365 // SGT and SLE on a non-positive tree, and SLT and SGE on a non-negative
5366 // tree, are tautologies (always true or always false). Leave those to
5367 // InstCombine rather than mapping them here. Remaining signed inequalities
5368 // also need one extra sign bit so the sum cannot flip sign.
5369 if (!IsNonNegative &&
5370 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE))
5371 return false;
5372 if (!IsNonPositive &&
5373 (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE))
5374 return false;
5375 if ((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE ||
5376 Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) &&
5377 Log2_32(Value: NumElts) >= NumSignBits - 1)
5378 return false;
5379
5380 InstructionCost OrigCost = TTI.getArithmeticReductionCost(
5381 Opcode: Instruction::Add, Ty: VecTy, FMF: std::nullopt, CostKind);
5382 InstructionCost OrCost = TTI.getArithmeticReductionCost(
5383 Opcode: Instruction::Or, Ty: VecTy, FMF: std::nullopt, CostKind);
5384 InstructionCost UmaxCost = TTI.getMinMaxReductionCost(
5385 IID: Intrinsic::umax, Ty: VecTy, FMF: FastMathFlags(), CostKind);
5386 if (!OrCost.isValid() && !UmaxCost.isValid())
5387 return false;
5388 bool UseOr = OrCost.isValid() && (!UmaxCost.isValid() || OrCost <= UmaxCost);
5389 InstructionCost AltCost = UseOr ? OrCost : UmaxCost;
5390 if (AltCost > OrigCost)
5391 return false;
5392
5393 Builder.SetInsertPoint(&I);
5394 Value *NewReduce = UseOr ? Builder.CreateOrReduce(Src: Vec)
5395 : Builder.CreateIntrinsic(
5396 ID: Intrinsic::vector_reduce_umax, OverloadTypes: {VecTy}, Args: {Vec});
5397 Worklist.pushValue(V: NewReduce);
5398 Value *NewCmp = Builder.CreateICmp(
5399 P: NewPred, LHS: NewReduce, RHS: ConstantInt::getNullValue(Ty: VecTy->getScalarType()));
5400 replaceValue(Old&: I, New&: *NewCmp);
5401 return true;
5402}
5403
5404/// Returns true if this ShuffleVectorInst eventually feeds into a
5405/// vector reduction intrinsic (e.g., vector_reduce_add) by only following
5406/// chains of shuffles and binary operators (in any combination/order).
5407/// The search does not go deeper than the given Depth.
5408static bool feedsIntoVectorReduction(ShuffleVectorInst *SVI) {
5409 constexpr unsigned MaxVisited = 32;
5410 SmallPtrSet<Instruction *, 8> Visited;
5411 SmallVector<Instruction *, 4> WorkList;
5412 bool FoundReduction = false;
5413
5414 WorkList.push_back(Elt: SVI);
5415 while (!WorkList.empty()) {
5416 Instruction *I = WorkList.pop_back_val();
5417 for (User *U : I->users()) {
5418 auto *UI = cast<Instruction>(Val: U);
5419 if (!UI || !Visited.insert(Ptr: UI).second)
5420 continue;
5421 if (Visited.size() > MaxVisited)
5422 return false;
5423 if (auto *II = dyn_cast<IntrinsicInst>(Val: UI)) {
5424 // More than one reduction reached
5425 if (FoundReduction)
5426 return false;
5427 switch (II->getIntrinsicID()) {
5428 case Intrinsic::vector_reduce_add:
5429 case Intrinsic::vector_reduce_mul:
5430 case Intrinsic::vector_reduce_and:
5431 case Intrinsic::vector_reduce_or:
5432 case Intrinsic::vector_reduce_xor:
5433 case Intrinsic::vector_reduce_smin:
5434 case Intrinsic::vector_reduce_smax:
5435 case Intrinsic::vector_reduce_umin:
5436 case Intrinsic::vector_reduce_umax:
5437 FoundReduction = true;
5438 continue;
5439 default:
5440 return false;
5441 }
5442 }
5443
5444 if (!isa<BinaryOperator>(Val: UI) && !isa<ShuffleVectorInst>(Val: UI))
5445 return false;
5446
5447 WorkList.emplace_back(Args&: UI);
5448 }
5449 }
5450 return FoundReduction;
5451}
5452
5453/// This method looks for groups of shuffles acting on binops, of the form:
5454/// %x = shuffle ...
5455/// %y = shuffle ...
5456/// %a = binop %x, %y
5457/// %b = binop %x, %y
5458/// shuffle %a, %b, selectmask
5459/// We may, especially if the shuffle is wider than legal, be able to convert
5460/// the shuffle to a form where only parts of a and b need to be computed. On
5461/// architectures with no obvious "select" shuffle, this can reduce the total
5462/// number of operations if the target reports them as cheaper.
5463bool VectorCombine::foldSelectShuffle(Instruction &I, bool FromReduction) {
5464 auto *SVI = cast<ShuffleVectorInst>(Val: &I);
5465 auto *VT = cast<FixedVectorType>(Val: I.getType());
5466 auto *Op0 = dyn_cast<Instruction>(Val: SVI->getOperand(i_nocapture: 0));
5467 auto *Op1 = dyn_cast<Instruction>(Val: SVI->getOperand(i_nocapture: 1));
5468 if (!Op0 || !Op1 || Op0 == Op1 || !Op0->isBinaryOp() || !Op1->isBinaryOp() ||
5469 VT != Op0->getType())
5470 return false;
5471
5472 auto *SVI0A = dyn_cast<Instruction>(Val: Op0->getOperand(i: 0));
5473 auto *SVI0B = dyn_cast<Instruction>(Val: Op0->getOperand(i: 1));
5474 auto *SVI1A = dyn_cast<Instruction>(Val: Op1->getOperand(i: 0));
5475 auto *SVI1B = dyn_cast<Instruction>(Val: Op1->getOperand(i: 1));
5476 SmallPtrSet<Instruction *, 4> InputShuffles({SVI0A, SVI0B, SVI1A, SVI1B});
5477 auto checkSVNonOpUses = [&](Instruction *I) {
5478 if (!I || I->getOperand(i: 0)->getType() != VT)
5479 return true;
5480 return any_of(Range: I->users(), P: [&](User *U) {
5481 return U != Op0 && U != Op1 &&
5482 !(isa<ShuffleVectorInst>(Val: U) &&
5483 (InputShuffles.contains(Ptr: cast<Instruction>(Val: U)) ||
5484 isInstructionTriviallyDead(I: cast<Instruction>(Val: U))));
5485 });
5486 };
5487 if (checkSVNonOpUses(SVI0A) || checkSVNonOpUses(SVI0B) ||
5488 checkSVNonOpUses(SVI1A) || checkSVNonOpUses(SVI1B))
5489 return false;
5490
5491 // Collect all the uses that are shuffles that we can transform together. We
5492 // may not have a single shuffle, but a group that can all be transformed
5493 // together profitably.
5494 SmallVector<ShuffleVectorInst *> Shuffles;
5495 auto collectShuffles = [&](Instruction *I) {
5496 for (auto *U : I->users()) {
5497 auto *SV = dyn_cast<ShuffleVectorInst>(Val: U);
5498 if (!SV || SV->getType() != VT)
5499 return false;
5500 if ((SV->getOperand(i_nocapture: 0) != Op0 && SV->getOperand(i_nocapture: 0) != Op1) ||
5501 (SV->getOperand(i_nocapture: 1) != Op0 && SV->getOperand(i_nocapture: 1) != Op1))
5502 return false;
5503 if (!llvm::is_contained(Range&: Shuffles, Element: SV))
5504 Shuffles.push_back(Elt: SV);
5505 }
5506 return true;
5507 };
5508 if (!collectShuffles(Op0) || !collectShuffles(Op1))
5509 return false;
5510 // From a reduction, we need to be processing a single shuffle, otherwise the
5511 // other uses will not be lane-invariant.
5512 if (FromReduction && Shuffles.size() > 1)
5513 return false;
5514
5515 // Add any shuffle uses for the shuffles we have found, to include them in our
5516 // cost calculations.
5517 if (!FromReduction) {
5518 for (size_t Idx = 0, E = Shuffles.size(); Idx != E; ++Idx) {
5519 for (auto *U : Shuffles[Idx]->users()) {
5520 ShuffleVectorInst *SSV = dyn_cast<ShuffleVectorInst>(Val: U);
5521 if (SSV && isa<UndefValue>(Val: SSV->getOperand(i_nocapture: 1)) && SSV->getType() == VT)
5522 Shuffles.push_back(Elt: SSV);
5523 }
5524 }
5525 }
5526
5527 // For each of the output shuffles, we try to sort all the first vector
5528 // elements to the beginning, followed by the second array elements at the
5529 // end. If the binops are legalized to smaller vectors, this may reduce total
5530 // number of binops. We compute the ReconstructMask mask needed to convert
5531 // back to the original lane order.
5532 SmallVector<std::pair<int, int>> V1, V2;
5533 SmallVector<SmallVector<int>> OrigReconstructMasks;
5534 int MaxV1Elt = 0, MaxV2Elt = 0;
5535 unsigned NumElts = VT->getNumElements();
5536 for (ShuffleVectorInst *SVN : Shuffles) {
5537 SmallVector<int> Mask;
5538 SVN->getShuffleMask(Result&: Mask);
5539
5540 // Check the operands are the same as the original, or reversed (in which
5541 // case we need to commute the mask).
5542 Value *SVOp0 = SVN->getOperand(i_nocapture: 0);
5543 Value *SVOp1 = SVN->getOperand(i_nocapture: 1);
5544 if (isa<UndefValue>(Val: SVOp1)) {
5545 auto *SSV = cast<ShuffleVectorInst>(Val: SVOp0);
5546 SVOp0 = SSV->getOperand(i_nocapture: 0);
5547 SVOp1 = SSV->getOperand(i_nocapture: 1);
5548 for (int &Elem : Mask) {
5549 if (Elem >= static_cast<int>(SSV->getShuffleMask().size()))
5550 return false;
5551 Elem = Elem < 0 ? Elem : SSV->getMaskValue(Elt: Elem);
5552 }
5553 }
5554 if (SVOp0 == Op1 && SVOp1 == Op0) {
5555 std::swap(a&: SVOp0, b&: SVOp1);
5556 ShuffleVectorInst::commuteShuffleMask(Mask, InVecNumElts: NumElts);
5557 }
5558 if (SVOp0 != Op0 || SVOp1 != Op1)
5559 return false;
5560
5561 // Calculate the reconstruction mask for this shuffle, as the mask needed to
5562 // take the packed values from Op0/Op1 and reconstructing to the original
5563 // order.
5564 SmallVector<int> ReconstructMask;
5565 for (unsigned I = 0; I < Mask.size(); I++) {
5566 if (Mask[I] < 0) {
5567 ReconstructMask.push_back(Elt: -1);
5568 } else if (Mask[I] < static_cast<int>(NumElts)) {
5569 MaxV1Elt = std::max(a: MaxV1Elt, b: Mask[I]);
5570 auto It = find_if(Range&: V1, P: [&](const std::pair<int, int> &A) {
5571 return Mask[I] == A.first;
5572 });
5573 if (It != V1.end())
5574 ReconstructMask.push_back(Elt: It - V1.begin());
5575 else {
5576 ReconstructMask.push_back(Elt: V1.size());
5577 V1.emplace_back(Args&: Mask[I], Args: V1.size());
5578 }
5579 } else {
5580 MaxV2Elt = std::max<int>(a: MaxV2Elt, b: Mask[I] - NumElts);
5581 auto It = find_if(Range&: V2, P: [&](const std::pair<int, int> &A) {
5582 return Mask[I] - static_cast<int>(NumElts) == A.first;
5583 });
5584 if (It != V2.end())
5585 ReconstructMask.push_back(Elt: NumElts + It - V2.begin());
5586 else {
5587 ReconstructMask.push_back(Elt: NumElts + V2.size());
5588 V2.emplace_back(Args: Mask[I] - NumElts, Args: NumElts + V2.size());
5589 }
5590 }
5591 }
5592
5593 // For reductions, we know that the lane ordering out doesn't alter the
5594 // result. In-order can help simplify the shuffle away.
5595 if (FromReduction)
5596 sort(C&: ReconstructMask);
5597 OrigReconstructMasks.push_back(Elt: std::move(ReconstructMask));
5598 }
5599
5600 // If the Maximum element used from V1 and V2 are not larger than the new
5601 // vectors, the vectors are already packes and performing the optimization
5602 // again will likely not help any further. This also prevents us from getting
5603 // stuck in a cycle in case the costs do not also rule it out.
5604 if (V1.empty() || V2.empty() ||
5605 (MaxV1Elt == static_cast<int>(V1.size()) - 1 &&
5606 MaxV2Elt == static_cast<int>(V2.size()) - 1))
5607 return false;
5608
5609 // GetBaseMaskValue takes one of the inputs, which may either be a shuffle, a
5610 // shuffle of another shuffle, or not a shuffle (that is treated like a
5611 // identity shuffle).
5612 auto GetBaseMaskValue = [&](Instruction *I, int M) {
5613 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5614 if (!SV)
5615 return M;
5616 if (isa<UndefValue>(Val: SV->getOperand(i_nocapture: 1)))
5617 if (auto *SSV = dyn_cast<ShuffleVectorInst>(Val: SV->getOperand(i_nocapture: 0)))
5618 if (InputShuffles.contains(Ptr: SSV))
5619 return SSV->getMaskValue(Elt: SV->getMaskValue(Elt: M));
5620 return SV->getMaskValue(Elt: M);
5621 };
5622
5623 // Attempt to sort the inputs my ascending mask values to make simpler input
5624 // shuffles and push complex shuffles down to the uses. We sort on the first
5625 // of the two input shuffle orders, to try and get at least one input into a
5626 // nice order.
5627 auto SortBase = [&](Instruction *A, std::pair<int, int> X,
5628 std::pair<int, int> Y) {
5629 int MXA = GetBaseMaskValue(A, X.first);
5630 int MYA = GetBaseMaskValue(A, Y.first);
5631 return MXA < MYA;
5632 };
5633 stable_sort(Range&: V1, C: [&](std::pair<int, int> A, std::pair<int, int> B) {
5634 return SortBase(SVI0A, A, B);
5635 });
5636 stable_sort(Range&: V2, C: [&](std::pair<int, int> A, std::pair<int, int> B) {
5637 return SortBase(SVI1A, A, B);
5638 });
5639 // Calculate our ReconstructMasks from the OrigReconstructMasks and the
5640 // modified order of the input shuffles.
5641 SmallVector<SmallVector<int>> ReconstructMasks;
5642 for (const auto &Mask : OrigReconstructMasks) {
5643 SmallVector<int> ReconstructMask;
5644 for (int M : Mask) {
5645 auto FindIndex = [](const SmallVector<std::pair<int, int>> &V, int M) {
5646 auto It = find_if(Range: V, P: [M](auto A) { return A.second == M; });
5647 assert(It != V.end() && "Expected all entries in Mask");
5648 return std::distance(first: V.begin(), last: It);
5649 };
5650 if (M < 0)
5651 ReconstructMask.push_back(Elt: -1);
5652 else if (M < static_cast<int>(NumElts)) {
5653 ReconstructMask.push_back(Elt: FindIndex(V1, M));
5654 } else {
5655 ReconstructMask.push_back(Elt: NumElts + FindIndex(V2, M));
5656 }
5657 }
5658 ReconstructMasks.push_back(Elt: std::move(ReconstructMask));
5659 }
5660
5661 // Calculate the masks needed for the new input shuffles, which get padded
5662 // with undef
5663 SmallVector<int> V1A, V1B, V2A, V2B;
5664 for (unsigned I = 0; I < V1.size(); I++) {
5665 V1A.push_back(Elt: GetBaseMaskValue(SVI0A, V1[I].first));
5666 V1B.push_back(Elt: GetBaseMaskValue(SVI0B, V1[I].first));
5667 }
5668 for (unsigned I = 0; I < V2.size(); I++) {
5669 V2A.push_back(Elt: GetBaseMaskValue(SVI1A, V2[I].first));
5670 V2B.push_back(Elt: GetBaseMaskValue(SVI1B, V2[I].first));
5671 }
5672 while (V1A.size() < NumElts) {
5673 V1A.push_back(Elt: PoisonMaskElem);
5674 V1B.push_back(Elt: PoisonMaskElem);
5675 }
5676 while (V2A.size() < NumElts) {
5677 V2A.push_back(Elt: PoisonMaskElem);
5678 V2B.push_back(Elt: PoisonMaskElem);
5679 }
5680
5681 auto AddShuffleCost = [&](InstructionCost C, Instruction *I) {
5682 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5683 if (!SV)
5684 return C;
5685 return C + TTI.getShuffleCost(Kind: isa<UndefValue>(Val: SV->getOperand(i_nocapture: 1))
5686 ? TTI::SK_PermuteSingleSrc
5687 : TTI::SK_PermuteTwoSrc,
5688 DstTy: VT, SrcTy: VT, CostKind, Mask: SV->getShuffleMask());
5689 };
5690 auto AddShuffleMaskCost = [&](InstructionCost C, ArrayRef<int> Mask) {
5691 return C +
5692 TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: VT, SrcTy: VT, CostKind, Mask);
5693 };
5694
5695 unsigned ElementSize = VT->getElementType()->getPrimitiveSizeInBits();
5696 unsigned MaxVectorSize =
5697 TTI.getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector);
5698 unsigned MaxElementsInVector = MaxVectorSize / ElementSize;
5699 if (MaxElementsInVector == 0)
5700 return false;
5701 // When there are multiple shufflevector operations on the same input,
5702 // especially when the vector length is larger than the register size,
5703 // identical shuffle patterns may occur across different groups of elements.
5704 // To avoid overestimating the cost by counting these repeated shuffles more
5705 // than once, we only account for unique shuffle patterns. This adjustment
5706 // prevents inflated costs in the cost model for wide vectors split into
5707 // several register-sized groups.
5708 std::set<SmallVector<int, 4>> UniqueShuffles;
5709 auto AddShuffleMaskAdjustedCost = [&](InstructionCost C, ArrayRef<int> Mask) {
5710 // Compute the cost for performing the shuffle over the full vector.
5711 auto ShuffleCost =
5712 TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: VT, SrcTy: VT, CostKind, Mask);
5713 unsigned NumFullVectors = Mask.size() / MaxElementsInVector;
5714 if (NumFullVectors < 2)
5715 return C + ShuffleCost;
5716 SmallVector<int, 4> SubShuffle(MaxElementsInVector);
5717 unsigned NumUniqueGroups = 0;
5718 unsigned NumGroups = Mask.size() / MaxElementsInVector;
5719 // For each group of MaxElementsInVector contiguous elements,
5720 // collect their shuffle pattern and insert into the set of unique patterns.
5721 for (unsigned I = 0; I < NumFullVectors; ++I) {
5722 for (unsigned J = 0; J < MaxElementsInVector; ++J)
5723 SubShuffle[J] = Mask[MaxElementsInVector * I + J];
5724 if (UniqueShuffles.insert(x: SubShuffle).second)
5725 NumUniqueGroups += 1;
5726 }
5727 return C + ShuffleCost * NumUniqueGroups / NumGroups;
5728 };
5729 auto AddShuffleAdjustedCost = [&](InstructionCost C, Instruction *I) {
5730 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5731 if (!SV)
5732 return C;
5733 SmallVector<int, 16> Mask;
5734 SV->getShuffleMask(Result&: Mask);
5735 return AddShuffleMaskAdjustedCost(C, Mask);
5736 };
5737 // Check that input consists of ShuffleVectors applied to the same input
5738 auto AllShufflesHaveSameOperands =
5739 [](SmallPtrSetImpl<Instruction *> &InputShuffles) {
5740 if (InputShuffles.size() < 2)
5741 return false;
5742 ShuffleVectorInst *FirstSV =
5743 dyn_cast<ShuffleVectorInst>(Val: *InputShuffles.begin());
5744 if (!FirstSV)
5745 return false;
5746
5747 Value *In0 = FirstSV->getOperand(i_nocapture: 0), *In1 = FirstSV->getOperand(i_nocapture: 1);
5748 return std::all_of(
5749 first: std::next(x: InputShuffles.begin()), last: InputShuffles.end(),
5750 pred: [&](Instruction *I) {
5751 ShuffleVectorInst *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5752 return SV && SV->getOperand(i_nocapture: 0) == In0 && SV->getOperand(i_nocapture: 1) == In1;
5753 });
5754 };
5755
5756 // Get the costs of the shuffles + binops before and after with the new
5757 // shuffle masks.
5758 InstructionCost CostBefore =
5759 TTI.getArithmeticInstrCost(Opcode: Op0->getOpcode(), Ty: VT, CostKind) +
5760 TTI.getArithmeticInstrCost(Opcode: Op1->getOpcode(), Ty: VT, CostKind);
5761 CostBefore += std::accumulate(first: Shuffles.begin(), last: Shuffles.end(),
5762 init: InstructionCost(0), binary_op: AddShuffleCost);
5763 if (AllShufflesHaveSameOperands(InputShuffles)) {
5764 UniqueShuffles.clear();
5765 CostBefore += std::accumulate(first: InputShuffles.begin(), last: InputShuffles.end(),
5766 init: InstructionCost(0), binary_op: AddShuffleAdjustedCost);
5767 } else {
5768 CostBefore += std::accumulate(first: InputShuffles.begin(), last: InputShuffles.end(),
5769 init: InstructionCost(0), binary_op: AddShuffleCost);
5770 }
5771
5772 // The new binops will be unused for lanes past the used shuffle lengths.
5773 // These types attempt to get the correct cost for that from the target.
5774 FixedVectorType *Op0SmallVT =
5775 FixedVectorType::get(ElementType: VT->getScalarType(), NumElts: V1.size());
5776 FixedVectorType *Op1SmallVT =
5777 FixedVectorType::get(ElementType: VT->getScalarType(), NumElts: V2.size());
5778 InstructionCost CostAfter =
5779 TTI.getArithmeticInstrCost(Opcode: Op0->getOpcode(), Ty: Op0SmallVT, CostKind) +
5780 TTI.getArithmeticInstrCost(Opcode: Op1->getOpcode(), Ty: Op1SmallVT, CostKind);
5781 UniqueShuffles.clear();
5782 CostAfter += std::accumulate(first: ReconstructMasks.begin(), last: ReconstructMasks.end(),
5783 init: InstructionCost(0), binary_op: AddShuffleMaskAdjustedCost);
5784 std::set<SmallVector<int>> OutputShuffleMasks({V1A, V1B, V2A, V2B});
5785 CostAfter +=
5786 std::accumulate(first: OutputShuffleMasks.begin(), last: OutputShuffleMasks.end(),
5787 init: InstructionCost(0), binary_op: AddShuffleMaskCost);
5788
5789 LLVM_DEBUG(dbgs() << "Found a binop select shuffle pattern: " << I << "\n");
5790 LLVM_DEBUG(dbgs() << " CostBefore: " << CostBefore
5791 << " vs CostAfter: " << CostAfter << "\n");
5792 if (CostBefore < CostAfter ||
5793 (CostBefore == CostAfter && !feedsIntoVectorReduction(SVI)))
5794 return false;
5795
5796 // The cost model has passed, create the new instructions.
5797 auto GetShuffleOperand = [&](Instruction *I, unsigned Op) -> Value * {
5798 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5799 if (!SV)
5800 return I;
5801 if (isa<UndefValue>(Val: SV->getOperand(i_nocapture: 1)))
5802 if (auto *SSV = dyn_cast<ShuffleVectorInst>(Val: SV->getOperand(i_nocapture: 0)))
5803 if (InputShuffles.contains(Ptr: SSV))
5804 return SSV->getOperand(i_nocapture: Op);
5805 return SV->getOperand(i_nocapture: Op);
5806 };
5807 Builder.SetInsertPoint(*SVI0A->getInsertionPointAfterDef());
5808 Value *NSV0A = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI0A, 0),
5809 V2: GetShuffleOperand(SVI0A, 1), Mask: V1A);
5810 Builder.SetInsertPoint(*SVI0B->getInsertionPointAfterDef());
5811 Value *NSV0B = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI0B, 0),
5812 V2: GetShuffleOperand(SVI0B, 1), Mask: V1B);
5813 Builder.SetInsertPoint(*SVI1A->getInsertionPointAfterDef());
5814 Value *NSV1A = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI1A, 0),
5815 V2: GetShuffleOperand(SVI1A, 1), Mask: V2A);
5816 Builder.SetInsertPoint(*SVI1B->getInsertionPointAfterDef());
5817 Value *NSV1B = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI1B, 0),
5818 V2: GetShuffleOperand(SVI1B, 1), Mask: V2B);
5819 Builder.SetInsertPoint(Op0);
5820 Value *NOp0 = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Op0->getOpcode(),
5821 LHS: NSV0A, RHS: NSV0B);
5822 if (auto *I = dyn_cast<Instruction>(Val: NOp0))
5823 I->copyIRFlags(V: Op0, IncludeWrapFlags: true);
5824 Builder.SetInsertPoint(Op1);
5825 Value *NOp1 = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Op1->getOpcode(),
5826 LHS: NSV1A, RHS: NSV1B);
5827 if (auto *I = dyn_cast<Instruction>(Val: NOp1))
5828 I->copyIRFlags(V: Op1, IncludeWrapFlags: true);
5829
5830 for (int S = 0, E = ReconstructMasks.size(); S != E; S++) {
5831 Builder.SetInsertPoint(Shuffles[S]);
5832 Value *NSV = Builder.CreateShuffleVector(V1: NOp0, V2: NOp1, Mask: ReconstructMasks[S]);
5833 replaceValue(Old&: *Shuffles[S], New&: *NSV, Erase: false);
5834 }
5835
5836 Worklist.pushValue(V: NSV0A);
5837 Worklist.pushValue(V: NSV0B);
5838 Worklist.pushValue(V: NSV1A);
5839 Worklist.pushValue(V: NSV1B);
5840 return true;
5841}
5842
5843/// Check if instruction depends on ZExt and this ZExt can be moved after the
5844/// instruction. Move ZExt if it is profitable. For example:
5845/// logic(zext(x),y) -> zext(logic(x,trunc(y)))
5846/// lshr((zext(x),y) -> zext(lshr(x,trunc(y)))
5847/// Cost model calculations takes into account if zext(x) has other users and
5848/// whether it can be propagated through them too.
5849bool VectorCombine::shrinkType(Instruction &I) {
5850 Value *ZExted, *OtherOperand;
5851 if (!match(V: &I, P: m_c_BitwiseLogic(L: m_ZExt(Op: m_Value(V&: ZExted)),
5852 R: m_Value(V&: OtherOperand))) &&
5853 !match(V: &I, P: m_LShr(L: m_ZExt(Op: m_Value(V&: ZExted)), R: m_Value(V&: OtherOperand))))
5854 return false;
5855
5856 Value *ZExtOperand = I.getOperand(i: I.getOperand(i: 0) == OtherOperand ? 1 : 0);
5857
5858 auto *BigTy = cast<FixedVectorType>(Val: I.getType());
5859 auto *SmallTy = cast<FixedVectorType>(Val: ZExted->getType());
5860 unsigned BW = SmallTy->getElementType()->getPrimitiveSizeInBits();
5861
5862 if (I.getOpcode() == Instruction::LShr) {
5863 // Check that the shift amount is less than the number of bits in the
5864 // smaller type. Otherwise, the smaller lshr will return a poison value.
5865 KnownBits ShAmtKB = computeKnownBits(V: I.getOperand(i: 1), DL: *DL);
5866 if (ShAmtKB.getMaxValue().uge(RHS: BW))
5867 return false;
5868 } else {
5869 // Check that the expression overall uses at most the same number of bits as
5870 // ZExted
5871 KnownBits KB = computeKnownBits(V: &I, DL: *DL);
5872 if (KB.countMaxActiveBits() > BW)
5873 return false;
5874 }
5875
5876 // Calculate costs of leaving current IR as it is and moving ZExt operation
5877 // later, along with adding truncates if needed
5878 InstructionCost ZExtCost = TTI.getCastInstrCost(
5879 Opcode: Instruction::ZExt, Dst: BigTy, Src: SmallTy,
5880 CCH: TargetTransformInfo::CastContextHint::None, CostKind);
5881 InstructionCost CurrentCost = ZExtCost;
5882 InstructionCost ShrinkCost = 0;
5883
5884 // Calculate total cost and check that we can propagate through all ZExt users
5885 for (User *U : ZExtOperand->users()) {
5886 auto *UI = cast<Instruction>(Val: U);
5887 if (UI == &I) {
5888 CurrentCost +=
5889 TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: BigTy, CostKind);
5890 ShrinkCost +=
5891 TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: SmallTy, CostKind);
5892 ShrinkCost += ZExtCost;
5893 continue;
5894 }
5895
5896 if (!Instruction::isBinaryOp(Opcode: UI->getOpcode()))
5897 return false;
5898
5899 // Check if we can propagate ZExt through its other users
5900 KnownBits KB = computeKnownBits(V: UI, DL: *DL);
5901 if (KB.countMaxActiveBits() > BW)
5902 return false;
5903
5904 CurrentCost += TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: BigTy, CostKind);
5905 ShrinkCost +=
5906 TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: SmallTy, CostKind);
5907 ShrinkCost += ZExtCost;
5908 }
5909
5910 // If the other instruction operand is not a constant, we'll need to
5911 // generate a truncate instruction. So we have to adjust cost
5912 if (!isa<Constant>(Val: OtherOperand))
5913 ShrinkCost += TTI.getCastInstrCost(
5914 Opcode: Instruction::Trunc, Dst: SmallTy, Src: BigTy,
5915 CCH: TargetTransformInfo::CastContextHint::None, CostKind);
5916
5917 // If the cost of shrinking types and leaving the IR is the same, we'll lean
5918 // towards modifying the IR because shrinking opens opportunities for other
5919 // shrinking optimisations.
5920 if (ShrinkCost > CurrentCost)
5921 return false;
5922
5923 Builder.SetInsertPoint(&I);
5924 Value *Op0 = ZExted;
5925 Value *Op1 = Builder.CreateTrunc(V: OtherOperand, DestTy: SmallTy);
5926 // Keep the order of operands the same
5927 if (I.getOperand(i: 0) == OtherOperand)
5928 std::swap(a&: Op0, b&: Op1);
5929 Value *NewBinOp =
5930 Builder.CreateBinOp(Opc: (Instruction::BinaryOps)I.getOpcode(), LHS: Op0, RHS: Op1);
5931 cast<Instruction>(Val: NewBinOp)->copyIRFlags(V: &I);
5932 cast<Instruction>(Val: NewBinOp)->copyMetadata(SrcInst: I);
5933 Value *NewZExtr = Builder.CreateZExt(V: NewBinOp, DestTy: BigTy);
5934 replaceValue(Old&: I, New&: *NewZExtr);
5935 return true;
5936}
5937
5938/// insert (DstVec, (extract SrcVec, ExtIdx), InsIdx) -->
5939/// shuffle (DstVec, SrcVec, Mask)
5940bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
5941 Value *DstVec, *SrcVec;
5942 uint64_t ExtIdx, InsIdx;
5943 if (!match(V: &I,
5944 P: m_InsertElt(Val: m_Value(V&: DstVec),
5945 Elt: m_ExtractElt(Val: m_Value(V&: SrcVec), Idx: m_ConstantInt(V&: ExtIdx)),
5946 Idx: m_ConstantInt(V&: InsIdx))))
5947 return false;
5948
5949 auto *DstVecTy = dyn_cast<FixedVectorType>(Val: I.getType());
5950 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcVec->getType());
5951 // We can try combining vectors with different element sizes.
5952 if (!DstVecTy || !SrcVecTy ||
5953 SrcVecTy->getElementType() != DstVecTy->getElementType())
5954 return false;
5955
5956 unsigned NumDstElts = DstVecTy->getNumElements();
5957 unsigned NumSrcElts = SrcVecTy->getNumElements();
5958 if (InsIdx >= NumDstElts || ExtIdx >= NumSrcElts || NumDstElts == 1)
5959 return false;
5960
5961 // Insertion into poison is a cheaper single operand shuffle.
5962 TargetTransformInfo::ShuffleKind SK;
5963 SmallVector<int> Mask(NumDstElts, PoisonMaskElem);
5964
5965 bool NeedExpOrNarrow = NumSrcElts != NumDstElts;
5966 bool NeedDstSrcSwap = isa<PoisonValue>(Val: DstVec) && !isa<UndefValue>(Val: SrcVec);
5967 if (NeedDstSrcSwap) {
5968 SK = TargetTransformInfo::SK_PermuteSingleSrc;
5969 Mask[InsIdx] = ExtIdx % NumDstElts;
5970 std::swap(a&: DstVec, b&: SrcVec);
5971 } else {
5972 SK = TargetTransformInfo::SK_PermuteTwoSrc;
5973 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
5974 Mask[InsIdx] = (ExtIdx % NumDstElts) + NumDstElts;
5975 }
5976
5977 // Cost
5978 auto *Ins = cast<InsertElementInst>(Val: &I);
5979 auto *Ext = cast<ExtractElementInst>(Val: I.getOperand(i: 1));
5980 InstructionCost InsCost =
5981 TTI.getVectorInstrCost(I: *Ins, Val: DstVecTy, CostKind, Index: InsIdx);
5982 InstructionCost ExtCost =
5983 TTI.getVectorInstrCost(I: *Ext, Val: DstVecTy, CostKind, Index: ExtIdx);
5984 InstructionCost OldCost = ExtCost + InsCost;
5985
5986 InstructionCost NewCost = 0;
5987 SmallVector<int> ExtToVecMask;
5988 if (!NeedExpOrNarrow) {
5989 // Ignore 'free' identity insertion shuffle.
5990 // TODO: getShuffleCost should return TCC_Free for Identity shuffles.
5991 if (!ShuffleVectorInst::isIdentityMask(Mask, NumSrcElts))
5992 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: DstVecTy, CostKind, Mask, Index: 0,
5993 SubTp: nullptr, Args: {DstVec, SrcVec});
5994 } else {
5995 // When creating a length-changing-vector, always try to keep the relevant
5996 // element in an equivalent position, so that bulk shuffles are more likely
5997 // to be useful.
5998 ExtToVecMask.assign(NumElts: NumDstElts, Elt: PoisonMaskElem);
5999 ExtToVecMask[ExtIdx % NumDstElts] = ExtIdx;
6000 // Add cost for expanding or narrowing
6001 NewCost = TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
6002 DstTy: DstVecTy, SrcTy: SrcVecTy, CostKind, Mask: ExtToVecMask);
6003 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: DstVecTy, CostKind, Mask);
6004 }
6005
6006 if (!Ext->hasOneUse())
6007 NewCost += ExtCost;
6008
6009 LLVM_DEBUG(dbgs() << "Found a insert/extract shuffle-like pair: " << I
6010 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
6011 << "\n");
6012
6013 if (OldCost < NewCost)
6014 return false;
6015
6016 if (NeedExpOrNarrow) {
6017 if (!NeedDstSrcSwap)
6018 SrcVec = Builder.CreateShuffleVector(V: SrcVec, Mask: ExtToVecMask);
6019 else
6020 DstVec = Builder.CreateShuffleVector(V: DstVec, Mask: ExtToVecMask);
6021 }
6022
6023 // Canonicalize undef param to RHS to help further folds.
6024 if (isa<UndefValue>(Val: DstVec) && !isa<UndefValue>(Val: SrcVec)) {
6025 ShuffleVectorInst::commuteShuffleMask(Mask, InVecNumElts: NumDstElts);
6026 std::swap(a&: DstVec, b&: SrcVec);
6027 }
6028
6029 Value *Shuf = Builder.CreateShuffleVector(V1: DstVec, V2: SrcVec, Mask);
6030 replaceValue(Old&: I, New&: *Shuf);
6031
6032 return true;
6033}
6034
6035/// Fold away a matched pair of vector.deinterleave/interleave intrinsics
6036/// with a chain of elementwise operations on each between the
6037/// deinterleave and interleave.
6038///
6039/// For example:
6040/// ```
6041/// %d = call { <2 x i16>, <2 x i16> } @deinterleave2.v4i16(<4 x i16> %v)
6042/// %f0 = extractvalue { <2 x i16>, <2 x i16> } %d, 0
6043/// %f1 = extractvalue { <2 x i16>, <2 x i16> } %d, 1
6044///
6045/// %u0 = add <2 x i16> %f0, splat (i16 3)
6046/// %u1 = add <2 x i16> %f1, splat (i16 3)
6047///
6048/// %r = call <4 x i16> @interleave2.v4i16(<2 x i16> %u0, <2 x i16> %u1)
6049/// ```
6050/// Folds to:
6051/// ```
6052/// %r = add <4 x i16> %v, splat (i16 3)
6053/// ```
6054bool VectorCombine::foldDeinterleaveInterleavePair(Instruction &I) {
6055 auto *Deinterleave = dyn_cast<IntrinsicInst>(Val: &I);
6056 if (!Deinterleave)
6057 return false;
6058
6059 unsigned Factor =
6060 getDeinterleaveIntrinsicFactor(ID: Deinterleave->getIntrinsicID());
6061 if (!Factor || Deinterleave->hasOperandBundles() ||
6062 !Deinterleave->hasNUndroppableUses(N: Factor))
6063 return false;
6064
6065 const Intrinsic::ID ExpectedInterleaveIID =
6066 Intrinsic::getInterleaveIntrinsicID(Factor);
6067
6068 // Collect one extract for each deinterleaved field.
6069 SmallVector<Use *, 8> CurrentUses(Factor, nullptr);
6070 for (Use &U : Deinterleave->uses()) {
6071 if (U.getUser()->isDroppable())
6072 continue;
6073
6074 auto *Extract = dyn_cast<ExtractValueInst>(Val: U.getUser());
6075 if (!Extract || Extract->getNumIndices() != 1)
6076 return false;
6077
6078 unsigned Index = *Extract->idx_begin();
6079 if (Index >= Factor || CurrentUses[Index])
6080 return false;
6081
6082 CurrentUses[Index] = &U;
6083 }
6084
6085 using ElementwiseStep = SmallVector<Use *, 8>;
6086 SmallVector<ElementwiseStep, 4> Steps;
6087 IntrinsicInst *Interleave = nullptr;
6088 unsigned NumVisited = 0;
6089
6090 auto GetNumDataOperands = [](Instruction *Inst) {
6091 if (auto *CB = dyn_cast<CallBase>(Val: Inst))
6092 return CB->arg_size(); // Exclude callee operand and bundles.
6093 return Inst->getNumOperands();
6094 };
6095
6096 auto IsSupportedElementwise = [&](Instruction *Inst) {
6097 auto *ResultTy = dyn_cast<VectorType>(Val: Inst->getType());
6098 if (!ResultTy || !isSafeToSpeculativelyExecute(I: Inst))
6099 return false;
6100
6101 if (auto *II = dyn_cast<IntrinsicInst>(Val: Inst)) {
6102 if (II->hasOperandBundles() ||
6103 !isTriviallyVectorizable(ID: II->getIntrinsicID()))
6104 return false;
6105 } else if (!isa<BinaryOperator, UnaryOperator, CastInst, CmpInst,
6106 SelectInst, FreezeInst>(Val: Inst)) {
6107 return false;
6108 }
6109
6110 // Reject operations that change the element-count.
6111 // E.g., bitcast <vscale x 4 x i16> %v to <vscale x 8 x i8>
6112 for (unsigned Op = 0, E = GetNumDataOperands(Inst); Op != E; ++Op) {
6113 auto *OperandTy = dyn_cast<VectorType>(Val: Inst->getOperand(i: Op)->getType());
6114 if (OperandTy &&
6115 OperandTy->getElementCount() != ResultTy->getElementCount())
6116 return false;
6117 }
6118
6119 return true;
6120 };
6121
6122 // Traverse the Factor use chains with a breadth-first search.
6123 // At each level, expect every chain to perform the same operation with the
6124 // preceding chain value at the same operand position, until they all reach
6125 // the matching interleave.
6126 while (NumVisited + Factor <= MaxInstrsToScan) {
6127 NumVisited += Factor;
6128
6129 for (Use *&CurrentUse : CurrentUses) {
6130 Use *NextUse = CurrentUse->getUser()->getSingleUndroppableUse();
6131 auto *Next =
6132 NextUse ? dyn_cast<Instruction>(Val: NextUse->getUser()) : nullptr;
6133 if (!Next)
6134 return false;
6135
6136 CurrentUse = NextUse;
6137 }
6138
6139 // Check whether every chain has reached the same interleave.
6140 if (auto *II = dyn_cast<IntrinsicInst>(Val: CurrentUses.front()->getUser());
6141 II && II->getIntrinsicID() == ExpectedInterleaveIID) {
6142 if (II->hasOperandBundles())
6143 return false;
6144
6145 for (unsigned Index = 0; Index != Factor; ++Index)
6146 if (CurrentUses[Index]->getUser() != II ||
6147 CurrentUses[Index]->getOperandNo() != Index)
6148 return false;
6149
6150 Interleave = II;
6151 break;
6152 }
6153
6154 auto *FirstInst = cast<Instruction>(Val: CurrentUses.front()->getUser());
6155 if (!IsSupportedElementwise(FirstInst))
6156 return false;
6157
6158 unsigned ChainOperand = CurrentUses.front()->getOperandNo();
6159 bool MismatchedUse = any_of(Range&: CurrentUses, P: [&](Use *U) {
6160 auto *Inst = cast<Instruction>(Val: U->getUser());
6161 return Inst != FirstInst && (U->getOperandNo() != ChainOperand ||
6162 !FirstInst->isSameOperationAs(
6163 I: Inst, flags: Instruction::CompareCallTargets));
6164 });
6165 if (MismatchedUse)
6166 return false;
6167
6168 auto GetSplatOrScalar = [](Value *V) {
6169 return isa<VectorType>(Val: V->getType()) ? getSplatValue(V) : V;
6170 };
6171
6172 // Non-chain operands must be either the same scalar or splats of that
6173 // scalar. This intentionally rejects differing poison/undef or non-splat
6174 // vector operands between chains.
6175 for (unsigned Op = 0, E = GetNumDataOperands(FirstInst); Op != E; ++Op) {
6176 if (Op == ChainOperand)
6177 continue;
6178
6179 Value *CommonValue = GetSplatOrScalar(FirstInst->getOperand(i: Op));
6180 if (!CommonValue || any_of(Range&: CurrentUses, P: [&](Use *U) {
6181 Instruction *Inst = cast<Instruction>(Val: U->getUser());
6182 return Inst != FirstInst &&
6183 GetSplatOrScalar(Inst->getOperand(i: Op)) != CommonValue;
6184 }))
6185 return false;
6186 }
6187
6188 Steps.push_back(Elt: CurrentUses);
6189 }
6190
6191 if (!Interleave)
6192 return false;
6193
6194 // Rebuild the matched elementwise chain at the original vector width.
6195 Value *WideValue = Deinterleave->getArgOperand(i: 0);
6196 ElementCount WideEC =
6197 cast<VectorType>(Val: WideValue->getType())->getElementCount();
6198
6199 auto CreateWideInstruction = [&](Instruction *NarrowInst,
6200 ArrayRef<Value *> NewOperands,
6201 VectorType *WideResultTy) -> Value * {
6202 assert(IsSupportedElementwise(NarrowInst) &&
6203 "Expected supported elementwise");
6204 if (isa<BinaryOperator, UnaryOperator>(Val: NarrowInst))
6205 return Builder.CreateNAryOp(Opc: NarrowInst->getOpcode(), Ops: NewOperands);
6206 if (auto *Cast = dyn_cast<CastInst>(Val: NarrowInst))
6207 return Builder.CreateCast(Op: Cast->getOpcode(), V: NewOperands[0],
6208 DestTy: WideResultTy);
6209 if (auto *Cmp = dyn_cast<CmpInst>(Val: NarrowInst))
6210 return Builder.CreateCmp(Pred: Cmp->getPredicate(), LHS: NewOperands[0],
6211 RHS: NewOperands[1]);
6212 if (isa<SelectInst>(Val: NarrowInst))
6213 return Builder.CreateSelect(
6214 C: NewOperands[0], True: NewOperands[1], False: NewOperands[2], /*Name=*/"",
6215 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : NarrowInst);
6216 if (isa<FreezeInst>(Val: NarrowInst))
6217 return Builder.CreateFreeze(V: NewOperands[0]);
6218 if (auto *II = dyn_cast<IntrinsicInst>(Val: NarrowInst))
6219 return Builder.CreateIntrinsic(RetTy: WideResultTy, ID: II->getIntrinsicID(),
6220 Args: NewOperands);
6221 llvm_unreachable("Unsupported instruction");
6222 };
6223
6224 // The BFS has succeeded and collected multiple levels of instructions that
6225 // can be SLP-widened into a chain of wider instructions.
6226 for (const ElementwiseStep &Step : Steps) {
6227 Instruction *NarrowInst = cast<Instruction>(Val: Step.front()->getUser());
6228 unsigned ChainOperand = Step.front()->getOperandNo();
6229
6230 Builder.SetInsertPoint(NarrowInst);
6231 Builder.SetCurrentDebugLocation(NarrowInst->getDebugLoc());
6232
6233 unsigned NumOperands = GetNumDataOperands(NarrowInst);
6234 SmallVector<Value *, 4> NewOperands;
6235 NewOperands.reserve(N: NumOperands);
6236
6237 for (unsigned Op = 0; Op != NumOperands; ++Op) {
6238 Value *Operand = NarrowInst->getOperand(i: Op);
6239
6240 if (Op == ChainOperand)
6241 Operand = WideValue;
6242 else if (isa<VectorType>(Val: Operand->getType()))
6243 Operand = Builder.CreateVectorSplat(EC: WideEC, V: getSplatValue(V: Operand));
6244 NewOperands.push_back(Elt: Operand);
6245 }
6246
6247 auto *WideResultTy =
6248 VectorType::get(ElementType: NarrowInst->getType()->getScalarType(), EC: WideEC);
6249 Value *NewValue =
6250 CreateWideInstruction(NarrowInst, NewOperands, WideResultTy);
6251
6252 SmallVector<Value *> NarrowInsts =
6253 map_to_vector(C: Step, F: [](Use *U) { return cast<Value>(Val: U->getUser()); });
6254 propagateIRFlags(I: NewValue, VL: NarrowInsts);
6255
6256 if (auto *NewInst = dyn_cast<Instruction>(Val: NewValue))
6257 propagateMetadata(I: NewInst, VL: NarrowInsts);
6258
6259 WideValue = NewValue;
6260 }
6261
6262 assert(WideValue->getType() == Interleave->getType());
6263 replaceValue(Old&: *Interleave, New&: *WideValue);
6264 return true;
6265}
6266
6267/// If we're interleaving 2 constant splats, for instance `<vscale x 8 x i32>
6268/// <splat of 666>` and `<vscale x 8 x i32> <splat of 777>`, we can create a
6269/// larger splat `<vscale x 8 x i64> <splat of ((777 << 32) | 666)>` first
6270/// before casting it back into `<vscale x 16 x i32>`.
6271bool VectorCombine::foldInterleaveIntrinsics(Instruction &I) {
6272 const APInt *SplatVal0, *SplatVal1;
6273 if (!match(V: &I, P: m_Intrinsic<Intrinsic::vector_interleave2>(
6274 Ops: m_APInt(Res&: SplatVal0), Ops: m_APInt(Res&: SplatVal1))))
6275 return false;
6276
6277 LLVM_DEBUG(dbgs() << "VC: Folding interleave2 with two splats: " << I
6278 << "\n");
6279
6280 auto *VTy =
6281 cast<VectorType>(Val: cast<IntrinsicInst>(Val&: I).getArgOperand(i: 0)->getType());
6282 auto *ExtVTy = VectorType::getExtendedElementVectorType(VTy);
6283 unsigned Width = VTy->getElementType()->getIntegerBitWidth();
6284
6285 // Just in case the cost of interleave2 intrinsic and bitcast are both
6286 // invalid, in which case we want to bail out, we use <= rather
6287 // than < here. Even they both have valid and equal costs, it's probably
6288 // not a good idea to emit a high-cost constant splat.
6289 if (TTI.getInstructionCost(U: &I, CostKind) <=
6290 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: I.getType(), Src: ExtVTy,
6291 CCH: TTI::CastContextHint::None, CostKind)) {
6292 LLVM_DEBUG(dbgs() << "VC: The cost to cast from " << *ExtVTy << " to "
6293 << *I.getType() << " is too high.\n");
6294 return false;
6295 }
6296
6297 APInt NewSplatVal = SplatVal1->zext(width: Width * 2);
6298 NewSplatVal <<= Width;
6299 NewSplatVal |= SplatVal0->zext(width: Width * 2);
6300 auto *NewSplat = ConstantVector::getSplat(
6301 EC: ExtVTy->getElementCount(), Elt: ConstantInt::get(Context&: F.getContext(), V: NewSplatVal));
6302
6303 IRBuilder<> Builder(&I);
6304 replaceValue(Old&: I, New&: *Builder.CreateBitCast(V: NewSplat, DestTy: I.getType()));
6305 return true;
6306}
6307
6308/// Given this sequence:
6309/// ```
6310/// %d = llvm.vector.deinterleave2 <vscale x 16 x i32> %v
6311/// %f0 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d, 0
6312/// %f1 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d, 1
6313///
6314/// %low0 = and <vscale x 8 x i32> %f0, splat (i32 65535)
6315/// %low1 = shl <vscale x 8 x i32> %f1, splat (i32 16)
6316/// %merge0 = or disjoint <vscale x 8 x i32> %low0, %low1
6317///
6318/// %high0 = and <vscale x 8 x i32> %f1, splat (i32 -65536)
6319/// %high1 = lshr <vscale x 8 x i32> %f0, splat (i32 16)
6320/// %merge1 = or disjoint <vscale x 8 x i32> %high0, %high1
6321/// ```
6322/// It is actually just de-interleaving a 16-bit vector with double the
6323/// vector length. More generally speaking, it's de-interleaving on a vector
6324/// with half the element width as the original vector.
6325///
6326/// Therefore, we can turn it into:
6327/// ```
6328/// %narrow.v = bitcast <vscale x 16 x i32> %v to <vscale x 32 x i16>
6329/// %d = llvm.vector.deinterleave2 <vscale x 32 x i16> %narrow.v
6330/// %f0 = extractvalue { <vscale x 16 x i16>, <vscale x 16 x i16> } %d, 0
6331/// %f1 = extractvalue { <vscale x 16 x i16>, <vscale x 16 x i16> } %d, 1
6332///
6333/// %merge0 = bitcast <vscale x 16 x i16> %f0 to <vscale x 8 x i32>
6334/// %merge1 = bitcast <vscale x 16 x i16> %f1 to <vscale x 8 x i32>
6335/// ```
6336bool VectorCombine::foldDeinterleaveIntrinsics(Instruction &I) {
6337 if (foldDeinterleaveInterleavePair(I))
6338 return true;
6339
6340 // This pattern involves bitcast that is not compatible with big endian.
6341 if (DL->isBigEndian())
6342 return false;
6343
6344 using namespace PatternMatch;
6345 Value *DeinterleavedVal;
6346 if (!match(V: &I, P: m_Deinterleave2(Op: m_Value(V&: DeinterleavedVal))))
6347 return false;
6348
6349 VectorType *VecTy = cast<VectorType>(Val: DeinterleavedVal->getType());
6350 IntegerType *ElementTy = dyn_cast<IntegerType>(Val: VecTy->getElementType());
6351 if (!ElementTy)
6352 return false;
6353 unsigned ElementWidth = ElementTy->getBitWidth();
6354 if (ElementWidth < 2 || !isPowerOf2_32(Value: ElementWidth))
6355 return false;
6356 unsigned HalfElementWidth = ElementWidth / 2;
6357
6358 if (!I.hasNUses(N: 2))
6359 return false;
6360 std::array<ExtractValueInst *, 2> OrigFields{};
6361 for (User *Usr : I.users()) {
6362 auto *E = dyn_cast<ExtractValueInst>(Val: Usr);
6363 // The deinterleave result can only be used by extractions.
6364 if (!E || E->getNumIndices() != 1)
6365 return false;
6366 unsigned Idx = *E->idx_begin();
6367 // A single field cannot be extracted more than once.
6368 if (Idx >= 2 || OrigFields[Idx] || !E->hasNUses(N: 2))
6369 return false;
6370 OrigFields[Idx] = E;
6371 }
6372
6373 // Find the merge instruction (i.e. OR) first.
6374 SmallVector<Instruction *, 2> MergeInsts;
6375 for (auto *FieldUsr : OrigFields[0]->users()) {
6376 if (!FieldUsr->hasOneUse() || !isa<Instruction>(Val: FieldUsr->user_back()))
6377 return false;
6378 MergeInsts.push_back(Elt: cast<Instruction>(Val: FieldUsr->user_back()));
6379 }
6380 assert(MergeInsts.size() == 2);
6381
6382 // Pattern match bottom-up from the merge instructions.
6383 auto MatchMerge = [&](void) -> bool {
6384 APInt LoMask = APInt::getLowBitsSet(numBits: ElementWidth, loBitsSet: HalfElementWidth);
6385 APInt HiMask = APInt::getHighBitsSet(numBits: ElementWidth, hiBitsSet: HalfElementWidth);
6386 return match(V: MergeInsts[0],
6387 P: m_c_Or(L: m_And(L: m_Specific(V: OrigFields[0]), R: m_SpecificInt(V: LoMask)),
6388 R: m_Shl(L: m_Specific(V: OrigFields[1]),
6389 R: m_SpecificInt(V: HalfElementWidth)))) &&
6390 match(V: MergeInsts[1],
6391 P: m_c_Or(L: m_And(L: m_Specific(V: OrigFields[1]), R: m_SpecificInt(V: HiMask)),
6392 R: m_LShr(L: m_Specific(V: OrigFields[0]),
6393 R: m_SpecificInt(V: HalfElementWidth))));
6394 };
6395 if (!MatchMerge()) {
6396 std::swap(a&: MergeInsts[0], b&: MergeInsts[1]);
6397 if (!MatchMerge())
6398 return false;
6399 }
6400
6401 // Profitability check.
6402 InstructionCost OldCost =
6403 TTI.getInstructionCost(U: MergeInsts[0], CostKind) +
6404 TTI.getInstructionCost(U: cast<Instruction>(Val: MergeInsts[0]->getOperand(i: 0)),
6405 CostKind) +
6406 TTI.getInstructionCost(U: cast<Instruction>(Val: MergeInsts[0]->getOperand(i: 1)),
6407 CostKind);
6408 // There are two fields (assuming SHL has the same cost as LSHR).
6409 OldCost *= 2;
6410
6411 auto *NewFieldTy = VecTy->getWithNewBitWidth(NewBitWidth: HalfElementWidth);
6412 auto *NewVecTy =
6413 VectorType::getDoubleElementsVectorType(VTy: cast<VectorType>(Val: NewFieldTy));
6414 InstructionCost NewCost =
6415 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: VecTy, Src: NewVecTy,
6416 CCH: TTI::CastContextHint::None, CostKind) +
6417 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: NewFieldTy,
6418 Src: MergeInsts[0]->getType(), CCH: TTI::CastContextHint::None,
6419 CostKind) *
6420 2;
6421 if (OldCost <= NewCost || !NewCost.isValid()) {
6422 LLVM_DEBUG(
6423 dbgs() << "VC: New deinterleave2 sequence cost (" << NewCost << ")"
6424 << " is higher than that of the old one (" << OldCost << ")\n");
6425 return false;
6426 }
6427
6428 // Do the replacement.
6429 IRBuilder<> Builder(&I);
6430 Value *NewVecCast = Builder.CreateBitCast(V: DeinterleavedVal, DestTy: NewVecTy);
6431 Value *NewDeinterleave = Builder.CreateIntrinsic(
6432 ID: Intrinsic::vector_deinterleave2, OverloadTypes: {NewVecTy}, Args: {NewVecCast});
6433 for (auto [Idx, MergeInst] : enumerate(First&: MergeInsts)) {
6434 Value *NewField = Builder.CreateExtractValue(Agg: NewDeinterleave, Idxs: Idx);
6435 NewField = Builder.CreateBitCast(V: NewField, DestTy: MergeInst->getType());
6436 replaceValue(Old&: *MergeInst, New&: *NewField);
6437 }
6438
6439 return true;
6440}
6441
6442bool VectorCombine::foldBitcastOfVPLoad(Instruction &I) {
6443 const DataLayout &DL = I.getDataLayout();
6444 auto *Cast = dyn_cast<CastInst>(Val: &I);
6445 if (!Cast || !Cast->isNoopCast(DL) || !isa<VectorType>(Val: Cast->getDestTy()))
6446 return false;
6447
6448 // Fold away bit casts of the loaded value by loading the desired type,
6449 // if the mask is all-ones.
6450 Value *EVL;
6451 auto *II = dyn_cast<VPIntrinsic>(Val: I.getOperand(i: 0));
6452 if (!II || !match(V: II, P: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::vp_load>(
6453 Ops: m_Value(), Ops: m_AllOnes(), Ops: m_Value(V&: EVL)))))
6454 return false;
6455
6456 VectorType *OrigVecTy = cast<VectorType>(Val: II->getType());
6457 Align OrigAlign =
6458 DL.getValueOrABITypeAlignment(Alignment: II->getPointerAlignment(), Ty: OrigVecTy);
6459 ElementCount OrigVecCnt = OrigVecTy->getElementCount();
6460 VectorType *NewVecTy = cast<VectorType>(Val: Cast->getDestTy());
6461 ElementCount NewVecCnt = NewVecTy->getElementCount();
6462
6463 // Right now we only support cases where the NewVec is longer, because for
6464 // cases where it's shorter, we have to be sure that EVL can be exactly
6465 // divided, otherwise it might yield incorrect results or even page faults
6466 // (if we round-up during the division).
6467 if (!(OrigVecCnt.isScalable() == NewVecCnt.isScalable() &&
6468 NewVecCnt.hasKnownScalarFactor(RHS: OrigVecCnt)))
6469 return false;
6470
6471 InstructionCost OldCost =
6472 TTI.getMemIntrinsicInstrCost(MICA: {Intrinsic::vp_load, OrigVecTy,
6473 II->getMemoryPointerParam(), false,
6474 OrigAlign},
6475 CostKind) +
6476 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: Cast->getType(), Src: OrigVecTy,
6477 CCH: TTI::CastContextHint::None, CostKind);
6478 InstructionCost NewCost = TTI.getMemIntrinsicInstrCost(
6479 MICA: {Intrinsic::vp_load, NewVecTy, II->getMemoryPointerParam(), false,
6480 OrigAlign},
6481 CostKind);
6482 LLVM_DEBUG(dbgs() << "foldBitcastOfVPLoad: OldCost=" << OldCost
6483 << " NewCost=" << NewCost << "\n");
6484 if (NewCost > OldCost || !NewCost.isValid())
6485 return false;
6486
6487 Builder.SetInsertPoint(II);
6488 unsigned Factor = NewVecCnt.getKnownScalarFactor(RHS: OrigVecCnt);
6489 Value *NewEVL = Builder.CreateNUWMul(LHS: EVL, RHS: Builder.getInt32(C: Factor));
6490 Value *NewMask = Builder.CreateVectorSplat(EC: NewVecCnt, V: Builder.getTrue());
6491 CallInst *NewVP = Builder.CreateIntrinsicWithoutFolding(
6492 RetTy: NewVecTy, ID: Intrinsic::vp_load,
6493 Args: {II->getMemoryPointerParam(), NewMask, NewEVL});
6494 // Preserve the original alignment.
6495 NewVP->addParamAttrs(
6496 ArgNo: 0, B: AttrBuilder(II->getContext()).addAlignmentAttr(Align: OrigAlign));
6497 replaceValue(Old&: *Cast, New&: *NewVP);
6498 return true;
6499}
6500/// Fold the following cases into a single byte-level bit-reverse operation
6501/// and accepts bswap and bitreverse intrinsics:
6502/// bswap(bitreverse(x)) --> bitcast(bitreverse(bitcast(x)))
6503/// bitreverse(bswap(x)) <--> bitcast(bitreverse(bitcast(x)))
6504/// The direction of the fold is cost-model driven.
6505/// Also supports:
6506/// bitcast(bitreverse(bitcast(x))) --> bitreverse(fshl(x))
6507bool VectorCombine::foldBitOrderReverseAndSwap(Instruction &I) {
6508 Value *X;
6509
6510 if (match(V: &I, P: m_BitCast(Op: m_BitReverse(Op0: m_BitCast(Op: m_Value(V&: X)))))) {
6511 Type *Ty = X->getType();
6512 Type *VecTy = I.getOperand(i: 0)->getType();
6513 // Detect the case when bitreversing every octet in X individually. Then we
6514 // can use bswap to reorder the octets before doing a single bitreverse.
6515 bool CanUseBswap =
6516 Ty->isIntegerTy() && Ty == I.getType() && isa<FixedVectorType>(Val: VecTy) &&
6517 cast<FixedVectorType>(Val: VecTy)->getElementType()->isIntegerTy(BitWidth: 8) &&
6518 Ty->getIntegerBitWidth() % 16 == 0;
6519 // Detect the case when bitreversing upper and lower half of X
6520 // individually. Then we can use fshl as a rotate operation, to swap the
6521 // halves before doing a single bitreverse.
6522 bool CanUseFshl =
6523 Ty->isIntegerTy() && Ty == I.getType() && isa<FixedVectorType>(Val: VecTy) &&
6524 cast<FixedVectorType>(Val: VecTy)->getElementType()->isIntegerTy() &&
6525 cast<FixedVectorType>(Val: VecTy)->getNumElements() == 2;
6526 if (CanUseBswap || CanUseFshl) {
6527 auto *InnerCall = dyn_cast<Instruction>(Val: I.getOperand(i: 0));
6528 if (!InnerCall)
6529 return false;
6530 auto *InnerBitCast = dyn_cast<BitCastInst>(Val: InnerCall->getOperand(i: 0));
6531 if (!InnerBitCast)
6532 return false;
6533 Constant *HalfBW = ConstantInt::get(Ty, V: Ty->getIntegerBitWidth() / 2);
6534 InstructionCost OldCost = TTI.getInstructionCost(U: InnerBitCast, CostKind) +
6535 TTI.getInstructionCost(U: InnerCall, CostKind) +
6536 TTI.getInstructionCost(U: &I, CostKind);
6537 IntrinsicCostAttributes ICABSwap(Intrinsic::bswap, Ty, {Ty});
6538 IntrinsicCostAttributes ICABFshl(Intrinsic::fshl, Ty, {X, X, HalfBW},
6539 {Ty, Ty, Ty});
6540 IntrinsicCostAttributes ICABRev(Intrinsic::bitreverse, Ty, {Ty});
6541 InstructionCost NewCost =
6542 TTI.getIntrinsicInstrCost(ICA: CanUseBswap ? ICABSwap : ICABFshl,
6543 CostKind) +
6544 TTI.getIntrinsicInstrCost(ICA: ICABRev, CostKind);
6545 if (!InnerCall->hasOneUse())
6546 NewCost += TTI.getInstructionCost(U: InnerCall, CostKind) +
6547 TTI.getInstructionCost(U: InnerBitCast, CostKind);
6548 else if (!InnerBitCast->hasOneUse())
6549 NewCost += TTI.getInstructionCost(U: InnerBitCast, CostKind);
6550 LLVM_DEBUG(dbgs() << "Found bitreverse vector roundtrip: " << I
6551 << "\n OldCost: " << OldCost
6552 << " vs NewCost: " << NewCost << "\n");
6553 if (NewCost.isValid() && NewCost < OldCost) {
6554 Builder.SetInsertPoint(&I);
6555 Value *Swap =
6556 CanUseBswap
6557 ? Builder.CreateUnaryIntrinsic(ID: Intrinsic::bswap, Op: X)
6558 : Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::fshl, Args: {X, X, HalfBW});
6559 Worklist.pushValue(V: Swap);
6560 Value *BRev = Builder.CreateUnaryIntrinsic(ID: Intrinsic::bitreverse, Op: Swap);
6561 replaceValue(Old&: I, New&: *BRev);
6562 return true;
6563 }
6564 }
6565 }
6566
6567 if (!match(V: &I, P: m_BitReverse(Op0: m_BSwap(Op0: m_Value(V&: X)))) &&
6568 !match(V: &I, P: m_BSwap(Op0: m_BitReverse(Op0: m_Value(V&: X)))))
6569 return false;
6570 Type *Ty = I.getType();
6571 Type *I8Ty = Builder.getInt8Ty();
6572 TypeSize ElementSize = DL->getTypeStoreSize(Ty);
6573 ElementCount NewVecCnt = ElementCount::get(MinVal: ElementSize.getKnownMinValue(),
6574 Scalable: ElementSize.isScalable());
6575 Type *NewVecTy = VectorType::get(ElementType: I8Ty, EC: NewVecCnt);
6576 auto *II = cast<IntrinsicInst>(Val: &I);
6577 auto *InnerII = cast<IntrinsicInst>(Val: II->getArgOperand(i: 0));
6578 // OldCost = cost of bitreverse/bswap + cost of bswap/bitreverse
6579 InstructionCost OldCost = TTI.getInstructionCost(U: II, CostKind) +
6580 TTI.getInstructionCost(U: InnerII, CostKind);
6581 // NewCost = cost of bitcast to byte vector +
6582 // cost of bitreverse/bswap on byte vector +
6583 // cost of bitcast back to original type
6584 InstructionCost CastToVecCost = TTI.getCastInstrCost(
6585 Opcode: Instruction::BitCast, Dst: NewVecTy, Src: Ty, CCH: TTI::CastContextHint::None, CostKind);
6586 InstructionCost CastToOrigCost = TTI.getCastInstrCost(
6587 Opcode: Instruction::BitCast, Dst: Ty, Src: NewVecTy, CCH: TTI::CastContextHint::None, CostKind);
6588 IntrinsicCostAttributes ICANew(Intrinsic::bitreverse, NewVecTy, {NewVecTy});
6589 InstructionCost NewIntrinsicCost =
6590 TTI.getIntrinsicInstrCost(ICA: ICANew, CostKind);
6591 InstructionCost NewCost = CastToVecCost + NewIntrinsicCost + CastToOrigCost;
6592 if (!InnerII->hasOneUse())
6593 NewCost += TTI.getInstructionCost(U: InnerII, CostKind);
6594 LLVM_DEBUG(dbgs() << "Found bitorder reverse and swap: " << I
6595 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
6596 << "\n");
6597 if (!NewCost.isValid() || NewCost >= OldCost)
6598 return false;
6599 // Perform transform: bitcast(arg, <N x i8>), bitreverse, bitcast back
6600 Builder.SetInsertPoint(II);
6601 Value *CastToVec = Builder.CreateBitCast(V: X, DestTy: NewVecTy);
6602 Value *NewCall =
6603 Builder.CreateUnaryIntrinsic(ID: Intrinsic::bitreverse, Op: CastToVec);
6604 Value *CastToOrig = Builder.CreateBitCast(V: NewCall, DestTy: Ty);
6605 replaceValue(Old&: I, New&: *CastToOrig);
6606 return true;
6607}
6608
6609/// Given the maximum shuffle index and load vector type, compute the number of
6610/// elements for the shrunk load, rounding up to the next full vector register
6611/// boundary to avoid scalar remainders that legalize poorly.
6612static unsigned getAlignedNumElements(unsigned MaxIdx, FixedVectorType *LoadTy,
6613 const TargetTransformInfo &TTI,
6614 const DataLayout &DL) {
6615 unsigned RawNumElements = MaxIdx + 1u;
6616 Type *ElemTy = LoadTy->getElementType();
6617 // Skip alignment for illegal element types.
6618 if (!TTI.isTypeLegal(Ty: ElemTy))
6619 return RawNumElements;
6620
6621 TypeSize ElemSize = DL.getTypeSizeInBits(Ty: ElemTy);
6622 if (ElemSize.isScalable() || ElemSize.isZero())
6623 return RawNumElements;
6624
6625 TypeSize RegSize =
6626 TTI.getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector);
6627 if (RegSize.isScalable() || RegSize.isZero())
6628 return RawNumElements;
6629
6630 unsigned ElemsPerReg = RegSize.getFixedValue() / ElemSize.getFixedValue();
6631 // If the load already fits in a register, keep the exact size.
6632 // Otherwise round up to the next full register boundary.
6633 if (ElemsPerReg == 0 || RawNumElements <= ElemsPerReg)
6634 return RawNumElements;
6635
6636 return alignTo(Value: RawNumElements, Align: ElemsPerReg);
6637}
6638
6639// Attempt to shrink loads that are only used by shufflevector instructions.
6640bool VectorCombine::shrinkLoadForShuffles(Instruction &I) {
6641 auto *OldLoad = dyn_cast<LoadInst>(Val: &I);
6642 if (!OldLoad || !OldLoad->isSimple())
6643 return false;
6644
6645 auto *OldLoadTy = dyn_cast<FixedVectorType>(Val: OldLoad->getType());
6646 if (!OldLoadTy)
6647 return false;
6648
6649 unsigned const OldNumElements = OldLoadTy->getNumElements();
6650
6651 // Search all uses of load. If all uses are shufflevector instructions, and
6652 // the second operands are all poison values, find the minimum and maximum
6653 // indices of the vector elements referenced by all shuffle masks.
6654 // Otherwise return `std::nullopt`.
6655 using IndexRange = std::pair<int, int>;
6656 auto GetIndexRangeInShuffles = [&]() -> std::optional<IndexRange> {
6657 IndexRange OutputRange = IndexRange(OldNumElements, -1);
6658 for (llvm::Use &Use : I.uses()) {
6659 // Ensure all uses match the required pattern.
6660 User *Shuffle = Use.getUser();
6661 ArrayRef<int> Mask;
6662
6663 if (!match(V: Shuffle,
6664 P: m_Shuffle(v1: m_Specific(V: OldLoad), v2: m_Undef(), mask: m_Mask(Mask))))
6665 return std::nullopt;
6666
6667 // Ignore shufflevector instructions that have no uses.
6668 if (Shuffle->use_empty())
6669 continue;
6670
6671 // Find the min and max indices used by the shufflevector instruction.
6672 for (int Index : Mask) {
6673 if (Index >= 0 && Index < static_cast<int>(OldNumElements)) {
6674 OutputRange.first = std::min(a: Index, b: OutputRange.first);
6675 OutputRange.second = std::max(a: Index, b: OutputRange.second);
6676 }
6677 }
6678 }
6679
6680 if (OutputRange.second < OutputRange.first)
6681 return std::nullopt;
6682
6683 return OutputRange;
6684 };
6685
6686 // Get the range of vector elements used by shufflevector instructions.
6687 if (std::optional<IndexRange> Indices = GetIndexRangeInShuffles()) {
6688 unsigned const NewNumElements =
6689 getAlignedNumElements(MaxIdx: Indices->second, LoadTy: OldLoadTy, TTI, DL: *DL);
6690
6691 // If the range of vector elements is smaller than the full load, attempt
6692 // to create a smaller load.
6693 if (NewNumElements < OldNumElements) {
6694 IRBuilder Builder(&I);
6695 Builder.SetCurrentDebugLocation(I.getDebugLoc());
6696
6697 // Calculate costs of old and new ops.
6698 Type *ElemTy = OldLoadTy->getElementType();
6699 FixedVectorType *NewLoadTy = FixedVectorType::get(ElementType: ElemTy, NumElts: NewNumElements);
6700 Value *PtrOp = OldLoad->getPointerOperand();
6701
6702 InstructionCost OldCost = TTI.getMemoryOpCost(
6703 Opcode: Instruction::Load, Src: OldLoad->getType(), Alignment: OldLoad->getAlign(),
6704 AddressSpace: OldLoad->getPointerAddressSpace(), CostKind);
6705 InstructionCost NewCost =
6706 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: NewLoadTy, Alignment: OldLoad->getAlign(),
6707 AddressSpace: OldLoad->getPointerAddressSpace(), CostKind);
6708
6709 using UseEntry = std::pair<ShuffleVectorInst *, std::vector<int>>;
6710 SmallVector<UseEntry, 4u> NewUses;
6711 unsigned const MaxIndex = NewNumElements * 2u;
6712
6713 for (llvm::Use &Use : I.uses()) {
6714 auto *Shuffle = cast<ShuffleVectorInst>(Val: Use.getUser());
6715
6716 // Ignore shufflevector instructions that have no uses.
6717 if (Shuffle->use_empty())
6718 continue;
6719
6720 ArrayRef<int> OldMask = Shuffle->getShuffleMask();
6721
6722 // Create entry for new use.
6723 NewUses.push_back(Elt: {Shuffle, OldMask});
6724
6725 // Validate mask indices.
6726 for (int Index : OldMask) {
6727 if (Index >= static_cast<int>(MaxIndex))
6728 return false;
6729 }
6730
6731 // Update costs.
6732 OldCost +=
6733 TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: Shuffle->getType(),
6734 SrcTy: OldLoadTy, CostKind, Mask: OldMask);
6735 NewCost +=
6736 TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: Shuffle->getType(),
6737 SrcTy: NewLoadTy, CostKind, Mask: OldMask);
6738 }
6739
6740 LLVM_DEBUG(
6741 dbgs() << "Found a load used only by shufflevector instructions: "
6742 << I << "\n OldCost: " << OldCost
6743 << " vs NewCost: " << NewCost << "\n");
6744
6745 if (OldCost < NewCost || !NewCost.isValid())
6746 return false;
6747
6748 // Create new load of smaller vector.
6749 auto *NewLoad = cast<LoadInst>(
6750 Val: Builder.CreateAlignedLoad(Ty: NewLoadTy, Ptr: PtrOp, Align: OldLoad->getAlign()));
6751 NewLoad->copyMetadata(SrcInst: I);
6752
6753 // Replace all uses.
6754 for (UseEntry &Use : NewUses) {
6755 ShuffleVectorInst *Shuffle = Use.first;
6756 std::vector<int> &NewMask = Use.second;
6757
6758 Builder.SetInsertPoint(Shuffle);
6759 Builder.SetCurrentDebugLocation(Shuffle->getDebugLoc());
6760 Value *NewShuffle = Builder.CreateShuffleVector(
6761 V1: NewLoad, V2: PoisonValue::get(T: NewLoadTy), Mask: NewMask);
6762
6763 replaceValue(Old&: *Shuffle, New&: *NewShuffle, Erase: false);
6764 }
6765
6766 return true;
6767 }
6768 }
6769 return false;
6770}
6771
6772// Attempt to narrow a phi of shufflevector instructions where the two incoming
6773// values have the same operands but different masks. If the two shuffle masks
6774// are offsets of one another we can use one branch to rotate the incoming
6775// vector and perform one larger shuffle after the phi.
6776bool VectorCombine::shrinkPhiOfShuffles(Instruction &I) {
6777 auto *Phi = dyn_cast<PHINode>(Val: &I);
6778 if (!Phi || Phi->getNumIncomingValues() != 2u)
6779 return false;
6780
6781 Value *Op = nullptr;
6782 ArrayRef<int> Mask0;
6783 ArrayRef<int> Mask1;
6784
6785 if (!match(V: Phi->getOperand(i_nocapture: 0u),
6786 P: m_OneUse(SubPattern: m_Shuffle(v1: m_Value(V&: Op), v2: m_Poison(), mask: m_Mask(Mask0)))) ||
6787 !match(V: Phi->getOperand(i_nocapture: 1u),
6788 P: m_OneUse(SubPattern: m_Shuffle(v1: m_Specific(V: Op), v2: m_Poison(), mask: m_Mask(Mask1)))))
6789 return false;
6790
6791 auto *Shuf = cast<ShuffleVectorInst>(Val: Phi->getOperand(i_nocapture: 0u));
6792
6793 // Ensure result vectors are wider than the argument vector.
6794 auto *InputVT = cast<FixedVectorType>(Val: Op->getType());
6795 auto *ResultVT = cast<FixedVectorType>(Val: Shuf->getType());
6796 auto const InputNumElements = InputVT->getNumElements();
6797
6798 if (InputNumElements >= ResultVT->getNumElements())
6799 return false;
6800
6801 // Take the difference of the two shuffle masks at each index. Ignore poison
6802 // values at the same index in both masks.
6803 SmallVector<int, 16> NewMask;
6804 NewMask.reserve(N: Mask0.size());
6805
6806 for (auto [M0, M1] : zip(t&: Mask0, u&: Mask1)) {
6807 if (M0 >= 0 && M1 >= 0)
6808 NewMask.push_back(Elt: M0 - M1);
6809 else if (M0 == -1 && M1 == -1)
6810 continue;
6811 else
6812 return false;
6813 }
6814
6815 // Ensure all elements of the new mask are equal. If the difference between
6816 // the incoming mask elements is the same, the two must be constant offsets
6817 // of one another.
6818 if (NewMask.empty() || !all_equal(Range&: NewMask))
6819 return false;
6820
6821 // Create new mask using difference of the two incoming masks.
6822 int MaskOffset = NewMask[0u];
6823 unsigned Index = (InputNumElements + MaskOffset) % InputNumElements;
6824 NewMask.clear();
6825
6826 for (unsigned I = 0u; I < InputNumElements; ++I) {
6827 NewMask.push_back(Elt: Index);
6828 Index = (Index + 1u) % InputNumElements;
6829 }
6830
6831 // Calculate costs for worst cases and compare.
6832 auto const Kind = TTI::SK_PermuteSingleSrc;
6833 auto OldCost =
6834 std::max(a: TTI.getShuffleCost(Kind, DstTy: ResultVT, SrcTy: InputVT, CostKind, Mask: Mask0),
6835 b: TTI.getShuffleCost(Kind, DstTy: ResultVT, SrcTy: InputVT, CostKind, Mask: Mask1));
6836 auto NewCost = TTI.getShuffleCost(Kind, DstTy: InputVT, SrcTy: InputVT, CostKind, Mask: NewMask) +
6837 TTI.getShuffleCost(Kind, DstTy: ResultVT, SrcTy: InputVT, CostKind, Mask: Mask1);
6838
6839 LLVM_DEBUG(dbgs() << "Found a phi of mergeable shuffles: " << I
6840 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
6841 << "\n");
6842
6843 if (NewCost > OldCost)
6844 return false;
6845
6846 // Create new shuffles and narrowed phi.
6847 auto Builder = IRBuilder(Shuf);
6848 Builder.SetCurrentDebugLocation(Shuf->getDebugLoc());
6849 auto *PoisonVal = PoisonValue::get(T: InputVT);
6850 auto *NewShuf0 = Builder.CreateShuffleVector(V1: Op, V2: PoisonVal, Mask: NewMask);
6851 Worklist.push(I: cast<Instruction>(Val: NewShuf0));
6852
6853 Builder.SetInsertPoint(Phi);
6854 Builder.SetCurrentDebugLocation(Phi->getDebugLoc());
6855 auto *NewPhi = Builder.CreatePHI(Ty: NewShuf0->getType(), NumReservedValues: 2u);
6856 NewPhi->addIncoming(V: NewShuf0, BB: Phi->getIncomingBlock(i: 0u));
6857 NewPhi->addIncoming(V: Op, BB: Phi->getIncomingBlock(i: 1u));
6858
6859 Builder.SetInsertPoint(*NewPhi->getInsertionPointAfterDef());
6860 PoisonVal = PoisonValue::get(T: NewPhi->getType());
6861 auto *NewShuf1 = Builder.CreateShuffleVector(V1: NewPhi, V2: PoisonVal, Mask: Mask1);
6862
6863 replaceValue(Old&: *Phi, New&: *NewShuf1);
6864 return true;
6865}
6866
6867/// This is the entry point for all transforms. Pass manager differences are
6868/// handled in the callers of this function.
6869bool VectorCombine::run() {
6870 if (DisableVectorCombine)
6871 return false;
6872
6873 // Don't attempt vectorization if the target does not support vectors.
6874 if (!TTI.getNumberOfRegisters(ClassID: TTI.getRegisterClassForType(/*Vector*/ true)))
6875 return false;
6876
6877 LLVM_DEBUG(dbgs() << "\n\nVECTORCOMBINE on " << F.getName() << "\n");
6878
6879 auto FoldInst = [this](Instruction &I) {
6880 Builder.SetInsertPoint(&I);
6881 bool IsVectorType = isa<VectorType>(Val: I.getType());
6882 bool IsFixedVectorType = isa<FixedVectorType>(Val: I.getType());
6883 auto Opcode = I.getOpcode();
6884
6885 LLVM_DEBUG(dbgs() << "VC: Visiting: " << I << '\n');
6886
6887 // These folds should be beneficial regardless of when this pass is run
6888 // in the optimization pipeline.
6889 // The type checking is for run-time efficiency. We can avoid wasting time
6890 // dispatching to folding functions if there's no chance of matching.
6891 if (IsFixedVectorType) {
6892 switch (Opcode) {
6893 case Instruction::InsertElement:
6894 if (vectorizeLoadInsert(I))
6895 return true;
6896 break;
6897 case Instruction::ShuffleVector:
6898 if (widenSubvectorLoad(I))
6899 return true;
6900 break;
6901 default:
6902 break;
6903 }
6904 }
6905
6906 // This transform works with scalable and fixed vectors
6907 // TODO: Identify and allow other scalable transforms
6908 if (IsVectorType) {
6909 if (scalarizeOpOrCmp(I))
6910 return true;
6911 if (scalarizeLoad(I))
6912 return true;
6913 if (scalarizeExtExtract(I))
6914 return true;
6915 if (foldInterleaveIntrinsics(I))
6916 return true;
6917 if (foldBitcastOfVPLoad(I))
6918 return true;
6919 }
6920
6921 if (foldDeinterleaveIntrinsics(I))
6922 return true;
6923
6924 if (Opcode == Instruction::Store)
6925 if (foldInsertElementsToStores(I))
6926 return true;
6927
6928 // If this is an early pipeline invocation of this pass, we are done.
6929 if (TryEarlyFoldsOnly)
6930 return false;
6931
6932 if (Opcode == Instruction::Call)
6933 if (foldBitOrderReverseAndSwap(I))
6934 return true;
6935 if (Opcode == Instruction::BitCast)
6936 if (foldBitOrderReverseAndSwap(I))
6937 return true;
6938
6939 // Otherwise, try folds that improve codegen but may interfere with
6940 // early IR canonicalizations.
6941 // The type checking is for run-time efficiency. We can avoid wasting time
6942 // dispatching to folding functions if there's no chance of matching.
6943 if (IsFixedVectorType) {
6944 switch (Opcode) {
6945 case Instruction::InsertElement:
6946 if (foldInsExtFNeg(I))
6947 return true;
6948 if (foldInsExtBinop(I))
6949 return true;
6950 if (foldInsExtVectorToShuffle(I))
6951 return true;
6952 break;
6953 case Instruction::ShuffleVector:
6954 if (foldPermuteOfBinops(I))
6955 return true;
6956 if (foldShuffleOfBinops(I))
6957 return true;
6958 if (foldShuffleOfSelects(I))
6959 return true;
6960 if (foldShuffleOfCastops(I))
6961 return true;
6962 if (foldShuffleOfShuffles(I))
6963 return true;
6964 if (foldPermuteOfIntrinsic(I))
6965 return true;
6966 if (foldShufflesOfLengthChangingShuffles(I))
6967 return true;
6968 if (foldShuffleOfIntrinsics(I))
6969 return true;
6970 if (foldSelectShuffle(I))
6971 return true;
6972 if (foldShuffleToIdentity(I))
6973 return true;
6974 break;
6975 case Instruction::Load:
6976 if (shrinkLoadForShuffles(I))
6977 return true;
6978 break;
6979 case Instruction::BitCast:
6980 if (foldBitcastShuffle(I))
6981 return true;
6982 if (foldSelectsFromBitcast(I))
6983 return true;
6984 break;
6985 case Instruction::And:
6986 case Instruction::Or:
6987 case Instruction::Xor:
6988 if (foldBitOpOfCastops(I))
6989 return true;
6990 if (foldBitOpOfCastConstant(I))
6991 return true;
6992 break;
6993 case Instruction::PHI:
6994 if (shrinkPhiOfShuffles(I))
6995 return true;
6996 break;
6997 default:
6998 if (shrinkType(I))
6999 return true;
7000 break;
7001 }
7002 } else {
7003 switch (Opcode) {
7004 case Instruction::Call:
7005 if (foldShuffleFromReductions(I))
7006 return true;
7007 if (foldCastFromReductions(I))
7008 return true;
7009 break;
7010 case Instruction::ExtractElement:
7011 if (foldShuffleChainsToReduce(I))
7012 return true;
7013 break;
7014 case Instruction::ICmp:
7015 if (foldSignBitReductionCmp(I))
7016 return true;
7017 if (foldICmpEqZeroVectorReduce(I))
7018 return true;
7019 if (foldReductionZeroTest(I))
7020 return true;
7021 if (foldEquivalentReductionCmp(I))
7022 return true;
7023 if (foldReduceAddCmpZero(I))
7024 return true;
7025 [[fallthrough]];
7026 case Instruction::FCmp:
7027 if (foldExtractExtract(I))
7028 return true;
7029 break;
7030 case Instruction::Or:
7031 if (foldConcatOfBoolMasks(I))
7032 return true;
7033 [[fallthrough]];
7034 default:
7035 if (Instruction::isBinaryOp(Opcode)) {
7036 if (foldExtractExtract(I))
7037 return true;
7038 if (foldExtractedCmps(I))
7039 return true;
7040 if (foldBinopOfReductions(I))
7041 return true;
7042 }
7043 break;
7044 }
7045 }
7046 return false;
7047 };
7048
7049 bool MadeChange = false;
7050 for (BasicBlock &BB : F) {
7051 // Ignore unreachable basic blocks.
7052 if (!DT.isReachableFromEntry(A: &BB))
7053 continue;
7054 // Use early increment range so that we can erase instructions in loop.
7055 // make_early_inc_range is not applicable here, as the next iterator may
7056 // be invalidated by RecursivelyDeleteTriviallyDeadInstructions.
7057 // We manually maintain the next instruction and update it when it is about
7058 // to be deleted.
7059 Instruction *I = &BB.front();
7060 while (I) {
7061 NextInst = I->getNextNode();
7062 if (!I->isDebugOrPseudoInst())
7063 MadeChange |= FoldInst(*I);
7064 I = NextInst;
7065 }
7066 }
7067
7068 NextInst = nullptr;
7069
7070 while (!Worklist.isEmpty()) {
7071 Instruction *I = Worklist.removeOne();
7072 if (!I)
7073 continue;
7074
7075 if (isInstructionTriviallyDead(I)) {
7076 eraseInstruction(I&: *I);
7077 continue;
7078 }
7079
7080 MadeChange |= FoldInst(*I);
7081 }
7082
7083 return MadeChange;
7084}
7085
7086PreservedAnalyses VectorCombinePass::run(Function &F,
7087 FunctionAnalysisManager &FAM) {
7088 auto &AC = FAM.getResult<AssumptionAnalysis>(IR&: F);
7089 TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(IR&: F);
7090 DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(IR&: F);
7091 AAResults &AA = FAM.getResult<AAManager>(IR&: F);
7092 const DataLayout *DL = &F.getDataLayout();
7093 TTI::TargetCostKind CostKind =
7094 F.hasOptSize() ? TTI::TCK_CodeSize : TTI::TCK_RecipThroughput;
7095 VectorCombine Combiner(F, TTI, DT, AA, AC, DL, CostKind, TryEarlyFoldsOnly);
7096 if (!Combiner.run())
7097 return PreservedAnalyses::all();
7098 PreservedAnalyses PA;
7099 PA.preserveSet<CFGAnalyses>();
7100 return PA;
7101}
7102