1//===- BundleVec.cpp - A bundle-forming SLP-style vectorizer pass ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BundleVec.h"
10#include "llvm/ADT/SmallVector.h"
11#include "llvm/SandboxIR/Function.h"
12#include "llvm/SandboxIR/Instruction.h"
13#include "llvm/SandboxIR/Module.h"
14#include "llvm/SandboxIR/Region.h"
15#include "llvm/SandboxIR/Utils.h"
16#include "llvm/Support/ErrorHandling.h"
17#include "llvm/Transforms/Vectorize/SandboxVectorizer/Debug.h"
18#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
19
20namespace llvm {
21
22#ifndef NDEBUG
23static cl::opt<bool>
24 AlwaysVerify("sbvec-always-verify", cl::init(false), cl::Hidden,
25 cl::desc("Helps find bugs by verifying the IR whenever we "
26 "emit new instructions (*very* expensive)."));
27#endif // NDEBUG
28
29static constexpr unsigned long StopAtDisabled =
30 std::numeric_limits<unsigned long>::max();
31static cl::opt<unsigned long>
32 StopAt("sbvec-stop-at", cl::init(Val: StopAtDisabled), cl::Hidden,
33 cl::desc("Vectorize if the invocation count is < than this. 0 "
34 "disables vectorization."));
35
36static constexpr unsigned long StopBundleDisabled =
37 std::numeric_limits<unsigned long>::max();
38static cl::opt<unsigned long>
39 StopBundle("sbvec-stop-bndl", cl::init(Val: StopBundleDisabled), cl::Hidden,
40 cl::desc("Vectorize up to this many bundles."));
41
42namespace sandboxir {
43
44static BundleTy getOperand(BndlRef<Value *> Bndl, unsigned OpIdx) {
45 BundleTy Operands;
46 for (Value *BndlV : Bndl) {
47 auto *BndlI = cast<Instruction>(Val: BndlV);
48 Operands.push_back(Elt: BndlI->getOperand(OpIdx));
49 }
50 return Operands;
51}
52
53Value *BundleVec::createVectorInstr(BndlRef<Value *> Bndl,
54 BndlRef<Value *> Operands) {
55 auto CreateVectorInstr = [](BndlRef<Value *> Bndl,
56 BndlRef<Value *> Operands) -> Value * {
57 assert(all_of(Bndl, [](auto *V) { return isa<Instruction>(V); }) &&
58 "Expect Instructions!");
59 auto &Ctx = Bndl[0]->getContext();
60
61 Type *ScalarTy = VecUtils::getElementType(Ty: Utils::getExpectedType(V: Bndl[0]));
62 auto *VecTy = VecUtils::getWideType(ElemTy: ScalarTy, NumElts: VecUtils::getNumLanes(Bndl));
63
64 BasicBlock::iterator WhereIt = VecUtils::getInsertPointAfterInstrs(
65 Vals: Bndl, BB: cast<Instruction>(Val: Bndl[0])->getParent());
66
67 auto Opcode = cast<Instruction>(Val: Bndl[0])->getOpcode();
68 switch (Opcode) {
69 case Instruction::Opcode::ZExt:
70 case Instruction::Opcode::SExt:
71 case Instruction::Opcode::FPToUI:
72 case Instruction::Opcode::FPToSI:
73 case Instruction::Opcode::FPExt:
74 case Instruction::Opcode::PtrToInt:
75 case Instruction::Opcode::IntToPtr:
76 case Instruction::Opcode::SIToFP:
77 case Instruction::Opcode::UIToFP:
78 case Instruction::Opcode::Trunc:
79 case Instruction::Opcode::FPTrunc:
80 case Instruction::Opcode::BitCast: {
81 assert(Operands.size() == 1u && "Casts are unary!");
82 return CastInst::create(DestTy: VecTy, Op: Opcode, Operand: Operands[0], Pos: WhereIt, Ctx,
83 Name: "VCast");
84 }
85 case Instruction::Opcode::FCmp:
86 case Instruction::Opcode::ICmp: {
87 auto Pred = cast<CmpInst>(Val: Bndl[0])->getPredicate();
88 assert(all_of(drop_begin(Bndl),
89 [Pred](auto *SBV) {
90 return cast<CmpInst>(SBV)->getPredicate() == Pred;
91 }) &&
92 "Expected same predicate across bundle.");
93 return CmpInst::create(Pred, S1: Operands[0], S2: Operands[1], Pos: WhereIt, Ctx,
94 Name: "VCmp");
95 }
96 case Instruction::Opcode::Select: {
97 return SelectInst::create(Cond: Operands[0], True: Operands[1], False: Operands[2], Pos: WhereIt,
98 Ctx, Name: "Vec");
99 }
100 case Instruction::Opcode::FNeg: {
101 auto *UOp0 = cast<UnaryOperator>(Val: Bndl[0]);
102 auto OpC = UOp0->getOpcode();
103 return UnaryOperator::createWithCopiedFlags(Op: OpC, OpV: Operands[0], CopyFrom: UOp0,
104 Pos: WhereIt, Ctx, Name: "Vec");
105 }
106 case Instruction::Opcode::Add:
107 case Instruction::Opcode::FAdd:
108 case Instruction::Opcode::Sub:
109 case Instruction::Opcode::FSub:
110 case Instruction::Opcode::Mul:
111 case Instruction::Opcode::FMul:
112 case Instruction::Opcode::UDiv:
113 case Instruction::Opcode::SDiv:
114 case Instruction::Opcode::FDiv:
115 case Instruction::Opcode::URem:
116 case Instruction::Opcode::SRem:
117 case Instruction::Opcode::FRem:
118 case Instruction::Opcode::Shl:
119 case Instruction::Opcode::LShr:
120 case Instruction::Opcode::AShr:
121 case Instruction::Opcode::And:
122 case Instruction::Opcode::Or:
123 case Instruction::Opcode::Xor: {
124 auto *BinOp0 = cast<BinaryOperator>(Val: Bndl[0]);
125 auto *LHS = Operands[0];
126 auto *RHS = Operands[1];
127 return BinaryOperator::createWithCopiedFlags(
128 Op: BinOp0->getOpcode(), LHS, RHS, CopyFrom: BinOp0, Pos: WhereIt, Ctx, Name: "Vec");
129 }
130 case Instruction::Opcode::Load: {
131 auto *Ld0 = cast<LoadInst>(Val: Bndl[0]);
132 Value *Ptr = Ld0->getPointerOperand();
133 return LoadInst::create(Ty: VecTy, Ptr, Align: Ld0->getAlign(), Pos: WhereIt, Ctx,
134 Name: "VecL");
135 }
136 case Instruction::Opcode::Store: {
137 auto Align = cast<StoreInst>(Val: Bndl[0])->getAlign();
138 Value *Val = Operands[0];
139 Value *Ptr = Operands[1];
140 return StoreInst::create(V: Val, Ptr, Align, Pos: WhereIt, Ctx);
141 }
142 case Instruction::Opcode::UncondBr:
143 case Instruction::Opcode::CondBr:
144 case Instruction::Opcode::Ret:
145 case Instruction::Opcode::PHI:
146 case Instruction::Opcode::AddrSpaceCast:
147 case Instruction::Opcode::Call:
148 case Instruction::Opcode::GetElementPtr:
149 llvm_unreachable("Unimplemented");
150 break;
151 default:
152 llvm_unreachable("Unimplemented");
153 break;
154 }
155 llvm_unreachable("Missing switch case!");
156 // TODO: Propagate debug info.
157 };
158
159 auto *NewI = CreateVectorInstr(Bndl, Operands);
160 LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "New instr: " << *NewI << "\n");
161 return NewI;
162}
163
164Value *BundleVec::createShuffle(Value *VecOp, const ShuffleMask &Mask,
165 BasicBlock *UserBB) {
166 BasicBlock::iterator WhereIt =
167 VecUtils::getInsertPointAfterInstrs(Vals: {VecOp}, BB: UserBB);
168 return ShuffleVectorInst::create(V1: VecOp, V2: VecOp, Mask, Pos: WhereIt,
169 Ctx&: VecOp->getContext(), Name: "VShuf");
170}
171
172Value *BundleVec::createPack(BndlRef<Value *> ToPack, BasicBlock *UserBB) {
173 BasicBlock::iterator WhereIt =
174 VecUtils::getInsertPointAfterInstrs(Vals: ToPack, BB: UserBB);
175
176 Type *ScalarTy = VecUtils::getCommonScalarType(Bndl: ToPack);
177 unsigned Lanes = VecUtils::getNumLanes(Bndl: ToPack);
178 Type *VecTy = VecUtils::getWideType(ElemTy: ScalarTy, NumElts: Lanes);
179
180 // Create a series of pack instructions.
181 Value *LastInsert = PoisonValue::get(T: VecTy);
182
183 Context &Ctx = ToPack[0]->getContext();
184
185 unsigned InsertIdx = 0;
186 for (Value *Elm : ToPack) {
187 // An element can be either scalar or vector. We need to generate different
188 // IR for each case.
189 if (Elm->getType()->isVectorTy()) {
190 unsigned NumElms =
191 cast<FixedVectorType>(Val: Elm->getType())->getNumElements();
192 for (auto ExtrLane : seq<int>(Begin: 0, End: NumElms)) {
193 // We generate extract-insert pairs, for each lane in `Elm`.
194 Constant *ExtrLaneC =
195 ConstantInt::getSigned(Ty: Type::getInt32Ty(Ctx), V: ExtrLane);
196 // This may return a Constant if Elm is a Constant.
197 auto *ExtrI =
198 ExtractElementInst::create(Vec: Elm, Idx: ExtrLaneC, Pos: WhereIt, Ctx, Name: "VPack");
199 if (!isa<Constant>(Val: ExtrI))
200 WhereIt = std::next(x: cast<Instruction>(Val: ExtrI)->getIterator());
201 Constant *InsertLaneC =
202 ConstantInt::getSigned(Ty: Type::getInt32Ty(Ctx), V: InsertIdx++);
203 // This may also return a Constant if ExtrI is a Constant.
204 auto *InsertI = InsertElementInst::create(
205 Vec: LastInsert, NewElt: ExtrI, Idx: InsertLaneC, Pos: WhereIt, Ctx, Name: "VPack");
206 LastInsert = InsertI;
207 if (!isa<Constant>(Val: InsertI))
208 WhereIt = std::next(x: cast<Instruction>(Val: LastInsert)->getIterator());
209 }
210 } else {
211 Constant *InsertLaneC =
212 ConstantInt::getSigned(Ty: Type::getInt32Ty(Ctx), V: InsertIdx++);
213 // This may be folded into a Constant if LastInsert is a Constant. In
214 // that case we only collect the last constant.
215 LastInsert = InsertElementInst::create(Vec: LastInsert, NewElt: Elm, Idx: InsertLaneC,
216 Pos: WhereIt, Ctx, Name: "Pack");
217 if (auto *NewI = dyn_cast<Instruction>(Val: LastInsert))
218 WhereIt = std::next(x: NewI->getIterator());
219 }
220 }
221 return LastInsert;
222}
223
224Action *BundleVec::vectorizeRec(BndlRef<Value *> Bndl,
225 BndlRef<Value *> UserBndl, unsigned Depth,
226 LegalityAnalysis &Legality) {
227 bool StopForDebug =
228 DebugBndlCnt++ >= StopBundle && StopBundle != StopBundleDisabled;
229 LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "canVectorize() Bundle:\n";
230 VecUtils::dump(Bndl));
231 const auto &LegalityRes = StopForDebug ? Legality.getForcedPackForDebugging()
232 : Legality.canVectorize(Bndl);
233 LLVM_DEBUG(dbgs() << DEBUG_PREFIX << "Legality: " << LegalityRes << "\n");
234
235 if (Dir == SchedDirection::TopDown) {
236 // A non-Widen result means we can't extend the vectorized region into
237 // this bundle, so leave its instructions scalar and don't record an
238 // action for it.
239 if (LegalityRes.getSubclassID() != LegalityResultID::Widen)
240 return nullptr;
241
242 auto ActionPtr =
243 std::make_unique<Action>(args: &LegalityRes, args&: Bndl, args: BndlRef<Value *>(), args&: Depth);
244 Action *Action = ActionPtr.get();
245 IMaps->registerVector(Origs: Bndl, Vec: Action);
246 Actions.push_back(ActPtr: std::move(ActionPtr));
247
248 // Walk down the def-use chain. Each lane in \p Bndl may feed several
249 // users, so we form every compatible user bundle and recurse into each
250 // one.
251 SmallPtrSet<Instruction *, 4> Claimed;
252 for (const auto &NextUserBndl :
253 VecUtils::getNextUserBundles(Bndl, IMaps: *IMaps, Claimed))
254 vectorizeRec(Bndl: NextUserBndl, UserBndl: Bndl, Depth: Depth + 1, Legality);
255
256 return Action;
257 }
258
259 // Bottom up direction
260 auto ActionPtr =
261 std::make_unique<Action>(args: &LegalityRes, args&: Bndl, args&: UserBndl, args&: Depth);
262 SmallVector<Action *> Operands;
263 switch (LegalityRes.getSubclassID()) {
264 case LegalityResultID::Widen: {
265 auto *I = cast<Instruction>(Val: Bndl[0]);
266 switch (I->getOpcode()) {
267 case Instruction::Opcode::Load:
268 break;
269 case Instruction::Opcode::Store: {
270 // Don't recurse towards the pointer operand.
271 Action *OpA =
272 vectorizeRec(Bndl: getOperand(Bndl, OpIdx: 0), UserBndl: Bndl, Depth: Depth + 1, Legality);
273 Operands.push_back(Elt: OpA);
274 break;
275 }
276 default:
277 // Visit all operands.
278 for (auto OpIdx : seq<unsigned>(Size: I->getNumOperands())) {
279 Action *OpA =
280 vectorizeRec(Bndl: getOperand(Bndl, OpIdx), UserBndl: Bndl, Depth: Depth + 1, Legality);
281 Operands.push_back(Elt: OpA);
282 }
283 break;
284 }
285 // Update the maps to mark Bndl as "vectorized".
286 IMaps->registerVector(Origs: Bndl, Vec: ActionPtr.get());
287 break;
288 }
289 case LegalityResultID::DiamondReuse:
290 case LegalityResultID::DiamondReuseWithShuffle:
291 case LegalityResultID::DiamondReuseMultiInput:
292 case LegalityResultID::Pack:
293 break;
294 }
295 // Create actions in post-order.
296 ActionPtr->Operands = std::move(Operands);
297 auto *Action = ActionPtr.get();
298 Actions.push_back(ActPtr: std::move(ActionPtr));
299 return Action;
300}
301
302#ifndef NDEBUG
303void BundleVec::ActionsVector::print(raw_ostream &OS) const {
304 for (auto [Idx, Action] : enumerate(Actions)) {
305 Action->print(OS);
306 OS << "\n";
307 }
308}
309void BundleVec::ActionsVector::dump() const { print(dbgs()); }
310#endif // NDEBUG
311
312void BundleVec::emitUnpacksForExternalUses(BndlRef<Value *> Bndl, Value *Vec) {
313 // Find where we should emit the unpacks.
314 BasicBlock::iterator WhereIt;
315 if (auto *VecI = dyn_cast<Instruction>(Val: Vec)) {
316 WhereIt = std::next(x: VecI->getIterator());
317 } else {
318 // If Vec is a constant then it should be safe to emit the unpacks at the
319 // top of the block.
320 // Note: Extracts from constants are usually folded to constants.
321 assert(isa<Constant>(Vec) && "Expected constant!");
322 assert(isa<Instruction>(Bndl[0]) &&
323 "A widened Bndl should contain instrs!");
324 BasicBlock *BB = cast<Instruction>(Val: Bndl[0])->getParent();
325 WhereIt =
326 BB->empty()
327 ? BB->begin()
328 : std::next(
329 x: VecUtils::getLastPHIOrSelf(I: &*BB->begin())->getIterator());
330 }
331
332 for (auto [Lane, Elm] : VecUtils::enumerateLanes(Range: Bndl)) {
333 // Only redirect the external (non-vectorized) uses to an unpack and leave
334 // the vectorized users untouched. A blanket replaceAllUsesWith() would
335 // also rewrite the operands of users we are going to vectorize but have
336 // not emitted yet (in the top-down direction a user bundle is emitted
337 // after its operand bundle), which would corrupt those operands.
338 auto IsExternal = [this](const Use &U) {
339 return !IMaps->isVectorized(Orig: U.getUser());
340 };
341 // Don't emit a dead unpack if all uses are internal to the vector region.
342 if (none_of(Range: Elm->uses(), P: IsExternal))
343 continue;
344 auto *UnpackV = VecUtils::unpack(FromVec: Vec, ExtrTy: Elm->getType(), Lane, WhereIt);
345 Elm->replaceUsesWithIf(OtherV: UnpackV, ShouldReplace: IsExternal);
346 }
347}
348
349Value *BundleVec::emitVectors() {
350 Value *NewVec = nullptr;
351 for (const auto &ActionPtr : Actions) {
352 BndlRef<Value *> Bndl = ActionPtr->Bndl;
353 BndlRef<Value *> UserBndl = ActionPtr->UserBndl;
354 const LegalityResult &LegalityRes = *ActionPtr->LegalityRes;
355 unsigned Depth = ActionPtr->Depth;
356 auto *UserBB = !UserBndl.empty()
357 ? cast<Instruction>(Val: UserBndl.front())->getParent()
358 : cast<Instruction>(Val: Bndl[0])->getParent();
359
360 switch (LegalityRes.getSubclassID()) {
361 case LegalityResultID::Widen: {
362 auto *I = cast<Instruction>(Val: Bndl[0]);
363 SmallVector<Value *, 2> VecOperands;
364 if (Dir == SchedDirection::BottomUp) {
365 switch (I->getOpcode()) {
366 case Instruction::Opcode::Load:
367 VecOperands.push_back(Elt: cast<LoadInst>(Val: I)->getPointerOperand());
368 break;
369 case Instruction::Opcode::Store:
370 VecOperands.push_back(Elt: ActionPtr->Operands[0]->Vec);
371 VecOperands.push_back(Elt: cast<StoreInst>(Val: I)->getPointerOperand());
372 break;
373 default:
374 for (Action *OpA : ActionPtr->Operands)
375 VecOperands.push_back(Elt: OpA->Vec);
376 break;
377 }
378 } else {
379 switch (I->getOpcode()) {
380 case Instruction::Opcode::Load:
381 VecOperands.push_back(Elt: cast<LoadInst>(Val: I)->getPointerOperand());
382 break;
383 case Instruction::Opcode::Store: {
384 auto OpBndl = getOperand(Bndl, OpIdx: 0);
385 if (Action *OpA = IMaps->getVectorForOrig(Orig: OpBndl[0]))
386 VecOperands.push_back(Elt: OpA->Vec);
387 else
388 VecOperands.push_back(Elt: createPack(ToPack: OpBndl, UserBB));
389 VecOperands.push_back(Elt: cast<StoreInst>(Val: I)->getPointerOperand());
390 break;
391 }
392 default:
393 for (unsigned OpIdx = 0; OpIdx < I->getNumOperands(); ++OpIdx) {
394 BundleTy OpBndl = getOperand(Bndl, OpIdx);
395 if (Action *OpA = IMaps->getVectorForOrig(Orig: OpBndl[0]))
396 VecOperands.push_back(Elt: OpA->Vec);
397 else
398 VecOperands.push_back(Elt: createPack(ToPack: OpBndl, UserBB));
399 }
400 break;
401 }
402 }
403 NewVec = createVectorInstr(Bndl: ActionPtr->Bndl, Operands: VecOperands);
404 // Collect any potentially dead scalar instructions, including the
405 // original scalars and pointer operands of loads/stores.
406 if (NewVec != nullptr)
407 DeadInstrMorgue.collectPotentiallyDeadInstrs(Bndl);
408
409 // Emit unpacks for all external uses, if any.
410 emitUnpacksForExternalUses(Bndl: ActionPtr->Bndl, Vec: NewVec);
411 break;
412 }
413 case LegalityResultID::DiamondReuse: {
414 NewVec = cast<DiamondReuse>(Val: LegalityRes).getVector()->Vec;
415 break;
416 }
417 case LegalityResultID::DiamondReuseWithShuffle: {
418 auto *VecOp = cast<DiamondReuseWithShuffle>(Val: LegalityRes).getVector()->Vec;
419 const ShuffleMask &Mask =
420 cast<DiamondReuseWithShuffle>(Val: LegalityRes).getMask();
421 NewVec = createShuffle(VecOp, Mask, UserBB);
422 assert(NewVec->getType() == VecOp->getType() &&
423 "Expected same type! Bad mask ?");
424 break;
425 }
426 case LegalityResultID::DiamondReuseMultiInput: {
427 const auto &Descr =
428 cast<DiamondReuseMultiInput>(Val: LegalityRes).getCollectDescr();
429 Type *ResTy = VecUtils::getWideType(ElemTy: Bndl[0]->getType(), NumElts: Bndl.size());
430
431 // TODO: Try to get WhereIt without creating a vector.
432 SmallVector<Value *, 4> DescrInstrs;
433 for (const auto &ElmDescr : Descr.getDescrs()) {
434 auto *V = ElmDescr.needsExtract() ? ElmDescr.getValue()->Vec
435 : ElmDescr.getScalar();
436 if (auto *I = dyn_cast<Instruction>(Val: V))
437 DescrInstrs.push_back(Elt: I);
438 }
439 BasicBlock::iterator WhereIt =
440 VecUtils::getInsertPointAfterInstrs(Vals: DescrInstrs, BB: UserBB);
441
442 Value *LastV = PoisonValue::get(T: ResTy);
443 Context &Ctx = LastV->getContext();
444 unsigned Lane = 0;
445 for (const auto &ElmDescr : Descr.getDescrs()) {
446 Value *VecOp = nullptr;
447 Value *ValueToInsert;
448 if (ElmDescr.needsExtract()) {
449 VecOp = ElmDescr.getValue()->Vec;
450 ConstantInt *IdxC =
451 ConstantInt::get(Ty: Type::getInt32Ty(Ctx), V: ElmDescr.getExtractIdx());
452 ValueToInsert = ExtractElementInst::create(
453 Vec: VecOp, Idx: IdxC, Pos: WhereIt, Ctx&: VecOp->getContext(), Name: "VExt");
454 } else {
455 ValueToInsert = ElmDescr.getScalar();
456 }
457 auto NumLanesToInsert = VecUtils::getNumLanes(V: ValueToInsert);
458 if (NumLanesToInsert == 1) {
459 // If we are inserting a scalar element then we need a single insert.
460 // %VIns = insert %DstVec, %SrcScalar, Lane
461 ConstantInt *LaneC = ConstantInt::get(Ty: Type::getInt32Ty(Ctx), V: Lane);
462 LastV = InsertElementInst::create(Vec: LastV, NewElt: ValueToInsert, Idx: LaneC,
463 Pos: WhereIt, Ctx, Name: "VIns");
464 } else {
465 // If we are inserting a vector element then we need to extract and
466 // insert each vector element one by one with a chain of extracts and
467 // inserts, for example:
468 // %VExt0 = extract %SrcVec, 0
469 // %VIns0 = insert %DstVec, %Vect0, Lane + 0
470 // %VExt1 = extract %SrcVec, 1
471 // %VIns1 = insert %VIns0, %Vect0, Lane + 1
472 for (unsigned LnCnt = 0; LnCnt != NumLanesToInsert; ++LnCnt) {
473 auto *ExtrIdxC = ConstantInt::get(Ty: Type::getInt32Ty(Ctx), V: LnCnt);
474 auto *ExtrI = ExtractElementInst::create(Vec: ValueToInsert, Idx: ExtrIdxC,
475 Pos: WhereIt, Ctx, Name: "VExt");
476 unsigned InsLane = Lane + LnCnt;
477 auto *InsLaneC = ConstantInt::get(Ty: Type::getInt32Ty(Ctx), V: InsLane);
478 LastV = InsertElementInst::create(Vec: LastV, NewElt: ExtrI, Idx: InsLaneC, Pos: WhereIt,
479 Ctx, Name: "VIns");
480 }
481 }
482 Lane += NumLanesToInsert;
483 }
484 NewVec = LastV;
485 break;
486 }
487 case LegalityResultID::Pack: {
488 // If we can't vectorize the seeds then just return.
489 if (Depth == 0)
490 return nullptr;
491 NewVec = createPack(ToPack: Bndl, UserBB);
492 break;
493 }
494 }
495 if (NewVec != nullptr) {
496 Change = true;
497 ActionPtr->Vec = NewVec;
498 }
499#ifndef NDEBUG
500 if (AlwaysVerify) {
501 // This helps find broken IR by constantly verifying the function. Note
502 // that this is very expensive and should only be used for debugging.
503 Instruction *I0 = isa<Instruction>(Bndl[0])
504 ? cast<Instruction>(Bndl[0])
505 : cast<Instruction>(UserBndl[0]);
506 assert(!Utils::verifyFunction(I0->getParent()->getParent(), dbgs()) &&
507 "Broken function!");
508 }
509#endif // NDEBUG
510 }
511 return NewVec;
512}
513
514bool BundleVec::tryVectorize(BndlRef<Value *> Bndl,
515 LegalityAnalysis &Legality) {
516 Change = false;
517 if (LLVM_UNLIKELY(InvocationCnt++ >= StopAt && StopAt != StopAtDisabled))
518 return false;
519 Legality.clear();
520 Actions.clear();
521 DebugBndlCnt = 0;
522 vectorizeRec(Bndl, UserBndl: {}, /*Depth=*/0, Legality);
523 LLVM_DEBUG(dbgs() << DEBUG_PREFIX << schedDirectionToStr(Dir)
524 << "Vec: Vectorization Actions:\n";
525 Actions.dump());
526 emitVectors();
527 DeadInstrMorgue.tryEraseDeadInstrs();
528 return Change;
529}
530
531bool BundleVec::runOnRegion(Region &Rgn, const Analyses &A) {
532 const auto &SeedSlice = Rgn.getAux();
533 if (SeedSlice.size() < 2)
534 return false;
535 Function &F = *SeedSlice[0]->getParent()->getParent();
536 IMaps = std::make_unique<InstrMaps>();
537 LegalityAnalysis Legality(A.getAA(), A.getScalarEvolution(),
538 F.getParent()->getDataLayout(), F.getContext(),
539 *IMaps, Dir);
540
541 // TODO: Refactor to remove the unnecessary copy to SeedSliceVals.
542 SmallVector<Value *> SeedSliceVals(SeedSlice.begin(), SeedSlice.end());
543 // Try to vectorize starting from the seed slice. The returned value
544 // is true if we found vectorizable code and generated some vector
545 // code for it. It does not mean that the code is profitable.
546 return tryVectorize(Bndl: SeedSliceVals, Legality);
547}
548
549} // namespace sandboxir
550} // namespace llvm
551