1//===- SeedCollection.cpp - Seed collection pass --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h"
10#include "../../VectorizeOptions.h"
11#include "llvm/Analysis/TargetTransformInfo.h"
12#include "llvm/SandboxIR/Module.h"
13#include "llvm/Transforms/Vectorize/SandboxVectorizer/RegionWithScore.h"
14#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h"
15#include "llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h"
16#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
17
18namespace llvm {
19
20namespace sandboxir {
21
22SeedCollection::SeedCollection(StringRef Pipeline, StringRef AuxArg)
23 : FunctionPass("seed-collection"),
24 RPM("rpm", Pipeline, SandboxVectorizerPassBuilder::createRegionPass) {
25 ArgsRegistry.parse(ArgsStr: AuxArg);
26 // If the user has not specified seed types default to stores.
27 if (!CollectStores && !CollectLoads) {
28 CollectStores = true;
29 }
30}
31
32bool SeedCollection::runOnFunction(Function &F, const Analyses &A) {
33 const VectorizeOptions &Opts =
34 F.getContext().getLibraryOptions().get<VectorizeOptions>();
35 bool Change = false;
36 const auto &DL = F.getParent()->getDataLayout();
37
38 // TODO: Start from innermost BBs first
39 for (auto &BB : F) {
40 SeedCollector SC(&BB, A.getScalarEvolution(), CollectStores, CollectLoads,
41 AllowDiffTypes);
42 for (auto &SeedRange : {SC.getStoreSeeds(), SC.getLoadSeeds()}) {
43 for (SeedBundle &Seeds : SeedRange) {
44 if (Seeds.allUsed())
45 continue;
46 unsigned FirstUnusedIdx = Seeds.getFirstUnusedElementIdx();
47 unsigned ElmBits =
48 Utils::getNumBits(Ty: VecUtils::getElementType(Ty: Utils::getExpectedType(
49 V: Seeds[FirstUnusedIdx])),
50 DL);
51 unsigned AS = getLoadStoreAddressSpace(I: Seeds[FirstUnusedIdx]);
52 unsigned VecRegBits = Opts.sbvec_vec_reg_bits != 0
53 ? Opts.sbvec_vec_reg_bits
54 : A.getTTI().getLoadStoreVecRegBitWidth(AddrSpace: AS);
55
56 auto DivideBy2 = [](unsigned Num) {
57 auto Floor = VecUtils::getFloorPowerOf2(Num);
58 if (Floor == Num)
59 return Floor / 2;
60 return Floor;
61 };
62 // Try to create the largest vector supported by the target. If it fails
63 // reduce the vector size by half.
64 for (unsigned SliceElms = std::min(a: VecRegBits / ElmBits,
65 b: Seeds.getNumUnusedBits() / ElmBits);
66 SliceElms >= 2u; SliceElms = DivideBy2(SliceElms)) {
67 if (Seeds.allUsed())
68 break;
69 // Keep trying offsets after FirstUnusedElementIdx, until we vectorize
70 // the slice. This could be quite expensive, so we enforce a limit.
71 for (unsigned Offset = Seeds.getFirstUnusedElementIdx(),
72 OE = Seeds.size();
73 Offset + 1 < OE; Offset += 1) {
74 // Seeds are getting used as we vectorize, so skip them.
75 if (Seeds.isUsed(Element: Offset))
76 continue;
77 if (Seeds.allUsed())
78 break;
79
80 auto SeedSlice = Seeds.getSlice(StartIdx: Offset, MaxVecRegBits: SliceElms * ElmBits,
81 ForcePowOf2: !Opts.sbvec_allow_non_pow2);
82 if (SeedSlice.empty())
83 continue;
84
85 assert(SeedSlice.size() >= 2 && "Should have been rejected!");
86
87 // Create a region containing the seed slice.
88 auto &Ctx = F.getContext();
89 RegionWithScore Rgn(Ctx, A.getTTI());
90 Rgn.setAux(SeedSlice);
91 // Run the region pass pipeline.
92 Change |= RPM.runOnRegion(R&: Rgn, A);
93 Rgn.clearAux();
94 }
95 }
96 }
97 }
98 }
99 return Change;
100}
101} // namespace sandboxir
102} // namespace llvm
103