1//===- SeedCollection.cpp - Seed collection pass --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h"
10#include "llvm/Analysis/TargetTransformInfo.h"
11#include "llvm/SandboxIR/Module.h"
12#include "llvm/Transforms/Vectorize/SandboxVectorizer/RegionWithScore.h"
13#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h"
14#include "llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h"
15#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
16
17namespace llvm {
18
19static cl::opt<unsigned>
20 OverrideVecRegBits("sbvec-vec-reg-bits", cl::init(Val: 0), cl::Hidden,
21 cl::desc("Override the vector register size in bits, "
22 "which is otherwise found by querying TTI."));
23static cl::opt<bool>
24 AllowNonPow2("sbvec-allow-non-pow2", cl::init(Val: false), cl::Hidden,
25 cl::desc("Allow non-power-of-2 vectorization."));
26
27#define LoadSeedsDef "loads"
28#define StoreSeedsDef "stores"
29cl::opt<std::string> CollectSeeds(
30 "sbvec-collect-seeds", cl::init(StoreSeedsDef), cl::Hidden,
31 cl::desc("Collect these seeds. Use empty for none or a comma-separated "
32 "list of '" StoreSeedsDef "' and '" LoadSeedsDef "'."));
33
34namespace sandboxir {
35
36SeedCollection::SeedCollection(StringRef Pipeline, StringRef AuxArg)
37 : FunctionPass("seed-collection"),
38 RPM("rpm", Pipeline, SandboxVectorizerPassBuilder::createRegionPass) {
39 if (!AuxArg.empty()) {
40 if (AuxArg != DiffTypesArgStr) {
41 std::string ErrStr;
42 raw_string_ostream ErrSS(ErrStr);
43 ErrSS << "SeedCollection only supports '" << DiffTypesArgStr
44 << "' aux argument!\n";
45 reportFatalUsageError(reason: ErrStr.c_str());
46 }
47 AllowDiffTypes = true;
48 }
49}
50
51bool SeedCollection::runOnFunction(Function &F, const Analyses &A) {
52 bool Change = false;
53 const auto &DL = F.getParent()->getDataLayout();
54 bool CollectStores = CollectSeeds.find(StoreSeedsDef) != std::string::npos;
55 bool CollectLoads = CollectSeeds.find(LoadSeedsDef) != std::string::npos;
56
57 // TODO: Start from innermost BBs first
58 for (auto &BB : F) {
59 SeedCollector SC(&BB, A.getScalarEvolution(), CollectStores, CollectLoads,
60 AllowDiffTypes);
61 for (auto &SeedRange : {SC.getStoreSeeds(), SC.getLoadSeeds()}) {
62 for (SeedBundle &Seeds : SeedRange) {
63 if (Seeds.allUsed())
64 continue;
65 unsigned FirstUnusedIdx = Seeds.getFirstUnusedElementIdx();
66 unsigned ElmBits =
67 Utils::getNumBits(Ty: VecUtils::getElementType(Ty: Utils::getExpectedType(
68 V: Seeds[FirstUnusedIdx])),
69 DL);
70 unsigned AS = getLoadStoreAddressSpace(I: Seeds[FirstUnusedIdx]);
71 unsigned VecRegBits = OverrideVecRegBits != 0
72 ? OverrideVecRegBits
73 : A.getTTI().getLoadStoreVecRegBitWidth(AddrSpace: AS);
74
75 auto DivideBy2 = [](unsigned Num) {
76 auto Floor = VecUtils::getFloorPowerOf2(Num);
77 if (Floor == Num)
78 return Floor / 2;
79 return Floor;
80 };
81 // Try to create the largest vector supported by the target. If it fails
82 // reduce the vector size by half.
83 for (unsigned SliceElms = std::min(a: VecRegBits / ElmBits,
84 b: Seeds.getNumUnusedBits() / ElmBits);
85 SliceElms >= 2u; SliceElms = DivideBy2(SliceElms)) {
86 if (Seeds.allUsed())
87 break;
88 // Keep trying offsets after FirstUnusedElementIdx, until we vectorize
89 // the slice. This could be quite expensive, so we enforce a limit.
90 for (unsigned Offset = Seeds.getFirstUnusedElementIdx(),
91 OE = Seeds.size();
92 Offset + 1 < OE; Offset += 1) {
93 // Seeds are getting used as we vectorize, so skip them.
94 if (Seeds.isUsed(Element: Offset))
95 continue;
96 if (Seeds.allUsed())
97 break;
98
99 auto SeedSlice =
100 Seeds.getSlice(StartIdx: Offset, MaxVecRegBits: SliceElms * ElmBits, ForcePowOf2: !AllowNonPow2);
101 if (SeedSlice.empty())
102 continue;
103
104 assert(SeedSlice.size() >= 2 && "Should have been rejected!");
105
106 // Create a region containing the seed slice.
107 auto &Ctx = F.getContext();
108 RegionWithScore Rgn(Ctx, A.getTTI());
109 Rgn.setAux(SeedSlice);
110 // Run the region pass pipeline.
111 Change |= RPM.runOnRegion(R&: Rgn, A);
112 Rgn.clearAux();
113 }
114 }
115 }
116 }
117 }
118 return Change;
119}
120} // namespace sandboxir
121} // namespace llvm
122