1//===-- Target.cpp ----------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "Target.h"
9
10#include "LatencyBenchmarkRunner.h"
11#include "ParallelSnippetGenerator.h"
12#include "PerfHelper.h"
13#include "SerialSnippetGenerator.h"
14#include "UopsBenchmarkRunner.h"
15#include "llvm/ADT/Twine.h"
16#include "llvm/Support/Error.h"
17#include "llvm/TargetParser/SubtargetFeature.h"
18
19namespace llvm {
20namespace exegesis {
21
22cl::OptionCategory Options("llvm-exegesis options");
23cl::OptionCategory BenchmarkOptions("llvm-exegesis benchmark options");
24cl::OptionCategory AnalysisOptions("llvm-exegesis analysis options");
25
26ExegesisTarget::~ExegesisTarget() = default; // anchor.
27
28static ExegesisTarget *FirstTarget = nullptr;
29
30const ExegesisTarget *ExegesisTarget::lookup(Triple TT) {
31 for (const ExegesisTarget *T = FirstTarget; T != nullptr; T = T->Next) {
32 if (T->matchesArch(Arch: TT.getArch()))
33 return T;
34 }
35 return nullptr;
36}
37
38const char *
39ExegesisTarget::getIgnoredOpcodeReasonOrNull(const LLVMState &State,
40 unsigned Opcode) const {
41 const MCInstrDesc &InstrDesc = State.getIC().getInstr(Opcode).Description;
42 if (InstrDesc.isPseudo() || InstrDesc.usesCustomInsertionHook())
43 return "Unsupported opcode: isPseudo/usesCustomInserter";
44 if (InstrDesc.isBranch() || InstrDesc.isIndirectBranch())
45 return "Unsupported opcode: isBranch/isIndirectBranch";
46 if (InstrDesc.isCall() || InstrDesc.isReturn())
47 return "Unsupported opcode: isCall/isReturn";
48 if (InstrDesc.getSchedClass() == 0)
49 return "Unsupported opcode: No Sched Class";
50 return nullptr;
51}
52
53Expected<std::unique_ptr<pfm::CounterGroup>>
54ExegesisTarget::createCounter(StringRef CounterName, const LLVMState &State,
55 ArrayRef<const char *> ValidationCounters,
56 const pid_t ProcessID) const {
57 const PfmCountersInfo &PCI = State.getPfmCounters();
58
59 std::vector<pfm::PerfEvent> ValidationEvents;
60 for (const char *ValCounterName : ValidationCounters) {
61 ValidationEvents.emplace_back(args&: ValCounterName);
62 if (!ValidationEvents.back().valid())
63 return make_error<Failure>(
64 Args: Twine("Unable to create validation counter with name '")
65 .concat(Suffix: ValCounterName)
66 .concat(Suffix: "'"));
67 }
68
69 if (PCI.CycleCounterEventSelect != -1 &&
70 CounterName == StringRef(PCI.CycleCounter)) {
71 pfm::RawPerfEvent Event(PCI.CycleCounterEventSelect, PCI.CycleCounterUMask);
72 if (!Event.valid())
73 return make_error<Failure>(
74 Args: Twine("Unable to create raw counter with EventSelect: ")
75 .concat(Suffix: Twine(PCI.CycleCounterEventSelect))
76 .concat(Suffix: " UMask: ")
77 .concat(Suffix: Twine(PCI.CycleCounterUMask)));
78 return std::make_unique<pfm::CounterGroup>(
79 args: std::move(Event), args: std::move(ValidationEvents), args: ProcessID);
80 }
81
82 if (PCI.UopsCounterEventSelect != -1 &&
83 CounterName == StringRef(PCI.UopsCounter)) {
84 pfm::RawPerfEvent Event(PCI.UopsCounterEventSelect, PCI.UopsCounterUMask);
85 if (!Event.valid())
86 return make_error<Failure>(
87 Args: Twine("Unable to create raw counter with EventSelect: ")
88 .concat(Suffix: Twine(PCI.UopsCounterEventSelect))
89 .concat(Suffix: " UMask: ")
90 .concat(Suffix: Twine(PCI.UopsCounterUMask)));
91 return std::make_unique<pfm::CounterGroup>(
92 args: std::move(Event), args: std::move(ValidationEvents), args: ProcessID);
93 }
94
95 pfm::PerfEvent Event(CounterName);
96 if (!Event.valid())
97 return make_error<Failure>(Args: Twine("Unable to create counter with name '")
98 .concat(Suffix: CounterName)
99 .concat(Suffix: "'"));
100
101 return std::make_unique<pfm::CounterGroup>(
102 args: std::move(Event), args: std::move(ValidationEvents), args: ProcessID);
103}
104
105void ExegesisTarget::registerTarget(ExegesisTarget *Target) {
106 if (FirstTarget == nullptr) {
107 FirstTarget = Target;
108 return;
109 }
110 if (Target->Next != nullptr)
111 return; // Already registered.
112 Target->Next = FirstTarget;
113 FirstTarget = Target;
114}
115
116std::unique_ptr<SnippetGenerator> ExegesisTarget::createSnippetGenerator(
117 Benchmark::ModeE Mode, const LLVMState &State,
118 const SnippetGenerator::Options &Opts) const {
119 switch (Mode) {
120 case Benchmark::Unknown:
121 return nullptr;
122 case Benchmark::Latency:
123 return createSerialSnippetGenerator(State, Opts);
124 case Benchmark::Uops:
125 case Benchmark::InverseThroughput:
126 return createParallelSnippetGenerator(State, Opts);
127 }
128 return nullptr;
129}
130
131Expected<std::unique_ptr<BenchmarkRunner>>
132ExegesisTarget::createBenchmarkRunner(
133 Benchmark::ModeE Mode, const LLVMState &State,
134 BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
135 BenchmarkRunner::ExecutionModeE ExecutionMode,
136 unsigned BenchmarkRepeatCount, ArrayRef<ValidationEvent> ValidationCounters,
137 Benchmark::ResultAggregationModeE ResultAggMode) const {
138 PfmCountersInfo PfmCounters = State.getPfmCounters();
139 switch (Mode) {
140 case Benchmark::Unknown:
141 return nullptr;
142 case Benchmark::Latency:
143 case Benchmark::InverseThroughput:
144 if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure &&
145 !PfmCounters.CycleCounter &&
146 PfmCounters.CycleCounterEventSelect == -1) {
147 const char *ModeName = Mode == Benchmark::Latency
148 ? "latency"
149 : "inverse_throughput";
150 return make_error<Failure>(
151 Args: Twine("can't run '")
152 .concat(Suffix: ModeName)
153 .concat(
154 Suffix: "' mode, sched model does not define a cycle counter. You "
155 "can pass --benchmark-phase=... to skip the actual "
156 "benchmarking or --use-dummy-perf-counters to not query "
157 "the kernel for real event counts."));
158 }
159 return createLatencyBenchmarkRunner(
160 State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode,
161 ValidationCounters, BenchmarkRepeatCount);
162 case Benchmark::Uops:
163 if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure &&
164 !PfmCounters.UopsCounter && !PfmCounters.IssueCounters &&
165 PfmCounters.UopsCounterEventSelect == -1)
166 return make_error<Failure>(
167 Args: "can't run 'uops' mode, sched model does not define uops or issue "
168 "counters. You can pass --benchmark-phase=... to skip the actual "
169 "benchmarking or --use-dummy-perf-counters to not query the kernel "
170 "for real event counts.");
171 return createUopsBenchmarkRunner(State, BenchmarkPhaseSelector,
172 ResultAggMode, ExecutionMode,
173 ValidationCounters);
174 }
175 return nullptr;
176}
177
178std::unique_ptr<SnippetGenerator> ExegesisTarget::createSerialSnippetGenerator(
179 const LLVMState &State, const SnippetGenerator::Options &Opts) const {
180 return std::make_unique<SerialSnippetGenerator>(args: State, args: Opts);
181}
182
183std::unique_ptr<SnippetGenerator> ExegesisTarget::createParallelSnippetGenerator(
184 const LLVMState &State, const SnippetGenerator::Options &Opts) const {
185 return std::make_unique<ParallelSnippetGenerator>(args: State, args: Opts);
186}
187
188std::unique_ptr<BenchmarkRunner> ExegesisTarget::createLatencyBenchmarkRunner(
189 const LLVMState &State, Benchmark::ModeE Mode,
190 BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
191 Benchmark::ResultAggregationModeE ResultAggMode,
192 BenchmarkRunner::ExecutionModeE ExecutionMode,
193 ArrayRef<ValidationEvent> ValidationCounters,
194 unsigned BenchmarkRepeatCount) const {
195 return std::make_unique<LatencyBenchmarkRunner>(
196 args: State, args&: Mode, args&: BenchmarkPhaseSelector, args&: ResultAggMode, args&: ExecutionMode,
197 args&: ValidationCounters, args&: BenchmarkRepeatCount);
198}
199
200std::unique_ptr<BenchmarkRunner> ExegesisTarget::createUopsBenchmarkRunner(
201 const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
202 Benchmark::ResultAggregationModeE /*unused*/,
203 BenchmarkRunner::ExecutionModeE ExecutionMode,
204 ArrayRef<ValidationEvent> ValidationCounters) const {
205 return std::make_unique<UopsBenchmarkRunner>(
206 args: State, args&: BenchmarkPhaseSelector, args&: ExecutionMode, args&: ValidationCounters);
207}
208
209static_assert(std::is_trivial_v<PfmCountersInfo>,
210 "We shouldn't have dynamic initialization here");
211
212const PfmCountersInfo PfmCountersInfo::Default = {
213 .CycleCounter: nullptr, .CycleCounterEventSelect: -1, .CycleCounterUMask: 0, .UopsCounter: nullptr, .UopsCounterEventSelect: -1, .UopsCounterUMask: 0, .IssueCounters: nullptr, .NumIssueCounters: 0u, .ValidationEvents: nullptr, .NumValidationEvents: 0u};
214const PfmCountersInfo PfmCountersInfo::Dummy = {
215 .CycleCounter: pfm::PerfEvent::DummyEventString,
216 .CycleCounterEventSelect: -1,
217 .CycleCounterUMask: 0,
218 .UopsCounter: pfm::PerfEvent::DummyEventString,
219 .UopsCounterEventSelect: -1,
220 .UopsCounterUMask: 0,
221 .IssueCounters: nullptr,
222 .NumIssueCounters: 0u,
223 .ValidationEvents: nullptr,
224 .NumValidationEvents: 0u};
225
226const PfmCountersInfo &ExegesisTarget::getPfmCounters(StringRef CpuName) const {
227 assert(
228 is_sorted(CpuPfmCounters,
229 [](const CpuAndPfmCounters &LHS, const CpuAndPfmCounters &RHS) {
230 return strcmp(LHS.CpuName, RHS.CpuName) < 0;
231 }) &&
232 "CpuPfmCounters table is not sorted");
233
234 // Find entry
235 auto Found = lower_bound(Range: CpuPfmCounters, Value&: CpuName);
236 if (Found == CpuPfmCounters.end() || StringRef(Found->CpuName) != CpuName) {
237 // Use the default.
238 if (!CpuPfmCounters.empty() && CpuPfmCounters.begin()->CpuName[0] == '\0') {
239 Found = CpuPfmCounters.begin(); // The target specifies a default.
240 } else {
241 return PfmCountersInfo::Default; // No default for the target.
242 }
243 }
244 assert(Found->PCI && "Missing counters");
245 return *Found->PCI;
246}
247
248const PfmCountersInfo &ExegesisTarget::getDummyPfmCounters() const {
249 return PfmCountersInfo::Dummy;
250}
251
252ExegesisTarget::SavedState::~SavedState() = default; // anchor.
253
254namespace {
255
256bool opcodeIsNotAvailable(unsigned, const FeatureBitset &) { return false; }
257
258// Default implementation.
259class ExegesisDefaultTarget : public ExegesisTarget {
260public:
261 ExegesisDefaultTarget() : ExegesisTarget({}, opcodeIsNotAvailable) {}
262
263private:
264 std::vector<MCInst> setRegTo(const MCSubtargetInfo &STI, MCRegister Reg,
265 const APInt &Value) const override {
266 llvm_unreachable("Not yet implemented");
267 }
268
269 bool matchesArch(Triple::ArchType Arch) const override {
270 llvm_unreachable("never called");
271 return false;
272 }
273};
274
275} // namespace
276
277const ExegesisTarget &ExegesisTarget::getDefault() {
278 static ExegesisDefaultTarget Target;
279 return Target;
280}
281
282} // namespace exegesis
283} // namespace llvm
284