| 1 | //===-- llvm-exegesis.cpp ---------------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// \file |
| 10 | /// Measures execution properties (latencies/uops) of an instruction. |
| 11 | /// |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "lib/Analysis.h" |
| 15 | #include "lib/BenchmarkResult.h" |
| 16 | #include "lib/BenchmarkRunner.h" |
| 17 | #include "lib/Clustering.h" |
| 18 | #include "lib/CodeTemplate.h" |
| 19 | #include "lib/Error.h" |
| 20 | #include "lib/LlvmState.h" |
| 21 | #include "lib/PerfHelper.h" |
| 22 | #include "lib/ProgressMeter.h" |
| 23 | #include "lib/ResultAggregator.h" |
| 24 | #include "lib/SnippetFile.h" |
| 25 | #include "lib/SnippetRepetitor.h" |
| 26 | #include "lib/Target.h" |
| 27 | #include "lib/TargetSelect.h" |
| 28 | #include "lib/ValidationEvent.h" |
| 29 | #include "llvm/ADT/StringExtras.h" |
| 30 | #include "llvm/ADT/Twine.h" |
| 31 | #include "llvm/MC/MCInstBuilder.h" |
| 32 | #include "llvm/MC/MCObjectFileInfo.h" |
| 33 | #include "llvm/MC/MCParser/MCAsmParser.h" |
| 34 | #include "llvm/MC/MCParser/MCTargetAsmParser.h" |
| 35 | #include "llvm/MC/MCRegisterInfo.h" |
| 36 | #include "llvm/MC/MCSubtargetInfo.h" |
| 37 | #include "llvm/MC/TargetRegistry.h" |
| 38 | #include "llvm/Object/ObjectFile.h" |
| 39 | #include "llvm/Support/CommandLine.h" |
| 40 | #include "llvm/Support/FileSystem.h" |
| 41 | #include "llvm/Support/Format.h" |
| 42 | #include "llvm/Support/InitLLVM.h" |
| 43 | #include "llvm/Support/LibraryOptionsContext.h" |
| 44 | #include "llvm/Support/Path.h" |
| 45 | #include "llvm/Support/SourceMgr.h" |
| 46 | #include "llvm/Support/TargetSelect.h" |
| 47 | #include "llvm/TargetParser/Host.h" |
| 48 | #include <algorithm> |
| 49 | #include <string> |
| 50 | |
| 51 | namespace llvm { |
| 52 | namespace exegesis { |
| 53 | |
| 54 | static cl::opt<int> OpcodeIndex( |
| 55 | "opcode-index" , |
| 56 | cl::desc("opcode to measure, by index, or -1 to measure all opcodes" ), |
| 57 | cl::cat(BenchmarkOptions), cl::init(Val: 0)); |
| 58 | |
| 59 | static cl::opt<std::string> |
| 60 | OpcodeNames("opcode-name" , |
| 61 | cl::desc("comma-separated list of opcodes to measure, by name" ), |
| 62 | cl::cat(BenchmarkOptions), cl::init(Val: "" )); |
| 63 | |
| 64 | static cl::opt<std::string> SnippetsFile("snippets-file" , |
| 65 | cl::desc("code snippets to measure" ), |
| 66 | cl::cat(BenchmarkOptions), |
| 67 | cl::init(Val: "" )); |
| 68 | |
| 69 | static cl::opt<std::string> |
| 70 | BenchmarkFile("benchmarks-file" , |
| 71 | cl::desc("File to read (analysis mode) or write " |
| 72 | "(latency/uops/inverse_throughput modes) benchmark " |
| 73 | "results. “-” uses stdin/stdout." ), |
| 74 | cl::cat(Options), cl::init(Val: "" )); |
| 75 | |
| 76 | static cl::opt<Benchmark::ModeE> BenchmarkMode( |
| 77 | "mode" , cl::desc("the mode to run" ), cl::cat(Options), |
| 78 | cl::values(clEnumValN(Benchmark::Latency, "latency" , "Instruction Latency" ), |
| 79 | clEnumValN(Benchmark::InverseThroughput, "inverse_throughput" , |
| 80 | "Instruction Inverse Throughput" ), |
| 81 | clEnumValN(Benchmark::Uops, "uops" , "Uop Decomposition" ), |
| 82 | // When not asking for a specific benchmark mode, |
| 83 | // we'll analyse the results. |
| 84 | clEnumValN(Benchmark::Unknown, "analysis" , "Analysis" ))); |
| 85 | |
| 86 | static cl::opt<Benchmark::ResultAggregationModeE> ResultAggMode( |
| 87 | "result-aggregation-mode" , cl::desc("How to aggregate multi-values result" ), |
| 88 | cl::cat(BenchmarkOptions), |
| 89 | cl::values(clEnumValN(Benchmark::Min, "min" , "Keep min reading" ), |
| 90 | clEnumValN(Benchmark::Max, "max" , "Keep max reading" ), |
| 91 | clEnumValN(Benchmark::Mean, "mean" , |
| 92 | "Compute mean of all readings" ), |
| 93 | clEnumValN(Benchmark::MinVariance, "min-variance" , |
| 94 | "Keep readings set with min-variance" )), |
| 95 | cl::init(Val: Benchmark::Min)); |
| 96 | |
| 97 | static cl::opt<Benchmark::RepetitionModeE> RepetitionMode( |
| 98 | "repetition-mode" , cl::desc("how to repeat the instruction snippet" ), |
| 99 | cl::cat(BenchmarkOptions), |
| 100 | cl::values( |
| 101 | clEnumValN(Benchmark::Duplicate, "duplicate" , "Duplicate the snippet" ), |
| 102 | clEnumValN(Benchmark::Loop, "loop" , "Loop over the snippet" ), |
| 103 | clEnumValN(Benchmark::AggregateMin, "min" , |
| 104 | "All of the above and take the minimum of measurements" ), |
| 105 | clEnumValN(Benchmark::MiddleHalfDuplicate, "middle-half-duplicate" , |
| 106 | "Middle half duplicate mode" ), |
| 107 | clEnumValN(Benchmark::MiddleHalfLoop, "middle-half-loop" , |
| 108 | "Middle half loop mode" )), |
| 109 | cl::init(Val: Benchmark::Duplicate)); |
| 110 | |
| 111 | static cl::opt<bool> BenchmarkMeasurementsPrintProgress( |
| 112 | "measurements-print-progress" , |
| 113 | cl::desc("Produce progress indicator when performing measurements" ), |
| 114 | cl::cat(BenchmarkOptions), cl::init(Val: false)); |
| 115 | |
| 116 | static cl::opt<BenchmarkPhaseSelectorE> BenchmarkPhaseSelector( |
| 117 | "benchmark-phase" , |
| 118 | cl::desc( |
| 119 | "it is possible to stop the benchmarking process after some phase" ), |
| 120 | cl::cat(BenchmarkOptions), |
| 121 | cl::values( |
| 122 | clEnumValN(BenchmarkPhaseSelectorE::PrepareSnippet, "prepare-snippet" , |
| 123 | "Only generate the minimal instruction sequence" ), |
| 124 | clEnumValN(BenchmarkPhaseSelectorE::PrepareAndAssembleSnippet, |
| 125 | "prepare-and-assemble-snippet" , |
| 126 | "Same as prepare-snippet, but also dumps an excerpt of the " |
| 127 | "sequence (hex encoded)" ), |
| 128 | clEnumValN(BenchmarkPhaseSelectorE::AssembleMeasuredCode, |
| 129 | "assemble-measured-code" , |
| 130 | "Same as prepare-and-assemble-snippet, but also creates the " |
| 131 | "full sequence " |
| 132 | "that can be dumped to a file using --dump-object-to-disk" ), |
| 133 | clEnumValN( |
| 134 | BenchmarkPhaseSelectorE::Measure, "measure" , |
| 135 | "Same as prepare-measured-code, but also runs the measurement " |
| 136 | "(default)" )), |
| 137 | cl::init(Val: BenchmarkPhaseSelectorE::Measure)); |
| 138 | |
| 139 | static cl::opt<bool> |
| 140 | UseDummyPerfCounters("use-dummy-perf-counters" , |
| 141 | cl::desc("Do not read real performance counters, use " |
| 142 | "dummy values (for testing)" ), |
| 143 | cl::cat(BenchmarkOptions), cl::init(Val: false)); |
| 144 | |
| 145 | static cl::opt<unsigned> |
| 146 | MinInstructions("min-instructions" , |
| 147 | cl::desc("The minimum number of instructions that should " |
| 148 | "be included in the snippet" ), |
| 149 | cl::cat(BenchmarkOptions), cl::init(Val: 10000)); |
| 150 | |
| 151 | static cl::opt<unsigned> |
| 152 | LoopBodySize("loop-body-size" , |
| 153 | cl::desc("when repeating the instruction snippet by looping " |
| 154 | "over it, duplicate the snippet until the loop body " |
| 155 | "contains at least this many instruction" ), |
| 156 | cl::cat(BenchmarkOptions), cl::init(Val: 0)); |
| 157 | |
| 158 | static cl::opt<unsigned> MaxConfigsPerOpcode( |
| 159 | "max-configs-per-opcode" , |
| 160 | cl::desc( |
| 161 | "allow to snippet generator to generate at most that many configs" ), |
| 162 | cl::cat(BenchmarkOptions), cl::init(Val: 1)); |
| 163 | |
| 164 | static cl::opt<bool> IgnoreInvalidSchedClass( |
| 165 | "ignore-invalid-sched-class" , |
| 166 | cl::desc("ignore instructions that do not define a sched class" ), |
| 167 | cl::cat(BenchmarkOptions), cl::init(Val: false)); |
| 168 | |
| 169 | static cl::opt<BenchmarkFilter> AnalysisSnippetFilter( |
| 170 | "analysis-filter" , cl::desc("Filter the benchmarks before analysing them" ), |
| 171 | cl::cat(BenchmarkOptions), |
| 172 | cl::values( |
| 173 | clEnumValN(BenchmarkFilter::All, "all" , |
| 174 | "Keep all benchmarks (default)" ), |
| 175 | clEnumValN(BenchmarkFilter::RegOnly, "reg-only" , |
| 176 | "Keep only those benchmarks that do *NOT* involve memory" ), |
| 177 | clEnumValN(BenchmarkFilter::WithMem, "mem-only" , |
| 178 | "Keep only the benchmarks that *DO* involve memory" )), |
| 179 | cl::init(Val: BenchmarkFilter::All)); |
| 180 | |
| 181 | static cl::opt<BenchmarkClustering::ModeE> AnalysisClusteringAlgorithm( |
| 182 | "analysis-clustering" , cl::desc("the clustering algorithm to use" ), |
| 183 | cl::cat(AnalysisOptions), |
| 184 | cl::values(clEnumValN(BenchmarkClustering::Dbscan, "dbscan" , |
| 185 | "use DBSCAN/OPTICS algorithm" ), |
| 186 | clEnumValN(BenchmarkClustering::Naive, "naive" , |
| 187 | "one cluster per opcode" )), |
| 188 | cl::init(Val: BenchmarkClustering::Dbscan)); |
| 189 | |
| 190 | static cl::opt<unsigned> AnalysisDbscanNumPoints( |
| 191 | "analysis-numpoints" , |
| 192 | cl::desc("minimum number of points in an analysis cluster (dbscan only)" ), |
| 193 | cl::cat(AnalysisOptions), cl::init(Val: 3)); |
| 194 | |
| 195 | static cl::opt<float> AnalysisClusteringEpsilon( |
| 196 | "analysis-clustering-epsilon" , |
| 197 | cl::desc("epsilon for benchmark point clustering" ), |
| 198 | cl::cat(AnalysisOptions), cl::init(Val: 0.1)); |
| 199 | |
| 200 | static cl::opt<float> AnalysisInconsistencyEpsilon( |
| 201 | "analysis-inconsistency-epsilon" , |
| 202 | cl::desc("epsilon for detection of when the cluster is different from the " |
| 203 | "LLVM schedule profile values" ), |
| 204 | cl::cat(AnalysisOptions), cl::init(Val: 0.1)); |
| 205 | |
| 206 | static cl::opt<std::string> |
| 207 | AnalysisClustersOutputFile("analysis-clusters-output-file" , cl::desc("" ), |
| 208 | cl::cat(AnalysisOptions), cl::init(Val: "" )); |
| 209 | static cl::opt<std::string> |
| 210 | AnalysisInconsistenciesOutputFile("analysis-inconsistencies-output-file" , |
| 211 | cl::desc("" ), cl::cat(AnalysisOptions), |
| 212 | cl::init(Val: "" )); |
| 213 | |
| 214 | static cl::opt<bool> AnalysisDisplayUnstableOpcodes( |
| 215 | "analysis-display-unstable-clusters" , |
| 216 | cl::desc("if there is more than one benchmark for an opcode, said " |
| 217 | "benchmarks may end up not being clustered into the same cluster " |
| 218 | "if the measured performance characteristics are different. by " |
| 219 | "default all such opcodes are filtered out. this flag will " |
| 220 | "instead show only such unstable opcodes" ), |
| 221 | cl::cat(AnalysisOptions), cl::init(Val: false)); |
| 222 | |
| 223 | static cl::opt<bool> AnalysisOverrideBenchmarksTripleAndCpu( |
| 224 | "analysis-override-benchmark-triple-and-cpu" , |
| 225 | cl::desc("By default, we analyze the benchmarks for the triple/CPU they " |
| 226 | "were measured for, but if you want to analyze them for some " |
| 227 | "other combination (specified via -mtriple/-mcpu), you can " |
| 228 | "pass this flag." ), |
| 229 | cl::cat(AnalysisOptions), cl::init(Val: false)); |
| 230 | |
| 231 | static cl::opt<std::string> |
| 232 | TripleName("mtriple" , |
| 233 | cl::desc("Target triple. See -version for available targets" ), |
| 234 | cl::cat(Options)); |
| 235 | |
| 236 | static cl::opt<std::string> |
| 237 | MCPU("mcpu" , |
| 238 | cl::desc("Target a specific cpu type (-mcpu=help for details)" ), |
| 239 | cl::value_desc("cpu-name" ), cl::cat(Options), cl::init(Val: "native" )); |
| 240 | |
| 241 | static cl::opt<std::string> |
| 242 | DumpObjectToDisk("dump-object-to-disk" , |
| 243 | cl::desc("dumps the generated benchmark object to disk " |
| 244 | "and prints a message to access it" ), |
| 245 | cl::ValueOptional, cl::cat(BenchmarkOptions)); |
| 246 | |
| 247 | static cl::opt<BenchmarkRunner::ExecutionModeE> ExecutionMode( |
| 248 | "execution-mode" , |
| 249 | cl::desc("Selects the execution mode to use for running snippets" ), |
| 250 | cl::cat(BenchmarkOptions), |
| 251 | cl::values(clEnumValN(BenchmarkRunner::ExecutionModeE::InProcess, |
| 252 | "inprocess" , |
| 253 | "Executes the snippets within the same process" ), |
| 254 | clEnumValN(BenchmarkRunner::ExecutionModeE::SubProcess, |
| 255 | "subprocess" , |
| 256 | "Spawns a subprocess for each snippet execution, " |
| 257 | "allows for the use of memory annotations" )), |
| 258 | cl::init(Val: BenchmarkRunner::ExecutionModeE::InProcess)); |
| 259 | |
| 260 | static cl::opt<unsigned> BenchmarkRepeatCount( |
| 261 | "benchmark-repeat-count" , |
| 262 | cl::desc("The number of times to repeat measurements on the benchmark k " |
| 263 | "before aggregating the results" ), |
| 264 | cl::cat(BenchmarkOptions), cl::init(Val: 30)); |
| 265 | |
| 266 | static cl::list<ValidationEvent> ValidationCounters( |
| 267 | "validation-counter" , |
| 268 | cl::desc( |
| 269 | "The name of a validation counter to run concurrently with the main " |
| 270 | "counter to validate benchmarking assumptions" ), |
| 271 | cl::CommaSeparated, cl::cat(BenchmarkOptions), ValidationEventOptions()); |
| 272 | |
| 273 | static cl::opt<int> BenchmarkProcessCPU( |
| 274 | "benchmark-process-cpu" , |
| 275 | cl::desc("The CPU number that the benchmarking process should executon on" ), |
| 276 | cl::cat(BenchmarkOptions), cl::init(Val: -1)); |
| 277 | |
| 278 | static cl::opt<std::string> MAttr( |
| 279 | "mattr" , cl::desc("comma-separated list of target architecture features" ), |
| 280 | cl::value_desc("+feature1,-feature2,..." ), cl::cat(Options), cl::init(Val: "" )); |
| 281 | |
| 282 | static ExitOnError ExitOnErr("llvm-exegesis error: " ); |
| 283 | |
| 284 | // Helper function that logs the error(s) and exits. |
| 285 | template <typename... ArgTs> static void ExitWithError(ArgTs &&... Args) { |
| 286 | ExitOnErr(make_error<Failure>(std::forward<ArgTs>(Args)...)); |
| 287 | } |
| 288 | |
| 289 | // Check Err. If it's in a failure state log the file error(s) and exit. |
| 290 | static void ExitOnFileError(const Twine &FileName, Error Err) { |
| 291 | if (Err) { |
| 292 | ExitOnErr(createFileError(F: FileName, E: std::move(Err))); |
| 293 | } |
| 294 | } |
| 295 | |
| 296 | // Check E. If it's in a success state then return the contained value. |
| 297 | // If it's in a failure state log the file error(s) and exit. |
| 298 | template <typename T> |
| 299 | T ExitOnFileError(const Twine &FileName, Expected<T> &&E) { |
| 300 | ExitOnFileError(FileName, E.takeError()); |
| 301 | return std::move(*E); |
| 302 | } |
| 303 | |
| 304 | // Checks that only one of OpcodeNames, OpcodeIndex or SnippetsFile is provided, |
| 305 | // and returns the opcode indices or {} if snippets should be read from |
| 306 | // `SnippetsFile`. |
| 307 | static std::vector<unsigned> getOpcodesOrDie(const LLVMState &State) { |
| 308 | const size_t NumSetFlags = (OpcodeNames.empty() ? 0 : 1) + |
| 309 | (OpcodeIndex == 0 ? 0 : 1) + |
| 310 | (SnippetsFile.empty() ? 0 : 1); |
| 311 | const auto &ET = State.getExegesisTarget(); |
| 312 | const auto AvailableFeatures = State.getSubtargetInfo().getFeatureBits(); |
| 313 | |
| 314 | if (NumSetFlags != 1) { |
| 315 | ExitOnErr.setBanner("llvm-exegesis: " ); |
| 316 | ExitWithError(Args: "please provide one and only one of 'opcode-index', " |
| 317 | "'opcode-name' or 'snippets-file'" ); |
| 318 | } |
| 319 | if (!SnippetsFile.empty()) |
| 320 | return {}; |
| 321 | if (OpcodeIndex > 0) |
| 322 | return {static_cast<unsigned>(OpcodeIndex)}; |
| 323 | if (OpcodeIndex < 0) { |
| 324 | std::vector<unsigned> Result; |
| 325 | unsigned NumOpcodes = State.getInstrInfo().getNumOpcodes(); |
| 326 | Result.reserve(n: NumOpcodes); |
| 327 | for (unsigned I = 0, E = NumOpcodes; I < E; ++I) { |
| 328 | if (!ET.isOpcodeAvailable(Opcode: I, Features: AvailableFeatures)) |
| 329 | continue; |
| 330 | Result.push_back(x: I); |
| 331 | } |
| 332 | return Result; |
| 333 | } |
| 334 | // Resolve opcode name -> opcode. |
| 335 | const auto ResolveName = [&State](StringRef OpcodeName) -> unsigned { |
| 336 | const auto &Map = State.getOpcodeNameToOpcodeIdxMapping(); |
| 337 | auto I = Map.find(Val: OpcodeName); |
| 338 | if (I != Map.end()) |
| 339 | return I->getSecond(); |
| 340 | return 0u; |
| 341 | }; |
| 342 | |
| 343 | SmallVector<StringRef, 2> Pieces; |
| 344 | StringRef(OpcodeNames.getValue()) |
| 345 | .split(A&: Pieces, Separator: "," , /* MaxSplit */ -1, /* KeepEmpty */ false); |
| 346 | std::vector<unsigned> Result; |
| 347 | Result.reserve(n: Pieces.size()); |
| 348 | for (const StringRef &OpcodeName : Pieces) { |
| 349 | if (unsigned Opcode = ResolveName(OpcodeName)) |
| 350 | Result.push_back(x: Opcode); |
| 351 | else |
| 352 | ExitWithError(Args: Twine("unknown opcode " ).concat(Suffix: OpcodeName)); |
| 353 | } |
| 354 | return Result; |
| 355 | } |
| 356 | |
| 357 | // Generates code snippets for opcode `Opcode`. |
| 358 | static Expected<std::vector<BenchmarkCode>> |
| 359 | generateSnippets(const LLVMState &State, unsigned Opcode, |
| 360 | const BitVector &ForbiddenRegs) { |
| 361 | // Ignore instructions that we cannot run. |
| 362 | if (const char *Reason = |
| 363 | State.getExegesisTarget().getIgnoredOpcodeReasonOrNull(State, Opcode)) |
| 364 | return make_error<Failure>(Args&: Reason); |
| 365 | |
| 366 | const Instruction &Instr = State.getIC().getInstr(Opcode); |
| 367 | const std::vector<InstructionTemplate> InstructionVariants = |
| 368 | State.getExegesisTarget().generateInstructionVariants( |
| 369 | Instr, MaxConfigsPerOpcode); |
| 370 | |
| 371 | SnippetGenerator::Options SnippetOptions; |
| 372 | SnippetOptions.MaxConfigsPerOpcode = MaxConfigsPerOpcode; |
| 373 | const std::unique_ptr<SnippetGenerator> Generator = |
| 374 | State.getExegesisTarget().createSnippetGenerator(Mode: BenchmarkMode, State, |
| 375 | Opts: SnippetOptions); |
| 376 | if (!Generator) |
| 377 | ExitWithError(Args: "cannot create snippet generator" ); |
| 378 | |
| 379 | std::vector<BenchmarkCode> Benchmarks; |
| 380 | for (const InstructionTemplate &Variant : InstructionVariants) { |
| 381 | if (Benchmarks.size() >= MaxConfigsPerOpcode) |
| 382 | break; |
| 383 | if (auto Err = Generator->generateConfigurations(Variant, Benchmarks, |
| 384 | ExtraForbiddenRegs: ForbiddenRegs)) |
| 385 | return std::move(Err); |
| 386 | } |
| 387 | return Benchmarks; |
| 388 | } |
| 389 | |
| 390 | static void runBenchmarkConfigurations( |
| 391 | const LLVMState &State, ArrayRef<BenchmarkCode> Configurations, |
| 392 | ArrayRef<std::unique_ptr<const SnippetRepetitor>> Repetitors, |
| 393 | const BenchmarkRunner &Runner) { |
| 394 | assert(!Configurations.empty() && "Don't have any configurations to run." ); |
| 395 | std::optional<raw_fd_ostream> FileOstr; |
| 396 | if (BenchmarkFile != "-" ) { |
| 397 | int ResultFD = 0; |
| 398 | // Create output file or open existing file and truncate it, once. |
| 399 | ExitOnErr(errorCodeToError(EC: openFileForWrite(Name: BenchmarkFile, ResultFD, |
| 400 | Disp: sys::fs::CD_CreateAlways, |
| 401 | Flags: sys::fs::OF_TextWithCRLF))); |
| 402 | FileOstr.emplace(args&: ResultFD, args: true /*shouldClose*/); |
| 403 | } |
| 404 | raw_ostream &Ostr = FileOstr ? *FileOstr : outs(); |
| 405 | |
| 406 | std::optional<ProgressMeter<>> Meter; |
| 407 | if (BenchmarkMeasurementsPrintProgress) |
| 408 | Meter.emplace(args: Configurations.size()); |
| 409 | |
| 410 | SmallVector<unsigned, 2> MinInstructionCounts = {MinInstructions}; |
| 411 | if (RepetitionMode == Benchmark::MiddleHalfDuplicate || |
| 412 | RepetitionMode == Benchmark::MiddleHalfLoop) |
| 413 | MinInstructionCounts.push_back(Elt: MinInstructions * 2); |
| 414 | |
| 415 | for (const BenchmarkCode &Conf : Configurations) { |
| 416 | ProgressMeter<>::ProgressMeterStep MeterStep(Meter ? &*Meter : nullptr); |
| 417 | SmallVector<Benchmark, 2> AllResults; |
| 418 | |
| 419 | for (const std::unique_ptr<const SnippetRepetitor> &Repetitor : |
| 420 | Repetitors) { |
| 421 | for (unsigned IterationRepetitions : MinInstructionCounts) { |
| 422 | auto RC = ExitOnErr(Runner.getRunnableConfiguration( |
| 423 | Configuration: Conf, MinInstructions: IterationRepetitions, LoopUnrollFactor: LoopBodySize, Repetitor: *Repetitor)); |
| 424 | std::optional<StringRef> DumpFile; |
| 425 | if (DumpObjectToDisk.getNumOccurrences()) |
| 426 | DumpFile = DumpObjectToDisk; |
| 427 | const std::optional<int> BenchmarkCPU = |
| 428 | BenchmarkProcessCPU == -1 |
| 429 | ? std::nullopt |
| 430 | : std::optional(BenchmarkProcessCPU.getValue()); |
| 431 | auto [Err, BenchmarkResult] = |
| 432 | Runner.runConfiguration(RC: std::move(RC), DumpFile, BenchmarkProcessCPU: BenchmarkCPU); |
| 433 | if (Err) { |
| 434 | // Errors from executing the snippets are fine. |
| 435 | // All other errors are a framework issue and should fail. |
| 436 | if (!Err.isA<SnippetExecutionFailure>()) |
| 437 | ExitOnErr(std::move(Err)); |
| 438 | |
| 439 | BenchmarkResult.Error = toString(E: std::move(Err)); |
| 440 | } |
| 441 | AllResults.push_back(Elt: std::move(BenchmarkResult)); |
| 442 | } |
| 443 | } |
| 444 | |
| 445 | Benchmark &Result = AllResults.front(); |
| 446 | |
| 447 | // If any of our measurements failed, pretend they all have failed. |
| 448 | if (AllResults.size() > 1 && |
| 449 | any_of(Range&: AllResults, P: [](const Benchmark &R) { |
| 450 | return R.Measurements.empty(); |
| 451 | })) |
| 452 | Result.Measurements.clear(); |
| 453 | |
| 454 | std::unique_ptr<ResultAggregator> ResultAgg = |
| 455 | ResultAggregator::CreateAggregator(RepetitionMode); |
| 456 | ResultAgg->AggregateResults(Result, |
| 457 | OtherResults: ArrayRef<Benchmark>(AllResults).drop_front()); |
| 458 | |
| 459 | // With dummy counters, measurements are rather meaningless, |
| 460 | // so drop them altogether. |
| 461 | if (UseDummyPerfCounters) |
| 462 | Result.Measurements.clear(); |
| 463 | |
| 464 | ExitOnFileError(FileName: BenchmarkFile, Err: Result.writeYamlTo(State, S&: Ostr)); |
| 465 | } |
| 466 | } |
| 467 | |
| 468 | void benchmarkMain(const cl::LibraryOptionsContext &LibraryOptions) { |
| 469 | if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure && |
| 470 | !UseDummyPerfCounters) { |
| 471 | #ifndef HAVE_LIBPFM |
| 472 | ExitWithError( |
| 473 | Args: "benchmarking unavailable, LLVM was built without libpfm. You can " |
| 474 | "pass --benchmark-phase=... to skip the actual benchmarking or " |
| 475 | "--use-dummy-perf-counters to not query the kernel for real event " |
| 476 | "counts." ); |
| 477 | #else |
| 478 | if (pfm::pfmInitialize()) |
| 479 | ExitWithError("cannot initialize libpfm" ); |
| 480 | #endif |
| 481 | } |
| 482 | |
| 483 | InitializeAllExegesisTargets(); |
| 484 | #define LLVM_EXEGESIS(TargetName) \ |
| 485 | LLVMInitialize##TargetName##AsmPrinter(); \ |
| 486 | LLVMInitialize##TargetName##AsmParser(); \ |
| 487 | LLVMInitialize##TargetName##Disassembler(); |
| 488 | #include "llvm/Config/TargetExegesis.def" |
| 489 | |
| 490 | const LLVMState State = ExitOnErr(LLVMState::create( |
| 491 | LibraryOptions, TripleName, CpuName: MCPU, Features: MAttr, UseDummyPerfCounters)); |
| 492 | |
| 493 | // Preliminary check to ensure features needed for requested |
| 494 | // benchmark mode are present on target CPU and/or OS. |
| 495 | if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure) |
| 496 | ExitOnErr(State.getExegesisTarget().checkFeatureSupport()); |
| 497 | |
| 498 | if (ExecutionMode == BenchmarkRunner::ExecutionModeE::SubProcess && |
| 499 | UseDummyPerfCounters) |
| 500 | ExitWithError(Args: "Dummy perf counters are not supported in the subprocess " |
| 501 | "execution mode." ); |
| 502 | |
| 503 | const std::unique_ptr<BenchmarkRunner> Runner = |
| 504 | ExitOnErr(State.getExegesisTarget().createBenchmarkRunner( |
| 505 | Mode: BenchmarkMode, State, BenchmarkPhaseSelector, ExecutionMode, |
| 506 | BenchmarkRepeatCount, ValidationCounters, ResultAggMode)); |
| 507 | if (!Runner) { |
| 508 | ExitWithError(Args: "cannot create benchmark runner" ); |
| 509 | } |
| 510 | |
| 511 | const auto Opcodes = getOpcodesOrDie(State); |
| 512 | std::vector<BenchmarkCode> Configurations; |
| 513 | |
| 514 | MCRegister LoopRegister = |
| 515 | State.getExegesisTarget().getDefaultLoopCounterRegister( |
| 516 | State.getTargetMachine().getTargetTriple()); |
| 517 | |
| 518 | if (Opcodes.empty()) { |
| 519 | Configurations = ExitOnErr(readSnippets(State, Filename: SnippetsFile)); |
| 520 | for (const auto &Configuration : Configurations) { |
| 521 | if (ExecutionMode != BenchmarkRunner::ExecutionModeE::SubProcess && |
| 522 | (Configuration.Key.MemoryMappings.size() != 0 || |
| 523 | Configuration.Key.MemoryValues.size() != 0 || |
| 524 | Configuration.Key.SnippetAddress != 0)) |
| 525 | ExitWithError(Args: "Memory and snippet address annotations are only " |
| 526 | "supported in subprocess " |
| 527 | "execution mode" ); |
| 528 | } |
| 529 | LoopRegister = Configurations[0].Key.LoopRegister; |
| 530 | } |
| 531 | |
| 532 | SmallVector<std::unique_ptr<const SnippetRepetitor>, 2> Repetitors; |
| 533 | if (RepetitionMode != Benchmark::RepetitionModeE::AggregateMin) |
| 534 | Repetitors.emplace_back( |
| 535 | Args: SnippetRepetitor::Create(Mode: RepetitionMode, State, LoopRegister)); |
| 536 | else { |
| 537 | for (Benchmark::RepetitionModeE RepMode : |
| 538 | {Benchmark::RepetitionModeE::Duplicate, |
| 539 | Benchmark::RepetitionModeE::Loop}) |
| 540 | Repetitors.emplace_back( |
| 541 | Args: SnippetRepetitor::Create(Mode: RepMode, State, LoopRegister)); |
| 542 | } |
| 543 | |
| 544 | BitVector AllReservedRegs; |
| 545 | for (const std::unique_ptr<const SnippetRepetitor> &Repetitor : Repetitors) |
| 546 | AllReservedRegs |= Repetitor->getReservedRegs(); |
| 547 | |
| 548 | if (!Opcodes.empty()) { |
| 549 | for (const unsigned Opcode : Opcodes) { |
| 550 | // Ignore instructions without a sched class if |
| 551 | // -ignore-invalid-sched-class is passed. |
| 552 | if (IgnoreInvalidSchedClass && |
| 553 | State.getInstrInfo().get(Opcode).getSchedClass() == 0) { |
| 554 | errs() << State.getInstrInfo().getName(Opcode) |
| 555 | << ": ignoring instruction without sched class\n" ; |
| 556 | continue; |
| 557 | } |
| 558 | |
| 559 | auto ConfigsForInstr = generateSnippets(State, Opcode, ForbiddenRegs: AllReservedRegs); |
| 560 | if (!ConfigsForInstr) { |
| 561 | logAllUnhandledErrors( |
| 562 | E: ConfigsForInstr.takeError(), OS&: errs(), |
| 563 | ErrorBanner: Twine(State.getInstrInfo().getName(Opcode)).concat(Suffix: ": " )); |
| 564 | continue; |
| 565 | } |
| 566 | std::move(first: ConfigsForInstr->begin(), last: ConfigsForInstr->end(), |
| 567 | result: std::back_inserter(x&: Configurations)); |
| 568 | } |
| 569 | } |
| 570 | |
| 571 | if (MinInstructions == 0) { |
| 572 | ExitOnErr.setBanner("llvm-exegesis: " ); |
| 573 | ExitWithError(Args: "--min-instructions must be greater than zero" ); |
| 574 | } |
| 575 | |
| 576 | // Write to standard output if file is not set. |
| 577 | if (BenchmarkFile.empty()) |
| 578 | BenchmarkFile = "-" ; |
| 579 | |
| 580 | if (!Configurations.empty()) |
| 581 | runBenchmarkConfigurations(State, Configurations, Repetitors, Runner: *Runner); |
| 582 | |
| 583 | pfm::pfmTerminate(); |
| 584 | } |
| 585 | |
| 586 | // Prints the results of running analysis pass `Pass` to file `OutputFilename` |
| 587 | // if OutputFilename is non-empty. |
| 588 | template <typename Pass> |
| 589 | static void maybeRunAnalysis(const Analysis &Analyzer, const std::string &Name, |
| 590 | const std::string &OutputFilename) { |
| 591 | if (OutputFilename.empty()) |
| 592 | return; |
| 593 | if (OutputFilename != "-" ) { |
| 594 | errs() << "Printing " << Name << " results to file '" << OutputFilename |
| 595 | << "'\n" ; |
| 596 | } |
| 597 | std::error_code ErrorCode; |
| 598 | raw_fd_ostream ClustersOS(OutputFilename, ErrorCode, |
| 599 | sys::fs::FA_Read | sys::fs::FA_Write); |
| 600 | if (ErrorCode) |
| 601 | ExitOnFileError(FileName: OutputFilename, Err: errorCodeToError(EC: ErrorCode)); |
| 602 | if (auto Err = Analyzer.run<Pass>(ClustersOS)) |
| 603 | ExitOnFileError(OutputFilename, std::move(Err)); |
| 604 | } |
| 605 | |
| 606 | static void filterPoints(MutableArrayRef<Benchmark> Points, |
| 607 | const MCInstrInfo &MCII) { |
| 608 | if (AnalysisSnippetFilter == BenchmarkFilter::All) |
| 609 | return; |
| 610 | |
| 611 | bool WantPointsWithMemOps = AnalysisSnippetFilter == BenchmarkFilter::WithMem; |
| 612 | for (Benchmark &Point : Points) { |
| 613 | if (!Point.Error.empty()) |
| 614 | continue; |
| 615 | if (WantPointsWithMemOps == |
| 616 | any_of(Range&: Point.Key.Instructions, P: [&MCII](const MCInst &Inst) { |
| 617 | const MCInstrDesc &MCDesc = MCII.get(Opcode: Inst.getOpcode()); |
| 618 | return MCDesc.mayLoad() || MCDesc.mayStore(); |
| 619 | })) |
| 620 | continue; |
| 621 | Point.Error = "filtered out by user" ; |
| 622 | } |
| 623 | } |
| 624 | |
| 625 | static void analysisMain(const cl::LibraryOptionsContext &LibraryOptions) { |
| 626 | ExitOnErr.setBanner("llvm-exegesis: " ); |
| 627 | if (BenchmarkFile.empty()) |
| 628 | ExitWithError(Args: "--benchmarks-file must be set" ); |
| 629 | |
| 630 | if (AnalysisClustersOutputFile.empty() && |
| 631 | AnalysisInconsistenciesOutputFile.empty()) { |
| 632 | ExitWithError( |
| 633 | Args: "for --mode=analysis: At least one of --analysis-clusters-output-file " |
| 634 | "and --analysis-inconsistencies-output-file must be specified" ); |
| 635 | } |
| 636 | |
| 637 | InitializeAllExegesisTargets(); |
| 638 | #define LLVM_EXEGESIS(TargetName) \ |
| 639 | LLVMInitialize##TargetName##AsmPrinter(); \ |
| 640 | LLVMInitialize##TargetName##Disassembler(); |
| 641 | #include "llvm/Config/TargetExegesis.def" |
| 642 | |
| 643 | auto MemoryBuffer = ExitOnFileError( |
| 644 | FileName: BenchmarkFile, |
| 645 | E: errorOrToExpected(EO: MemoryBuffer::getFile(Filename: BenchmarkFile, /*IsText=*/true))); |
| 646 | |
| 647 | const auto TriplesAndCpus = ExitOnFileError( |
| 648 | FileName: BenchmarkFile, |
| 649 | E: Benchmark::readTriplesAndCpusFromYamls(Buffer: *MemoryBuffer)); |
| 650 | if (TriplesAndCpus.empty()) { |
| 651 | errs() << "no benchmarks to analyze\n" ; |
| 652 | return; |
| 653 | } |
| 654 | if (TriplesAndCpus.size() > 1) { |
| 655 | ExitWithError(Args: "analysis file contains benchmarks from several CPUs. This " |
| 656 | "is unsupported." ); |
| 657 | } |
| 658 | auto TripleAndCpu = *TriplesAndCpus.begin(); |
| 659 | if (AnalysisOverrideBenchmarksTripleAndCpu) { |
| 660 | errs() << "overridding file CPU name (" << TripleAndCpu.CpuName |
| 661 | << ") with provided tripled (" << TripleName << ") and CPU name (" |
| 662 | << MCPU << ")\n" ; |
| 663 | TripleAndCpu.LLVMTriple = TripleName; |
| 664 | TripleAndCpu.CpuName = MCPU; |
| 665 | } |
| 666 | errs() << "using Triple '" << TripleAndCpu.LLVMTriple << "' and CPU '" |
| 667 | << TripleAndCpu.CpuName << "'\n" ; |
| 668 | |
| 669 | // Read benchmarks. |
| 670 | const LLVMState State = ExitOnErr(LLVMState::create( |
| 671 | LibraryOptions, TripleName: TripleAndCpu.LLVMTriple, CpuName: TripleAndCpu.CpuName)); |
| 672 | std::vector<Benchmark> Points = ExitOnFileError( |
| 673 | FileName: BenchmarkFile, E: Benchmark::readYamls(State, Buffer: *MemoryBuffer)); |
| 674 | |
| 675 | outs() << "Parsed " << Points.size() << " benchmark points\n" ; |
| 676 | if (Points.empty()) { |
| 677 | errs() << "no benchmarks to analyze\n" ; |
| 678 | return; |
| 679 | } |
| 680 | // FIXME: Merge points from several runs (latency and uops). |
| 681 | |
| 682 | filterPoints(Points, MCII: State.getInstrInfo()); |
| 683 | |
| 684 | const auto Clustering = ExitOnErr(BenchmarkClustering::create( |
| 685 | Points, Mode: AnalysisClusteringAlgorithm, DbscanMinPts: AnalysisDbscanNumPoints, |
| 686 | AnalysisClusteringEpsilon, SubtargetInfo: &State.getSubtargetInfo(), |
| 687 | InstrInfo: &State.getInstrInfo())); |
| 688 | |
| 689 | const Analysis Analyzer(State, Clustering, AnalysisInconsistencyEpsilon, |
| 690 | AnalysisDisplayUnstableOpcodes); |
| 691 | |
| 692 | maybeRunAnalysis<Analysis::PrintClusters>(Analyzer, Name: "analysis clusters" , |
| 693 | OutputFilename: AnalysisClustersOutputFile); |
| 694 | maybeRunAnalysis<Analysis::PrintSchedClassInconsistencies>( |
| 695 | Analyzer, Name: "sched class consistency analysis" , |
| 696 | OutputFilename: AnalysisInconsistenciesOutputFile); |
| 697 | } |
| 698 | |
| 699 | } // namespace exegesis |
| 700 | } // namespace llvm |
| 701 | |
| 702 | int main(int Argc, char **Argv) { |
| 703 | using namespace llvm; |
| 704 | |
| 705 | InitLLVM X(Argc, Argv); |
| 706 | |
| 707 | // Initialize targets so we can print them when flag --version is specified. |
| 708 | #define LLVM_EXEGESIS(TargetName) \ |
| 709 | LLVMInitialize##TargetName##Target(); \ |
| 710 | LLVMInitialize##TargetName##TargetInfo(); \ |
| 711 | LLVMInitialize##TargetName##TargetMC(); |
| 712 | #include "llvm/Config/TargetExegesis.def" |
| 713 | |
| 714 | // Register the Target and CPU printer for --version. |
| 715 | cl::AddExtraVersionPrinter(func: sys::printDefaultTargetAndDetectedCPU); |
| 716 | |
| 717 | // Enable printing of available targets when flag --version is specified. |
| 718 | cl::AddExtraVersionPrinter(func: TargetRegistry::printRegisteredTargetsForVersion); |
| 719 | |
| 720 | cl::HideUnrelatedOptions(Categories: {&exegesis::Options, &exegesis::BenchmarkOptions, |
| 721 | &exegesis::AnalysisOptions}); |
| 722 | |
| 723 | cl::LibraryOptionsContext LibraryOptions; |
| 724 | cl::parseCommandLineOptions(Context&: LibraryOptions, Args: ArrayRef(Argv, Argc), |
| 725 | Overview: "llvm host machine instruction characteristics " |
| 726 | "measurment and analysis.\n" ); |
| 727 | |
| 728 | exegesis::ExitOnErr.setExitCodeMapper([](const Error &Err) { |
| 729 | if (Err.isA<exegesis::ClusteringError>()) |
| 730 | return EXIT_SUCCESS; |
| 731 | return EXIT_FAILURE; |
| 732 | }); |
| 733 | |
| 734 | if (exegesis::BenchmarkMode == exegesis::Benchmark::Unknown) { |
| 735 | exegesis::analysisMain(LibraryOptions); |
| 736 | } else { |
| 737 | exegesis::benchmarkMain(LibraryOptions); |
| 738 | } |
| 739 | return EXIT_SUCCESS; |
| 740 | } |
| 741 | |