1//===-------------- RemarkSizeDiff.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Diffs instruction count and stack size remarks between two remark files.
11///
12/// This is intended for use by compiler developers who want to see how their
13/// changes impact program code size.
14///
15//===----------------------------------------------------------------------===//
16
17#include "RemarkUtilHelpers.h"
18#include "RemarkUtilRegistry.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/Support/FormatVariadic.h"
21#include "llvm/Support/JSON.h"
22
23using namespace llvm;
24using namespace remarks;
25using namespace remarkutil;
26static cl::SubCommand
27 RemarkSizeDiffUtil("size-diff",
28 "Diff instruction count and stack size remarks "
29 "between two remark files");
30enum ReportStyleOptions { human_output, json_output };
31static cl::opt<std::string> InputFileNameA(cl::Positional, cl::Required,
32 cl::sub(RemarkSizeDiffUtil),
33 cl::desc("remarks_a"));
34static cl::opt<std::string> InputFileNameB(cl::Positional, cl::Required,
35 cl::sub(RemarkSizeDiffUtil),
36 cl::desc("remarks_b"));
37static cl::opt<std::string> OutputFilename("o", cl::init(Val: "-"),
38 cl::sub(RemarkSizeDiffUtil),
39 cl::desc("Output"),
40 cl::value_desc("file"));
41INPUT_FORMAT_COMMAND_LINE_OPTIONS(RemarkSizeDiffUtil)
42static cl::opt<ReportStyleOptions> ReportStyle(
43 "report_style", cl::sub(RemarkSizeDiffUtil),
44 cl::init(Val: ReportStyleOptions::human_output),
45 cl::desc("Choose the report output format:"),
46 cl::values(clEnumValN(human_output, "human", "Human-readable format"),
47 clEnumValN(json_output, "json", "JSON format")));
48static cl::opt<bool> PrettyPrint("pretty", cl::sub(RemarkSizeDiffUtil),
49 cl::init(Val: false),
50 cl::desc("Pretty-print JSON"));
51
52/// Contains information from size remarks.
53// This is a little nicer to read than a std::pair.
54struct InstCountAndStackSize {
55 int64_t InstCount = 0;
56 int64_t StackSize = 0;
57};
58
59/// Represents which files a function appeared in.
60enum FilesPresent { A, B, BOTH };
61
62/// Contains the data from the remarks in file A and file B for some function.
63/// E.g. instruction count, stack size...
64struct FunctionDiff {
65 /// Function name from the remark.
66 std::string FuncName;
67 // Idx 0 = A, Idx 1 = B.
68 int64_t InstCount[2] = {0, 0};
69 int64_t StackSize[2] = {0, 0};
70
71 // Calculate diffs between the first and second files.
72 int64_t getInstDiff() const { return InstCount[1] - InstCount[0]; }
73 int64_t getStackDiff() const { return StackSize[1] - StackSize[0]; }
74
75 // Accessors for the remarks from the first file.
76 int64_t getInstCountA() const { return InstCount[0]; }
77 int64_t getStackSizeA() const { return StackSize[0]; }
78
79 // Accessors for the remarks from the second file.
80 int64_t getInstCountB() const { return InstCount[1]; }
81 int64_t getStackSizeB() const { return StackSize[1]; }
82
83 /// \returns which files this function was present in.
84 FilesPresent getFilesPresent() const {
85 if (getInstCountA() == 0)
86 return B;
87 if (getInstCountB() == 0)
88 return A;
89 return BOTH;
90 }
91
92 FunctionDiff(StringRef FuncName, const InstCountAndStackSize &A,
93 const InstCountAndStackSize &B)
94 : FuncName(FuncName) {
95 InstCount[0] = A.InstCount;
96 InstCount[1] = B.InstCount;
97 StackSize[0] = A.StackSize;
98 StackSize[1] = B.StackSize;
99 }
100};
101
102/// Organizes the diffs into 3 categories:
103/// - Functions which only appeared in the first file
104/// - Functions which only appeared in the second file
105/// - Functions which appeared in both files
106struct DiffsCategorizedByFilesPresent {
107 /// Diffs for functions which only appeared in the first file.
108 SmallVector<FunctionDiff> OnlyInA;
109
110 /// Diffs for functions which only appeared in the second file.
111 SmallVector<FunctionDiff> OnlyInB;
112
113 /// Diffs for functions which appeared in both files.
114 SmallVector<FunctionDiff> InBoth;
115
116 /// Add a diff to the appropriate list.
117 void addDiff(FunctionDiff &FD) {
118 switch (FD.getFilesPresent()) {
119 case A:
120 OnlyInA.push_back(Elt: FD);
121 break;
122 case B:
123 OnlyInB.push_back(Elt: FD);
124 break;
125 case BOTH:
126 InBoth.push_back(Elt: FD);
127 break;
128 }
129 }
130};
131
132static void printFunctionDiff(const FunctionDiff &FD, llvm::raw_ostream &OS) {
133 // Describe which files the function had remarks in.
134 FilesPresent FP = FD.getFilesPresent();
135 const std::string &FuncName = FD.FuncName;
136 const int64_t InstDiff = FD.getInstDiff();
137 assert(InstDiff && "Shouldn't get functions with no size change?");
138 const int64_t StackDiff = FD.getStackDiff();
139 // Output an indicator denoting which files the function was present in.
140 switch (FP) {
141 case FilesPresent::A:
142 OS << "-- ";
143 break;
144 case FilesPresent::B:
145 OS << "++ ";
146 break;
147 case FilesPresent::BOTH:
148 OS << "== ";
149 break;
150 }
151 // Output an indicator denoting if a function changed in size.
152 if (InstDiff > 0)
153 OS << "> ";
154 else
155 OS << "< ";
156 OS << FuncName << ", ";
157 OS << InstDiff << " instrs, ";
158 OS << StackDiff << " stack B";
159 OS << "\n";
160}
161
162/// Print an item in the summary section.
163///
164/// \p TotalA - Total count of the metric in file A.
165/// \p TotalB - Total count of the metric in file B.
166/// \p Metric - Name of the metric we want to print (e.g. instruction
167/// count).
168/// \p OS - The output stream.
169static void printSummaryItem(int64_t TotalA, int64_t TotalB, StringRef Metric,
170 llvm::raw_ostream &OS) {
171 OS << " " << Metric << ": ";
172 int64_t TotalDiff = TotalB - TotalA;
173 if (TotalDiff == 0) {
174 OS << "None\n";
175 return;
176 }
177 OS << TotalDiff << " (" << formatv(Fmt: "{0:p}", Vals: TotalDiff / (double)TotalA)
178 << ")\n";
179}
180
181/// Print all contents of \p Diff and a high-level summary of the differences.
182static void printDiffsCategorizedByFilesPresent(
183 DiffsCategorizedByFilesPresent &DiffsByFilesPresent,
184 llvm::raw_ostream &OS) {
185 int64_t InstrsA = 0;
186 int64_t InstrsB = 0;
187 int64_t StackA = 0;
188 int64_t StackB = 0;
189 // Helper lambda to sort + print a list of diffs.
190 auto PrintDiffList = [&](SmallVector<FunctionDiff> &FunctionDiffList) {
191 if (FunctionDiffList.empty())
192 return;
193 stable_sort(Range&: FunctionDiffList,
194 C: [](const FunctionDiff &LHS, const FunctionDiff &RHS) {
195 return LHS.getInstDiff() < RHS.getInstDiff();
196 });
197 for (const auto &FuncDiff : FunctionDiffList) {
198 // If there is a difference in instruction count, then print out info for
199 // the function.
200 if (FuncDiff.getInstDiff())
201 printFunctionDiff(FD: FuncDiff, OS);
202 InstrsA += FuncDiff.getInstCountA();
203 InstrsB += FuncDiff.getInstCountB();
204 StackA += FuncDiff.getStackSizeA();
205 StackB += FuncDiff.getStackSizeB();
206 }
207 };
208 PrintDiffList(DiffsByFilesPresent.OnlyInA);
209 PrintDiffList(DiffsByFilesPresent.OnlyInB);
210 PrintDiffList(DiffsByFilesPresent.InBoth);
211 OS << "\n### Summary ###\n";
212 OS << "Total change: \n";
213 printSummaryItem(TotalA: InstrsA, TotalB: InstrsB, Metric: "instruction count", OS);
214 printSummaryItem(TotalA: StackA, TotalB: StackB, Metric: "stack byte usage", OS);
215}
216
217/// Collects an expected integer value from a given argument index in a remark.
218///
219/// \p Remark - The remark.
220/// \p ArgIdx - The index where the integer value should be found.
221/// \p ExpectedKeyName - The expected key name for the index
222/// (e.g. "InstructionCount")
223///
224/// \returns the integer value at the index if it exists, and the key-value pair
225/// is what is expected. Otherwise, returns an Error.
226static Expected<int64_t> getIntValFromKey(const remarks::Remark &Remark,
227 unsigned ArgIdx,
228 StringRef ExpectedKeyName) {
229 auto KeyName = Remark.Args[ArgIdx].Key;
230 if (KeyName != ExpectedKeyName)
231 return createStringError(
232 EC: inconvertibleErrorCode(),
233 S: Twine("Unexpected key at argument index " + std::to_string(val: ArgIdx) +
234 ": Expected '" + ExpectedKeyName + "', got '" + KeyName + "'"));
235 long long Val;
236 auto ValStr = Remark.Args[ArgIdx].Val;
237 if (getAsSignedInteger(Str: ValStr, Radix: 0, Result&: Val))
238 return createStringError(
239 EC: inconvertibleErrorCode(),
240 S: Twine("Could not convert string to signed integer: " + ValStr));
241 return static_cast<int64_t>(Val);
242}
243
244/// Collects relevant size information from \p Remark if it is an size-related
245/// remark of some kind (e.g. instruction count). Otherwise records nothing.
246///
247/// \p Remark - The remark.
248/// \p FuncNameToSizeInfo - Maps function names to relevant size info.
249/// \p NumInstCountRemarksParsed - Keeps track of the number of instruction
250/// count remarks parsed. We need at least 1 in both files to produce a diff.
251static Error processRemark(const remarks::Remark &Remark,
252 StringMap<InstCountAndStackSize> &FuncNameToSizeInfo,
253 unsigned &NumInstCountRemarksParsed) {
254 const auto &RemarkName = Remark.RemarkName;
255 const auto &PassName = Remark.PassName;
256 // Collect remarks which contain the number of instructions in a function.
257 if (PassName == "asm-printer" && RemarkName == "InstructionCount") {
258 // Expecting the 0-th argument to have the key "NumInstructions" and an
259 // integer value.
260 auto MaybeInstCount =
261 getIntValFromKey(Remark, /*ArgIdx = */ 0, ExpectedKeyName: "NumInstructions");
262 if (!MaybeInstCount)
263 return MaybeInstCount.takeError();
264 FuncNameToSizeInfo[Remark.FunctionName].InstCount = *MaybeInstCount;
265 ++NumInstCountRemarksParsed;
266 }
267 // Collect remarks which contain the stack size of a function.
268 else if (PassName == "prologepilog" && RemarkName == "StackSize") {
269 // Expecting the 0-th argument to have the key "NumStackBytes" and an
270 // integer value.
271 auto MaybeStackSize =
272 getIntValFromKey(Remark, /*ArgIdx = */ 0, ExpectedKeyName: "NumStackBytes");
273 if (!MaybeStackSize)
274 return MaybeStackSize.takeError();
275 FuncNameToSizeInfo[Remark.FunctionName].StackSize = *MaybeStackSize;
276 }
277 // Either we collected a remark, or it's something we don't care about. In
278 // both cases, this is a success.
279 return Error::success();
280}
281
282/// Process all of the size-related remarks in a file.
283///
284/// \param[in] InputFileName - Name of file to read from.
285/// \param[in, out] FuncNameToSizeInfo - Maps function names to relevant
286/// size info.
287static Error readFileAndProcessRemarks(
288 StringRef InputFileName,
289 StringMap<InstCountAndStackSize> &FuncNameToSizeInfo) {
290
291 auto MaybeBuf = getInputMemoryBuffer(InputFileName);
292 if (!MaybeBuf)
293 return MaybeBuf.takeError();
294 auto MaybeParser =
295 createRemarkParserFromMeta(ParserFormat: InputFormat, Buf: (*MaybeBuf)->getBuffer());
296 if (!MaybeParser)
297 return MaybeParser.takeError();
298 auto &Parser = **MaybeParser;
299 auto MaybeRemark = Parser.next();
300 unsigned NumInstCountRemarksParsed = 0;
301 for (; MaybeRemark; MaybeRemark = Parser.next()) {
302 if (auto E = processRemark(Remark: **MaybeRemark, FuncNameToSizeInfo,
303 NumInstCountRemarksParsed))
304 return E;
305 }
306 auto E = MaybeRemark.takeError();
307 if (!E.isA<remarks::EndOfFileError>())
308 return E;
309 consumeError(Err: std::move(E));
310 // We need at least one instruction count remark in each file to produce a
311 // meaningful diff.
312 if (NumInstCountRemarksParsed == 0)
313 return createStringError(
314 EC: inconvertibleErrorCode(),
315 S: "File '" + InputFileName +
316 "' did not contain any instruction-count remarks!");
317 return Error::success();
318}
319
320/// Wrapper function for readFileAndProcessRemarks which handles errors.
321///
322/// \param[in] InputFileName - Name of file to read from.
323/// \param[out] FuncNameToSizeInfo - Populated with information from size
324/// remarks in the input file.
325///
326/// \returns true if readFileAndProcessRemarks returned no errors. False
327/// otherwise.
328static Error tryReadFileAndProcessRemarks(
329 StringRef InputFileName,
330 StringMap<InstCountAndStackSize> &FuncNameToSizeInfo) {
331 if (Error E = readFileAndProcessRemarks(InputFileName, FuncNameToSizeInfo)) {
332 return E;
333 }
334 return Error::success();
335}
336
337/// Populates \p FuncDiffs with the difference between \p
338/// FuncNameToSizeInfoA and \p FuncNameToSizeInfoB.
339///
340/// \param[in] FuncNameToSizeInfoA - Size info collected from the first
341/// remarks file.
342/// \param[in] FuncNameToSizeInfoB - Size info collected from
343/// the second remarks file.
344/// \param[out] DiffsByFilesPresent - Filled with the diff between \p
345/// FuncNameToSizeInfoA and \p FuncNameToSizeInfoB.
346static void
347computeDiff(const StringMap<InstCountAndStackSize> &FuncNameToSizeInfoA,
348 const StringMap<InstCountAndStackSize> &FuncNameToSizeInfoB,
349 DiffsCategorizedByFilesPresent &DiffsByFilesPresent) {
350 SmallSet<std::string, 10> FuncNames;
351 for (const auto &FuncName : FuncNameToSizeInfoA.keys())
352 FuncNames.insert(V: FuncName.str());
353 for (const auto &FuncName : FuncNameToSizeInfoB.keys())
354 FuncNames.insert(V: FuncName.str());
355 for (const std::string &FuncName : FuncNames) {
356 const auto &SizeInfoA = FuncNameToSizeInfoA.lookup(Key: FuncName);
357 const auto &SizeInfoB = FuncNameToSizeInfoB.lookup(Key: FuncName);
358 FunctionDiff FuncDiff(FuncName, SizeInfoA, SizeInfoB);
359 DiffsByFilesPresent.addDiff(FD&: FuncDiff);
360 }
361}
362
363/// Attempt to get the output stream for writing the diff.
364static ErrorOr<std::unique_ptr<ToolOutputFile>> getOutputStream() {
365 if (OutputFilename == "")
366 OutputFilename = "-";
367 std::error_code EC;
368 auto Out = std::make_unique<ToolOutputFile>(args&: OutputFilename, args&: EC,
369 args: sys::fs::OF_TextWithCRLF);
370 if (!EC)
371 return std::move(Out);
372 return EC;
373}
374
375/// \return a json::Array representing all FunctionDiffs in \p FunctionDiffs.
376/// \p WhichFiles represents which files the functions in \p FunctionDiffs
377/// appeared in (A, B, or both).
378json::Array
379getFunctionDiffListAsJSON(const SmallVector<FunctionDiff> &FunctionDiffs,
380 const FilesPresent &WhichFiles) {
381 json::Array FunctionDiffsAsJSON;
382 int64_t InstCountA, InstCountB, StackSizeA, StackSizeB;
383 for (auto &Diff : FunctionDiffs) {
384 InstCountA = InstCountB = StackSizeA = StackSizeB = 0;
385 switch (WhichFiles) {
386 case BOTH:
387 [[fallthrough]];
388 case A:
389 InstCountA = Diff.getInstCountA();
390 StackSizeA = Diff.getStackSizeA();
391 if (WhichFiles != BOTH)
392 break;
393 [[fallthrough]];
394 case B:
395 InstCountB = Diff.getInstCountB();
396 StackSizeB = Diff.getStackSizeB();
397 break;
398 }
399 // Each metric we care about is represented like:
400 // "Val": [A, B]
401 // This allows any consumer of the JSON to calculate the diff using B - A.
402 // This is somewhat wasteful for OnlyInA and OnlyInB (we only need A or B).
403 // However, this should make writing consuming tools easier, since the tool
404 // writer doesn't need to think about slightly different formats in each
405 // section.
406 json::Object FunctionObject({{.K: "FunctionName", .V: Diff.FuncName},
407 {.K: "InstCount", .V: {InstCountA, InstCountB}},
408 {.K: "StackSize", .V: {StackSizeA, StackSizeB}}});
409 FunctionDiffsAsJSON.push_back(E: std::move(FunctionObject));
410 }
411 return FunctionDiffsAsJSON;
412}
413
414/// Output all diffs in \p DiffsByFilesPresent as a JSON report. This is
415/// intended for consumption by external tools.
416///
417/// \p InputFileNameA - File A used to produce the report.
418/// \p InputFileNameB - File B used ot produce the report.
419/// \p OS - Output stream.
420///
421/// JSON output includes:
422/// - \p InputFileNameA and \p InputFileNameB under "Files".
423/// - Functions present in both files under "InBoth".
424/// - Functions present only in A in "OnlyInA".
425/// - Functions present only in B in "OnlyInB".
426/// - Instruction count and stack size differences for each function.
427///
428/// Differences are represented using [count_a, count_b]. The actual difference
429/// can be computed via count_b - count_a.
430static void
431outputJSONForAllDiffs(StringRef InputFileNameA, StringRef InputFileNameB,
432 const DiffsCategorizedByFilesPresent &DiffsByFilesPresent,
433 llvm::raw_ostream &OS) {
434 json::Object Output;
435 // Include file names in the report.
436 json::Object Files(
437 {{.K: "A", .V: InputFileNameA.str()}, {.K: "B", .V: InputFileNameB.str()}});
438 Output["Files"] = std::move(Files);
439 Output["OnlyInA"] = getFunctionDiffListAsJSON(FunctionDiffs: DiffsByFilesPresent.OnlyInA, WhichFiles: A);
440 Output["OnlyInB"] = getFunctionDiffListAsJSON(FunctionDiffs: DiffsByFilesPresent.OnlyInB, WhichFiles: B);
441 Output["InBoth"] =
442 getFunctionDiffListAsJSON(FunctionDiffs: DiffsByFilesPresent.InBoth, WhichFiles: BOTH);
443 json::OStream JOS(OS, PrettyPrint ? 2 : 0);
444 JOS.value(V: std::move(Output));
445 OS << '\n';
446}
447
448/// Output all diffs in \p DiffsByFilesPresent using the desired output style.
449/// \returns Error::success() on success, and an Error otherwise.
450/// \p InputFileNameA - Name of input file A; may be used in the report.
451/// \p InputFileNameB - Name of input file B; may be used in the report.
452static Error
453outputAllDiffs(StringRef InputFileNameA, StringRef InputFileNameB,
454 DiffsCategorizedByFilesPresent &DiffsByFilesPresent) {
455 auto MaybeOF = getOutputStream();
456 if (std::error_code EC = MaybeOF.getError())
457 return errorCodeToError(EC);
458 std::unique_ptr<ToolOutputFile> OF = std::move(*MaybeOF);
459 switch (ReportStyle) {
460 case human_output:
461 printDiffsCategorizedByFilesPresent(DiffsByFilesPresent, OS&: OF->os());
462 break;
463 case json_output:
464 outputJSONForAllDiffs(InputFileNameA, InputFileNameB, DiffsByFilesPresent,
465 OS&: OF->os());
466 break;
467 }
468 OF->keep();
469 return Error::success();
470}
471
472/// Boolean wrapper for outputDiff which handles errors.
473static Error
474tryOutputAllDiffs(StringRef InputFileNameA, StringRef InputFileNameB,
475 DiffsCategorizedByFilesPresent &DiffsByFilesPresent) {
476 if (Error E =
477 outputAllDiffs(InputFileNameA, InputFileNameB, DiffsByFilesPresent)) {
478 return E;
479 }
480 return Error::success();
481}
482
483static Error trySizeSiff() {
484 StringMap<InstCountAndStackSize> FuncNameToSizeInfoA;
485 StringMap<InstCountAndStackSize> FuncNameToSizeInfoB;
486 if (auto E =
487 tryReadFileAndProcessRemarks(InputFileName: InputFileNameA, FuncNameToSizeInfo&: FuncNameToSizeInfoA))
488 return E;
489 if (auto E =
490 tryReadFileAndProcessRemarks(InputFileName: InputFileNameB, FuncNameToSizeInfo&: FuncNameToSizeInfoB))
491 return E;
492 DiffsCategorizedByFilesPresent DiffsByFilesPresent;
493 computeDiff(FuncNameToSizeInfoA, FuncNameToSizeInfoB, DiffsByFilesPresent);
494 if (auto E = tryOutputAllDiffs(InputFileNameA, InputFileNameB,
495 DiffsByFilesPresent))
496 return E;
497 return Error::success();
498}
499
500static CommandRegistration RemarkSizeSiffRegister(&RemarkSizeDiffUtil,
501 trySizeSiff);
502