1//===- FuzzerFork.cpp - run fuzzing in separate subprocesses --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// Spawn and orchestrate separate fuzzing processes.
9//===----------------------------------------------------------------------===//
10
11#include "FuzzerCommand.h"
12#include "FuzzerFork.h"
13#include "FuzzerIO.h"
14#include "FuzzerInternal.h"
15#include "FuzzerMerge.h"
16#include "FuzzerSHA1.h"
17#include "FuzzerTracePC.h"
18#include "FuzzerUtil.h"
19
20#include <atomic>
21#include <chrono>
22#include <condition_variable>
23#include <fstream>
24#include <memory>
25#include <mutex>
26#include <queue>
27#include <sstream>
28#include <thread>
29
30namespace fuzzer {
31
32struct Stats {
33 size_t number_of_executed_units = 0;
34 size_t peak_rss_mb = 0;
35 size_t average_exec_per_sec = 0;
36};
37
38static Stats ParseFinalStatsFromLog(const std::string &LogPath) {
39 std::ifstream In(LogPath);
40 std::string Line;
41 Stats Res;
42 struct {
43 const char *Name;
44 size_t *Var;
45 } NameVarPairs[] = {
46 {.Name: "stat::number_of_executed_units:", .Var: &Res.number_of_executed_units},
47 {.Name: "stat::peak_rss_mb:", .Var: &Res.peak_rss_mb},
48 {.Name: "stat::average_exec_per_sec:", .Var: &Res.average_exec_per_sec},
49 {.Name: nullptr, .Var: nullptr},
50 };
51 while (std::getline(is&: In, str&: Line, dlm: '\n')) {
52 if (Line.find(s: "stat::") != 0) continue;
53 std::istringstream ISS(Line);
54 std::string Name;
55 size_t Val;
56 ISS >> Name >> Val;
57 for (size_t i = 0; NameVarPairs[i].Name; i++)
58 if (Name == NameVarPairs[i].Name)
59 *NameVarPairs[i].Var = Val;
60 }
61 return Res;
62}
63
64struct FuzzJob {
65 // Inputs.
66 Command Cmd;
67 std::string CorpusDir;
68 std::string FeaturesDir;
69 std::string LogPath;
70 std::string SeedListPath;
71 std::string CFPath;
72 size_t JobId;
73
74 int DftTimeInSeconds = 0;
75
76 // Fuzzing Outputs.
77 int ExitCode;
78
79 ~FuzzJob() {
80 RemoveFile(Path: CFPath);
81 RemoveFile(Path: LogPath);
82 RemoveFile(Path: SeedListPath);
83 RmDirRecursive(Dir: CorpusDir);
84 RmDirRecursive(Dir: FeaturesDir);
85 }
86};
87
88struct GlobalEnv {
89 std::vector<std::string> Args;
90 std::vector<std::string> CorpusDirs;
91 std::string MainCorpusDir;
92 std::string TempDir;
93 std::string DFTDir;
94 std::string DataFlowBinary;
95 std::set<uint32_t> Features, Cov;
96 std::set<std::string> FilesWithDFT;
97 std::vector<std::string> Files;
98 std::vector<std::size_t> FilesSizes;
99 Random *Rand;
100 std::chrono::system_clock::time_point ProcessStartTime;
101 std::chrono::system_clock::time_point LastNewCorpusTime;
102 int Verbosity = 0;
103 int Group = 0;
104 int NumCorpuses = 8;
105
106 size_t NumTimeouts = 0;
107 size_t NumOOMs = 0;
108 size_t NumCrashes = 0;
109
110
111 size_t NumRuns = 0;
112
113 std::string StopFile() { return DirPlusFile(DirPath: TempDir, FileName: "STOP"); }
114
115 size_t secondsSinceProcessStartUp() const {
116 return std::chrono::duration_cast<std::chrono::seconds>(
117 fd: std::chrono::system_clock::now() - ProcessStartTime)
118 .count();
119 }
120
121 size_t secondsSinceLastNewCorpus() const {
122 return std::chrono::duration_cast<std::chrono::seconds>(
123 fd: std::chrono::system_clock::now() - LastNewCorpusTime)
124 .count();
125 }
126
127 FuzzJob *CreateNewJob(size_t JobId) {
128 Command Cmd(Args);
129 Cmd.removeFlag(Flag: "fork");
130 Cmd.removeFlag(Flag: "runs");
131 Cmd.removeFlag(Flag: "collect_data_flow");
132 Cmd.removeFlag(Flag: "stale_corpus_timeout");
133 for (auto &C : CorpusDirs) // Remove all corpora from the args.
134 Cmd.removeArgument(Arg: C);
135 Cmd.addFlag(Flag: "reload", Value: "0"); // working in an isolated dir, no reload.
136 Cmd.addFlag(Flag: "print_final_stats", Value: "1");
137 Cmd.addFlag(Flag: "print_funcs", Value: "0"); // no need to spend time symbolizing.
138 Cmd.addFlag(Flag: "max_total_time", Value: std::to_string(val: std::min(a: (size_t)300, b: JobId)));
139 Cmd.addFlag(Flag: "stop_file", Value: StopFile());
140 if (!DataFlowBinary.empty()) {
141 Cmd.addFlag(Flag: "data_flow_trace", Value: DFTDir);
142 if (!Cmd.hasFlag(Flag: "focus_function"))
143 Cmd.addFlag(Flag: "focus_function", Value: "auto");
144 }
145 auto Job = new FuzzJob;
146 std::string Seeds;
147 if (size_t CorpusSubsetSize =
148 std::min(a: Files.size(), b: (size_t)sqrt(x: Files.size() + 2))) {
149 auto Time1 = std::chrono::system_clock::now();
150 if (Group) { // whether to group the corpus.
151 size_t AverageCorpusSize = Files.size() / NumCorpuses + 1;
152 size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageCorpusSize;
153 for (size_t i = 0; i < CorpusSubsetSize; i++) {
154 size_t RandNum = (*Rand)(AverageCorpusSize);
155 size_t Index = RandNum + StartIndex;
156 Index = Index < Files.size() ? Index
157 : Rand->SkewTowardsLast(n: Files.size());
158 auto &SF = Files[Index];
159 Seeds += (Seeds.empty() ? "" : ",") + SF;
160 CollectDFT(InputPath: SF);
161 }
162 } else {
163 for (size_t i = 0; i < CorpusSubsetSize; i++) {
164 auto &SF = Files[Rand->SkewTowardsLast(n: Files.size())];
165 Seeds += (Seeds.empty() ? "" : ",") + SF;
166 CollectDFT(InputPath: SF);
167 }
168 }
169 auto Time2 = std::chrono::system_clock::now();
170 auto DftTimeInSeconds = duration_cast<seconds>(fd: Time2 - Time1).count();
171 assert(DftTimeInSeconds < std::numeric_limits<int>::max());
172 Job->DftTimeInSeconds = static_cast<int>(DftTimeInSeconds);
173 }
174 if (!Seeds.empty()) {
175 Job->SeedListPath =
176 DirPlusFile(DirPath: TempDir, FileName: std::to_string(val: JobId) + ".seeds");
177 WriteToFile(Data: Seeds, Path: Job->SeedListPath);
178 Cmd.addFlag(Flag: "seed_inputs", Value: "@" + Job->SeedListPath);
179 }
180 Job->LogPath = DirPlusFile(DirPath: TempDir, FileName: std::to_string(val: JobId) + ".log");
181 Job->CorpusDir = DirPlusFile(DirPath: TempDir, FileName: "C" + std::to_string(val: JobId));
182 Job->FeaturesDir = DirPlusFile(DirPath: TempDir, FileName: "F" + std::to_string(val: JobId));
183 Job->CFPath = DirPlusFile(DirPath: TempDir, FileName: std::to_string(val: JobId) + ".merge");
184 Job->JobId = JobId;
185
186
187 Cmd.addArgument(Arg: Job->CorpusDir);
188 Cmd.addFlag(Flag: "features_dir", Value: Job->FeaturesDir);
189
190 for (auto &D : {Job->CorpusDir, Job->FeaturesDir}) {
191 RmDirRecursive(Dir: D);
192 MkDir(Path: D);
193 }
194
195 Cmd.setOutputFile(Job->LogPath);
196 Cmd.combineOutAndErr();
197
198 Job->Cmd = Cmd;
199
200 if (Verbosity >= 2)
201 Printf(Fmt: "Job %zd/%p Created: %s\n", JobId, Job,
202 Job->Cmd.toString().c_str());
203 // Start from very short runs and gradually increase them.
204 return Job;
205 }
206
207 void PrintStats(Stats *Stats, FuzzJob *Job) {
208 Printf(Fmt: "#%zd: cov: %zd ft: %zd corp: %zd exec/s: %zd oom/timeout/crash: "
209 "%zd/%zd/%zd time: %zds job: %zd dft_time: %d stale: %zd\n",
210 NumRuns, Cov.size(), Features.size(), Files.size(),
211 Stats->average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes,
212 secondsSinceProcessStartUp(), Job->JobId, Job->DftTimeInSeconds,
213 secondsSinceLastNewCorpus());
214 }
215
216 void RunOneMergeJob(FuzzJob *Job) {
217 auto Stats = ParseFinalStatsFromLog(LogPath: Job->LogPath);
218 NumRuns += Stats.number_of_executed_units;
219
220 std::vector<SizedFile> TempFiles, MergeCandidates;
221 // Read all newly created inputs and their feature sets.
222 // Choose only those inputs that have new features.
223 GetSizedFilesFromDir(Dir: Job->CorpusDir, V: &TempFiles);
224 std::sort(first: TempFiles.begin(), last: TempFiles.end());
225 for (auto &F : TempFiles) {
226 auto FeatureFile = F.File;
227 FeatureFile.replace(pos1: 0, n1: Job->CorpusDir.size(), str: Job->FeaturesDir);
228 auto FeatureBytes = FileToVector(Path: FeatureFile, MaxSize: 0, ExitOnError: false);
229 assert((FeatureBytes.size() % sizeof(uint32_t)) == 0);
230 std::vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t));
231 memcpy(dest: NewFeatures.data(), src: FeatureBytes.data(), n: FeatureBytes.size());
232 for (auto Ft : NewFeatures) {
233 if (!Features.count(k: Ft)) {
234 MergeCandidates.push_back(x: F);
235 break;
236 }
237 }
238 }
239
240 if (MergeCandidates.empty()) {
241 PrintStats(Stats: &Stats, Job);
242 return;
243 }
244
245 std::vector<std::string> FilesToAdd;
246 std::set<uint32_t> NewFeatures, NewCov;
247 bool IsSetCoverMerge =
248 !Job->Cmd.getFlagValue(Flag: "set_cover_merge").compare(s: "1");
249 CrashResistantMerge(Args, OldCorpus: {}, NewCorpus: MergeCandidates, NewFiles: &FilesToAdd, InitialFeatures: Features,
250 NewFeatures: &NewFeatures, InitialCov: Cov, NewCov: &NewCov, CFPath: Job->CFPath, Verbose: false,
251 IsSetCoverMerge);
252 for (auto &Path : FilesToAdd) {
253 auto U = FileToVector(Path);
254 auto NewPath = DirPlusFile(DirPath: MainCorpusDir, FileName: Hash(U));
255 WriteToFile(U, Path: NewPath);
256 if (Group) { // Insert the queue according to the size of the seed.
257 size_t UnitSize = U.size();
258 auto Idx =
259 std::upper_bound(first: FilesSizes.begin(), last: FilesSizes.end(), value: UnitSize) -
260 FilesSizes.begin();
261 FilesSizes.insert(position: FilesSizes.begin() + Idx, x: UnitSize);
262 Files.insert(position: Files.begin() + Idx, x: NewPath);
263 } else {
264 Files.push_back(x: NewPath);
265 }
266 }
267 Features.insert(first: NewFeatures.begin(), last: NewFeatures.end());
268 Cov.insert(first: NewCov.begin(), last: NewCov.end());
269 for (auto Idx : NewCov)
270 if (auto *TE = TPC.PCTableEntryByIdx(Idx))
271 if (TPC.PcIsFuncEntry(TE))
272 PrintPC(SymbolizedFMT: " NEW_FUNC: %p %F %L\n", FallbackFMT: "",
273 PC: TPC.GetNextInstructionPc(PC: TE->PC));
274 if (!FilesToAdd.empty())
275 LastNewCorpusTime = std::chrono::system_clock::now();
276 PrintStats(Stats: &Stats, Job);
277 }
278
279 void CollectDFT(const std::string &InputPath) {
280 if (DataFlowBinary.empty()) return;
281 if (!FilesWithDFT.insert(v: InputPath).second) return;
282 Command Cmd(Args);
283 Cmd.removeFlag(Flag: "fork");
284 Cmd.removeFlag(Flag: "runs");
285 Cmd.addFlag(Flag: "data_flow_trace", Value: DFTDir);
286 Cmd.addArgument(Arg: InputPath);
287 for (auto &C : CorpusDirs) // Remove all corpora from the args.
288 Cmd.removeArgument(Arg: C);
289 Cmd.setOutputFile(DirPlusFile(DirPath: TempDir, FileName: "dft.log"));
290 Cmd.combineOutAndErr();
291 // Printf("CollectDFT: %s\n", Cmd.toString().c_str());
292 ExecuteCommand(Cmd);
293 }
294
295};
296
297struct JobQueue {
298 std::queue<FuzzJob *> Qu;
299 std::mutex Mu;
300 std::condition_variable Cv;
301
302 void Push(FuzzJob *Job) {
303 {
304 std::lock_guard<std::mutex> Lock(Mu);
305 Qu.push(v: Job);
306 }
307 Cv.notify_one();
308 }
309 FuzzJob *Pop() {
310 std::unique_lock<std::mutex> Lk(Mu);
311 // std::lock_guard<std::mutex> Lock(Mu);
312 Cv.wait(lk&: Lk, pred: [&]{return !Qu.empty();});
313 assert(!Qu.empty());
314 auto Job = Qu.front();
315 Qu.pop();
316 return Job;
317 }
318};
319
320void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) {
321 while (auto Job = FuzzQ->Pop()) {
322 // Printf("WorkerThread: job %p\n", Job);
323 Job->ExitCode = ExecuteCommand(Cmd: Job->Cmd);
324 MergeQ->Push(Job);
325 }
326}
327
328// This is just a skeleton of an experimental -fork=1 feature.
329void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
330 const std::vector<std::string> &Args,
331 const std::vector<std::string> &CorpusDirs, int NumJobs) {
332 Printf(Fmt: "INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs);
333
334 GlobalEnv Env;
335 Env.Args = Args;
336 Env.CorpusDirs = CorpusDirs;
337 Env.Rand = &Rand;
338 Env.Verbosity = Options.Verbosity;
339 Env.ProcessStartTime = std::chrono::system_clock::now();
340 Env.DataFlowBinary = Options.CollectDataFlow;
341 Env.Group = Options.ForkCorpusGroups;
342
343 std::vector<SizedFile> SeedFiles;
344 for (auto &Dir : CorpusDirs)
345 GetSizedFilesFromDir(Dir, V: &SeedFiles);
346 std::sort(first: SeedFiles.begin(), last: SeedFiles.end());
347 Env.TempDir = TempPath(Prefix: "FuzzWithFork", Extension: ".dir");
348 Env.DFTDir = DirPlusFile(DirPath: Env.TempDir, FileName: "DFT");
349 RmDirRecursive(Dir: Env.TempDir); // in case there is a leftover from old runs.
350 MkDir(Path: Env.TempDir);
351 MkDir(Path: Env.DFTDir);
352
353
354 if (CorpusDirs.empty())
355 MkDir(Path: Env.MainCorpusDir = DirPlusFile(DirPath: Env.TempDir, FileName: "C"));
356 else
357 Env.MainCorpusDir = CorpusDirs[0];
358
359 if (Options.KeepSeed) {
360 for (auto &File : SeedFiles)
361 Env.Files.push_back(x: File.File);
362 } else {
363 auto CFPath = DirPlusFile(DirPath: Env.TempDir, FileName: "merge.txt");
364 std::set<uint32_t> NewFeatures, NewCov;
365 CrashResistantMerge(Args: Env.Args, OldCorpus: {}, NewCorpus: SeedFiles, NewFiles: &Env.Files, InitialFeatures: Env.Features,
366 NewFeatures: &NewFeatures, InitialCov: Env.Cov, NewCov: &NewCov, CFPath,
367 /*Verbose=*/false, /*IsSetCoverMerge=*/false);
368 Env.Features.insert(first: NewFeatures.begin(), last: NewFeatures.end());
369 Env.Cov.insert(first: NewCov.begin(), last: NewCov.end());
370 RemoveFile(Path: CFPath);
371 }
372
373 if (Env.Group) {
374 for (auto &path : Env.Files)
375 Env.FilesSizes.push_back(x: FileSize(Path: path));
376 }
377
378 Printf(Fmt: "INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs,
379 Env.Files.size(), Env.TempDir.c_str());
380
381 int ExitCode = 0;
382
383 JobQueue FuzzQ, MergeQ;
384
385 auto StopJobs = [&]() {
386 for (int i = 0; i < NumJobs; i++)
387 FuzzQ.Push(Job: nullptr);
388 MergeQ.Push(Job: nullptr);
389 WriteToFile(U: Unit({1}), Path: Env.StopFile());
390 };
391
392 size_t MergeCycle = 20;
393 size_t JobExecuted = 0;
394 size_t JobId = 1;
395 std::vector<std::thread> Threads;
396 for (int t = 0; t < NumJobs; t++) {
397 Threads.push_back(x: std::thread(WorkerThread, &FuzzQ, &MergeQ));
398 FuzzQ.Push(Job: Env.CreateNewJob(JobId: JobId++));
399 }
400
401 Env.LastNewCorpusTime = std::chrono::system_clock::now();
402
403 while (true) {
404 std::unique_ptr<FuzzJob> Job(MergeQ.Pop());
405 if (!Job)
406 break;
407 ExitCode = Job->ExitCode;
408 if (ExitCode == Options.InterruptExitCode) {
409 Printf(Fmt: "==%lu== libFuzzer: a child was interrupted; exiting\n", GetPid());
410 StopJobs();
411 break;
412 }
413 Fuzzer::MaybeExitGracefully();
414
415 Env.RunOneMergeJob(Job: Job.get());
416
417 // merge the corpus .
418 JobExecuted++;
419 if (Env.Group && JobExecuted >= MergeCycle) {
420 std::vector<SizedFile> CurrentSeedFiles;
421 for (auto &Dir : CorpusDirs)
422 GetSizedFilesFromDir(Dir, V: &CurrentSeedFiles);
423 std::sort(first: CurrentSeedFiles.begin(), last: CurrentSeedFiles.end());
424
425 auto CFPath = DirPlusFile(DirPath: Env.TempDir, FileName: "merge.txt");
426 std::set<uint32_t> TmpNewFeatures, TmpNewCov;
427 std::set<uint32_t> TmpFeatures, TmpCov;
428 Env.Files.clear();
429 Env.FilesSizes.clear();
430 CrashResistantMerge(Args: Env.Args, OldCorpus: {}, NewCorpus: CurrentSeedFiles, NewFiles: &Env.Files,
431 InitialFeatures: TmpFeatures, NewFeatures: &TmpNewFeatures, InitialCov: TmpCov, NewCov: &TmpNewCov,
432 CFPath, /*Verbose=*/false, /*IsSetCoverMerge=*/false);
433 for (auto &path : Env.Files)
434 Env.FilesSizes.push_back(x: FileSize(Path: path));
435 RemoveFile(Path: CFPath);
436 JobExecuted = 0;
437 MergeCycle += 5;
438 }
439
440 // Since the number of corpus seeds will gradually increase, in order to
441 // control the number in each group to be about three times the number of
442 // seeds selected each time, the number of groups is dynamically adjusted.
443 if (Env.Files.size() < 2000)
444 Env.NumCorpuses = 12;
445 else if (Env.Files.size() < 6000)
446 Env.NumCorpuses = 20;
447 else if (Env.Files.size() < 12000)
448 Env.NumCorpuses = 32;
449 else if (Env.Files.size() < 16000)
450 Env.NumCorpuses = 40;
451 else if (Env.Files.size() < 24000)
452 Env.NumCorpuses = 60;
453 else
454 Env.NumCorpuses = 80;
455
456 // Continue if our crash is one of the ignored ones.
457 if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode)
458 Env.NumTimeouts++;
459 else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode)
460 Env.NumOOMs++;
461 else if (ExitCode != 0) {
462 Env.NumCrashes++;
463 if (Options.IgnoreCrashes) {
464 std::ifstream In(Job->LogPath);
465 std::string Line;
466 while (std::getline(is&: In, str&: Line, dlm: '\n'))
467 if (Line.find(s: "ERROR:") != Line.npos ||
468 Line.find(s: "runtime error:") != Line.npos)
469 Printf(Fmt: "%s\n", Line.c_str());
470 } else {
471 // And exit if we don't ignore this crash.
472 Printf(Fmt: "INFO: log from the inner process:\n%s",
473 FileToString(Path: Job->LogPath).c_str());
474 StopJobs();
475 break;
476 }
477 }
478
479 // Stop if we are over the time budget.
480 // This is not precise, since other threads are still running
481 // and we will wait while joining them.
482 // We also don't stop instantly: other jobs need to finish.
483 if (Options.MaxTotalTimeSec > 0 &&
484 Env.secondsSinceProcessStartUp() > (size_t)Options.MaxTotalTimeSec) {
485 Printf(Fmt: "INFO: fuzzed for %zd seconds, wrapping up soon\n",
486 Env.secondsSinceProcessStartUp());
487 StopJobs();
488 break;
489 }
490 if (Options.StaleCorpusTimeoutSec > 0 &&
491 Env.secondsSinceLastNewCorpus() >
492 (size_t)Options.StaleCorpusTimeoutSec) {
493 Printf(Fmt: "INFO: no new corpus for %zd seconds, wrapping up soon\n",
494 Env.secondsSinceLastNewCorpus());
495 StopJobs();
496 break;
497 }
498 if (Env.NumRuns >= Options.MaxNumberOfRuns) {
499 Printf(Fmt: "INFO: fuzzed for %zd iterations, wrapping up soon\n",
500 Env.NumRuns);
501 StopJobs();
502 break;
503 }
504
505 FuzzQ.Push(Job: Env.CreateNewJob(JobId: JobId++));
506 }
507
508 for (auto &T : Threads)
509 T.join();
510
511 // The workers have terminated. Don't try to remove the directory before they
512 // terminate to avoid a race condition preventing cleanup on Windows.
513 RmDirRecursive(Dir: Env.TempDir);
514
515 // Use the exit code from the last child process.
516 Printf(Fmt: "INFO: exiting: %d time: %zds\n", ExitCode,
517 Env.secondsSinceProcessStartUp());
518 exit(status: ExitCode);
519}
520
521} // namespace fuzzer
522