From 58d43607862096aeb32d72173911c9df244a30f1 Mon Sep 17 00:00:00 2001 From: Chandler Carruth Date: Sat, 19 Jan 2019 08:50:56 +0000 Subject: Update the file headers across all of the LLVM projects in the monorepo to reflect the new license. We understand that people may be surprised that we're moving the header entirely to discuss the new license. We checked this carefully with the Foundation's lawyer and we believe this is the correct approach. Essentially, all code in the project is now made available by the LLVM project under our new license, so you will see that the license headers include that license only. Some of our contributors have contributed code under our old license, and accordingly, we have retained a copy of our old license notice in the top-level files in each project and repository. git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@351636 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 5f3052a39..77b8a789a 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -1,9 +1,8 @@ //===- FuzzerMerge.cpp - merging corpora ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Merging corpora. -- cgit v1.2.1 From 4bdfbe8883e81de78f806e2539c0718af49c02c7 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Fri, 8 Feb 2019 21:27:23 +0000 Subject: [libFuzzer] introduce an experimental mode -fork=1, where fuzzing happens in a subprocess (still running multiple inputs per process), thus making the fuzzing more resilient to timeouts and OOMs. This is just a skeleton of the code, and some associated refactoring, not a fully working feature yet. git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@353570 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 42 +++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 77b8a789a..b4f05e3b8 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -122,7 +122,7 @@ size_t Merger::ApproximateMemoryConsumption() const { // Decides which files need to be merged (add thost to NewFiles). // Returns the number of new features added. -size_t Merger::Merge(const Set &InitialFeatures, +size_t Merger::Merge(const Set &InitialFeatures, Vector *NewFiles) { NewFiles->clear(); assert(NumFilesInFirstCorpus <= Files.size()); @@ -223,7 +223,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app); Set AllFeatures; for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { - MaybeExitGracefully(); + Fuzzer::MaybeExitGracefully(); auto U = FileToVector(M.Files[i].Name); if (U.size() > MaxInputLen) { U.resize(MaxInputLen); @@ -275,27 +275,18 @@ static void WriteNewControlFile(const std::string &CFPath, } // Outer process. Does not call the target code and thus sohuld not fail. -void Fuzzer::CrashResistantMerge(const Vector &Args, - const Vector &Corpora, - const char *CoverageSummaryInputPathOrNull, - const char *CoverageSummaryOutputPathOrNull, - const char *MergeControlFilePathOrNull) { - if (Corpora.size() <= 1) { - Printf("Merge requires two or more corpus dirs\n"); - return; - } - auto CFPath = - MergeControlFilePathOrNull - ? MergeControlFilePathOrNull - : DirPlusFile(TmpDir(), - "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); - +Vector +CrashResistantMerge(const Vector &Args, + const Vector &Corpora, + const std::string &CFPath, + const char *CoverageSummaryInputPathOrNull, + const char *CoverageSummaryOutputPathOrNull) { size_t NumAttempts = 0; - if (MergeControlFilePathOrNull && FileSize(MergeControlFilePathOrNull)) { + if (FileSize(CFPath)) { Printf("MERGE-OUTER: non-empty control file provided: '%s'\n", - MergeControlFilePathOrNull); + CFPath.c_str()); Merger M; - std::ifstream IF(MergeControlFilePathOrNull); + std::ifstream IF(CFPath); if (M.Parse(IF, /*ParseCoverage=*/false)) { Printf("MERGE-OUTER: control file ok, %zd files total," " first not processed file %zd\n", @@ -334,9 +325,10 @@ void Fuzzer::CrashResistantMerge(const Vector &Args, // Every inner process should execute at least one input. Command BaseCmd(Args); BaseCmd.removeFlag("merge"); + BaseCmd.removeFlag("fork"); bool Success = false; for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { - MaybeExitGracefully(); + Fuzzer::MaybeExitGracefully(); Printf("MERGE-OUTER: attempt %zd\n", Attempt); Command Cmd(BaseCmd); Cmd.addFlag("merge_control_file", CFPath); @@ -368,7 +360,6 @@ void Fuzzer::CrashResistantMerge(const Vector &Args, std::ofstream SummaryOut(CoverageSummaryOutputPathOrNull); M.PrintSummary(SummaryOut); } - Vector NewFiles; Set InitialFeatures; if (CoverageSummaryInputPathOrNull) { std::ifstream SummaryIn(CoverageSummaryInputPathOrNull); @@ -376,14 +367,11 @@ void Fuzzer::CrashResistantMerge(const Vector &Args, Printf("MERGE-OUTER: coverage summary loaded from %s, %zd features found\n", CoverageSummaryInputPathOrNull, InitialFeatures.size()); } + Vector NewFiles; size_t NumNewFeatures = M.Merge(InitialFeatures, &NewFiles); Printf("MERGE-OUTER: %zd new files with %zd new features added\n", NewFiles.size(), NumNewFeatures); - for (auto &F: NewFiles) - WriteToOutputCorpus(FileToVector(F, MaxInputLen)); - // We are done, delete the control file if it was a temporary one. - if (!MergeControlFilePathOrNull) - RemoveFile(CFPath); + return NewFiles; } } // namespace fuzzer -- cgit v1.2.1 From 4c775333c188cc37182c72eb2a7261f7eaeb32a5 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Fri, 8 Feb 2019 22:02:37 +0000 Subject: [libFuzzer] remove two unused experimental flags git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@353573 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 45 +-------------------------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index b4f05e3b8..0d4971af0 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -168,16 +168,6 @@ size_t Merger::Merge(const Set &InitialFeatures, return AllFeatures.size() - InitialNumFeatures; } -void Merger::PrintSummary(std::ostream &OS) { - for (auto &File : Files) { - OS << std::hex; - OS << File.Name << " size: " << File.Size << " features: "; - for (auto Feature : File.Features) - OS << " " << Feature; - OS << "\n"; - } -} - Set Merger::AllFeatures() const { Set S; for (auto &File : Files) @@ -185,25 +175,6 @@ Set Merger::AllFeatures() const { return S; } -Set Merger::ParseSummary(std::istream &IS) { - std::string Line, Tmp; - Set Res; - while (std::getline(IS, Line, '\n')) { - size_t N; - std::istringstream ISS1(Line); - ISS1 >> Tmp; // Name - ISS1 >> Tmp; // size: - assert(Tmp == "size:" && "Corrupt summary file"); - ISS1 >> std::hex; - ISS1 >> N; // File Size - ISS1 >> Tmp; // features: - assert(Tmp == "features:" && "Corrupt summary file"); - while (ISS1 >> std::hex >> N) - Res.insert(N); - } - return Res; -} - // Inner process. May crash if the target crashes. void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str()); @@ -278,9 +249,7 @@ static void WriteNewControlFile(const std::string &CFPath, Vector CrashResistantMerge(const Vector &Args, const Vector &Corpora, - const std::string &CFPath, - const char *CoverageSummaryInputPathOrNull, - const char *CoverageSummaryOutputPathOrNull) { + const std::string &CFPath) { size_t NumAttempts = 0; if (FileSize(CFPath)) { Printf("MERGE-OUTER: non-empty control file provided: '%s'\n", @@ -354,19 +323,7 @@ CrashResistantMerge(const Vector &Args, IF.close(); Printf("MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); - if (CoverageSummaryOutputPathOrNull) { - Printf("MERGE-OUTER: writing coverage summary for %zd files to %s\n", - M.Files.size(), CoverageSummaryOutputPathOrNull); - std::ofstream SummaryOut(CoverageSummaryOutputPathOrNull); - M.PrintSummary(SummaryOut); - } Set InitialFeatures; - if (CoverageSummaryInputPathOrNull) { - std::ifstream SummaryIn(CoverageSummaryInputPathOrNull); - InitialFeatures = M.ParseSummary(SummaryIn); - Printf("MERGE-OUTER: coverage summary loaded from %s, %zd features found\n", - CoverageSummaryInputPathOrNull, InitialFeatures.size()); - } Vector NewFiles; size_t NumNewFeatures = M.Merge(InitialFeatures, &NewFiles); Printf("MERGE-OUTER: %zd new files with %zd new features added\n", -- cgit v1.2.1 From 170fdb7696b9c114106a40c7c3a9b523e6341db0 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Fri, 8 Feb 2019 22:59:03 +0000 Subject: [libFuzzer] refactor the merging code, NFC git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@353576 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 0d4971af0..5c590269b 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -230,13 +230,15 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { } static void WriteNewControlFile(const std::string &CFPath, - const Vector &AllFiles, - size_t NumFilesInFirstCorpus) { + const Vector &OldCorpus, + const Vector &NewCorpus) { RemoveFile(CFPath); std::ofstream ControlFile(CFPath); - ControlFile << AllFiles.size() << "\n"; - ControlFile << NumFilesInFirstCorpus << "\n"; - for (auto &SF: AllFiles) + ControlFile << (OldCorpus.size() + NewCorpus.size()) << "\n"; + ControlFile << OldCorpus.size() << "\n"; + for (auto &SF: OldCorpus) + ControlFile << SF.File << "\n"; + for (auto &SF: NewCorpus) ControlFile << SF.File << "\n"; if (!ControlFile) { Printf("MERGE-OUTER: failed to write to the control file: %s\n", @@ -245,10 +247,11 @@ static void WriteNewControlFile(const std::string &CFPath, } } -// Outer process. Does not call the target code and thus sohuld not fail. +// Outer process. Does not call the target code and thus should not fail. Vector CrashResistantMerge(const Vector &Args, - const Vector &Corpora, + const Vector &OldCorpus, + const Vector &NewCorpus, const std::string &CFPath) { size_t NumAttempts = 0; if (FileSize(CFPath)) { @@ -277,17 +280,10 @@ CrashResistantMerge(const Vector &Args, if (!NumAttempts) { // The supplied control file is empty or bad, create a fresh one. - Vector AllFiles; - GetSizedFilesFromDir(Corpora[0], &AllFiles); - size_t NumFilesInFirstCorpus = AllFiles.size(); - std::sort(AllFiles.begin(), AllFiles.end()); - for (size_t i = 1; i < Corpora.size(); i++) - GetSizedFilesFromDir(Corpora[i], &AllFiles); - std::sort(AllFiles.begin() + NumFilesInFirstCorpus, AllFiles.end()); - Printf("MERGE-OUTER: %zd files, %zd in the initial corpus\n", - AllFiles.size(), NumFilesInFirstCorpus); - WriteNewControlFile(CFPath, AllFiles, NumFilesInFirstCorpus); - NumAttempts = AllFiles.size(); + NumAttempts = OldCorpus.size() + NewCorpus.size(); + Printf("MERGE-OUTER: %zd files, %zd in the initial corpus\n", NumAttempts, + OldCorpus.size()); + WriteNewControlFile(CFPath, OldCorpus, NewCorpus); } // Execute the inner process until it passes. -- cgit v1.2.1 From 00ab2236146c035d7cabab47c76d13596a4ea315 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Sat, 9 Feb 2019 00:16:21 +0000 Subject: [libFuzzer] more refactoring; change some of the exit codes (timeout, OOM, interrupt) so that the parent process can distinguish those git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@353584 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 5c590269b..9760ac243 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -122,25 +122,26 @@ size_t Merger::ApproximateMemoryConsumption() const { // Decides which files need to be merged (add thost to NewFiles). // Returns the number of new features added. -size_t Merger::Merge(const Set &InitialFeatures, +size_t Merger::Merge(const Set &InitialFeatures, + Set *AllFeatures, Vector *NewFiles) { NewFiles->clear(); assert(NumFilesInFirstCorpus <= Files.size()); - Set AllFeatures(InitialFeatures); + *AllFeatures = InitialFeatures; // What features are in the initial corpus? for (size_t i = 0; i < NumFilesInFirstCorpus; i++) { auto &Cur = Files[i].Features; - AllFeatures.insert(Cur.begin(), Cur.end()); + AllFeatures->insert(Cur.begin(), Cur.end()); } - size_t InitialNumFeatures = AllFeatures.size(); + size_t InitialNumFeatures = AllFeatures->size(); // Remove all features that we already know from all other inputs. for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { auto &Cur = Files[i].Features; Vector Tmp; - std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(), - AllFeatures.end(), std::inserter(Tmp, Tmp.begin())); + std::set_difference(Cur.begin(), Cur.end(), AllFeatures->begin(), + AllFeatures->end(), std::inserter(Tmp, Tmp.begin())); Cur.swap(Tmp); } @@ -160,12 +161,12 @@ size_t Merger::Merge(const Set &InitialFeatures, auto &Cur = Files[i].Features; // Printf("%s -> sz %zd ft %zd\n", Files[i].Name.c_str(), // Files[i].Size, Cur.size()); - size_t OldSize = AllFeatures.size(); - AllFeatures.insert(Cur.begin(), Cur.end()); - if (AllFeatures.size() > OldSize) + size_t OldSize = AllFeatures->size(); + AllFeatures->insert(Cur.begin(), Cur.end()); + if (AllFeatures->size() > OldSize) NewFiles->push_back(Files[i].Name); } - return AllFeatures.size() - InitialNumFeatures; + return AllFeatures->size() - InitialNumFeatures; } Set Merger::AllFeatures() const { @@ -248,10 +249,12 @@ static void WriteNewControlFile(const std::string &CFPath, } // Outer process. Does not call the target code and thus should not fail. -Vector -CrashResistantMerge(const Vector &Args, +void CrashResistantMerge(const Vector &Args, const Vector &OldCorpus, const Vector &NewCorpus, + Vector *NewFiles, + const Set &InitialFeatures, + Set *NewFeatures, const std::string &CFPath) { size_t NumAttempts = 0; if (FileSize(CFPath)) { @@ -319,12 +322,9 @@ CrashResistantMerge(const Vector &Args, IF.close(); Printf("MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); - Set InitialFeatures; - Vector NewFiles; - size_t NumNewFeatures = M.Merge(InitialFeatures, &NewFiles); + size_t NumNewFeatures = M.Merge(InitialFeatures, NewFeatures, NewFiles); Printf("MERGE-OUTER: %zd new files with %zd new features added\n", - NewFiles.size(), NumNewFeatures); - return NewFiles; + NewFiles->size(), NumNewFeatures); } } // namespace fuzzer -- cgit v1.2.1 From c8bc1b3d9e088ca7fe256c9ee6a609f385b884b6 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Tue, 12 Feb 2019 00:12:33 +0000 Subject: [libFuzzer] extend the -fork=1 functionality. Still not fully usable, but good enough for the first unit test git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@353775 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 9760ac243..4d00f7ed9 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -120,28 +120,28 @@ size_t Merger::ApproximateMemoryConsumption() const { return Res; } -// Decides which files need to be merged (add thost to NewFiles). +// Decides which files need to be merged (add those to NewFiles). // Returns the number of new features added. size_t Merger::Merge(const Set &InitialFeatures, - Set *AllFeatures, + Set *NewFeatures, Vector *NewFiles) { NewFiles->clear(); assert(NumFilesInFirstCorpus <= Files.size()); - *AllFeatures = InitialFeatures; + Set AllFeatures = InitialFeatures; // What features are in the initial corpus? for (size_t i = 0; i < NumFilesInFirstCorpus; i++) { auto &Cur = Files[i].Features; - AllFeatures->insert(Cur.begin(), Cur.end()); + AllFeatures.insert(Cur.begin(), Cur.end()); } - size_t InitialNumFeatures = AllFeatures->size(); + size_t InitialNumFeatures = AllFeatures.size(); // Remove all features that we already know from all other inputs. for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { auto &Cur = Files[i].Features; Vector Tmp; - std::set_difference(Cur.begin(), Cur.end(), AllFeatures->begin(), - AllFeatures->end(), std::inserter(Tmp, Tmp.begin())); + std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(), + AllFeatures.end(), std::inserter(Tmp, Tmp.begin())); Cur.swap(Tmp); } @@ -161,12 +161,17 @@ size_t Merger::Merge(const Set &InitialFeatures, auto &Cur = Files[i].Features; // Printf("%s -> sz %zd ft %zd\n", Files[i].Name.c_str(), // Files[i].Size, Cur.size()); - size_t OldSize = AllFeatures->size(); - AllFeatures->insert(Cur.begin(), Cur.end()); - if (AllFeatures->size() > OldSize) + bool FoundNewFeatures = false; + for (auto Fe: Cur) { + if (AllFeatures.insert(Fe).second) { + FoundNewFeatures = true; + NewFeatures->insert(Fe); + } + } + if (FoundNewFeatures) NewFiles->push_back(Files[i].Name); } - return AllFeatures->size() - InitialNumFeatures; + return AllFeatures.size() - InitialNumFeatures; } Set Merger::AllFeatures() const { -- cgit v1.2.1 From 57bf2400336b66c1c1fbc112228907474a451fc2 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Tue, 12 Feb 2019 02:18:53 +0000 Subject: [libFuzzer] teach the fork mode to ignore OOMs and timeouts git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@353792 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 6 ------ 1 file changed, 6 deletions(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 4d00f7ed9..c61169a3a 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -299,7 +299,6 @@ void CrashResistantMerge(const Vector &Args, Command BaseCmd(Args); BaseCmd.removeFlag("merge"); BaseCmd.removeFlag("fork"); - bool Success = false; for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { Fuzzer::MaybeExitGracefully(); Printf("MERGE-OUTER: attempt %zd\n", Attempt); @@ -309,14 +308,9 @@ void CrashResistantMerge(const Vector &Args, auto ExitCode = ExecuteCommand(Cmd); if (!ExitCode) { Printf("MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); - Success = true; break; } } - if (!Success) { - Printf("MERGE-OUTER: zero succesfull attempts, exiting\n"); - exit(1); - } // Read the control file and do the merge. Merger M; std::ifstream IF(CFPath); -- cgit v1.2.1 From a098df1569900e6c3498ce9e9002db7374a284e6 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Tue, 12 Feb 2019 03:12:40 +0000 Subject: [libFuzzer] make the fork mode less verbose git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@353794 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index c61169a3a..9a86512df 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -255,42 +255,43 @@ static void WriteNewControlFile(const std::string &CFPath, // Outer process. Does not call the target code and thus should not fail. void CrashResistantMerge(const Vector &Args, - const Vector &OldCorpus, - const Vector &NewCorpus, - Vector *NewFiles, - const Set &InitialFeatures, - Set *NewFeatures, - const std::string &CFPath) { + const Vector &OldCorpus, + const Vector &NewCorpus, + Vector *NewFiles, + const Set &InitialFeatures, + Set *NewFeatures, const std::string &CFPath, + bool V /*Verbose*/) { size_t NumAttempts = 0; if (FileSize(CFPath)) { - Printf("MERGE-OUTER: non-empty control file provided: '%s'\n", + VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n", CFPath.c_str()); Merger M; std::ifstream IF(CFPath); if (M.Parse(IF, /*ParseCoverage=*/false)) { - Printf("MERGE-OUTER: control file ok, %zd files total," + VPrintf(V, "MERGE-OUTER: control file ok, %zd files total," " first not processed file %zd\n", M.Files.size(), M.FirstNotProcessedFile); if (!M.LastFailure.empty()) - Printf("MERGE-OUTER: '%s' will be skipped as unlucky " + VPrintf(V, "MERGE-OUTER: '%s' will be skipped as unlucky " "(merge has stumbled on it the last time)\n", M.LastFailure.c_str()); if (M.FirstNotProcessedFile >= M.Files.size()) { - Printf("MERGE-OUTER: nothing to do, merge has been completed before\n"); + VPrintf( + V, "MERGE-OUTER: nothing to do, merge has been completed before\n"); exit(0); } NumAttempts = M.Files.size() - M.FirstNotProcessedFile; } else { - Printf("MERGE-OUTER: bad control file, will overwrite it\n"); + VPrintf(V, "MERGE-OUTER: bad control file, will overwrite it\n"); } } if (!NumAttempts) { // The supplied control file is empty or bad, create a fresh one. NumAttempts = OldCorpus.size() + NewCorpus.size(); - Printf("MERGE-OUTER: %zd files, %zd in the initial corpus\n", NumAttempts, - OldCorpus.size()); + VPrintf(V, "MERGE-OUTER: %zd files, %zd in the initial corpus\n", + NumAttempts, OldCorpus.size()); WriteNewControlFile(CFPath, OldCorpus, NewCorpus); } @@ -301,13 +302,17 @@ void CrashResistantMerge(const Vector &Args, BaseCmd.removeFlag("fork"); for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { Fuzzer::MaybeExitGracefully(); - Printf("MERGE-OUTER: attempt %zd\n", Attempt); + VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt); Command Cmd(BaseCmd); Cmd.addFlag("merge_control_file", CFPath); Cmd.addFlag("merge_inner", "1"); + if (!V) { + Cmd.setOutputFile("/dev/null"); // TODO: need to handle this on Windows? + Cmd.combineOutAndErr(); + } auto ExitCode = ExecuteCommand(Cmd); if (!ExitCode) { - Printf("MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); + VPrintf(V, "MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); break; } } @@ -315,14 +320,16 @@ void CrashResistantMerge(const Vector &Args, Merger M; std::ifstream IF(CFPath); IF.seekg(0, IF.end); - Printf("MERGE-OUTER: the control file has %zd bytes\n", (size_t)IF.tellg()); + VPrintf(V, "MERGE-OUTER: the control file has %zd bytes\n", + (size_t)IF.tellg()); IF.seekg(0, IF.beg); M.ParseOrExit(IF, true); IF.close(); - Printf("MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", - M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); + VPrintf(V, + "MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", + M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); size_t NumNewFeatures = M.Merge(InitialFeatures, NewFeatures, NewFiles); - Printf("MERGE-OUTER: %zd new files with %zd new features added\n", + VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added\n", NewFiles->size(), NumNewFeatures); } -- cgit v1.2.1 From 92f7768ce940f6437b32ecc0985a1446cd040f7a Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Wed, 13 Feb 2019 04:04:45 +0000 Subject: [libFuzzer] a bit of refactoring of the fork mode git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@353910 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 9a86512df..556a231f1 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -261,6 +261,7 @@ void CrashResistantMerge(const Vector &Args, const Set &InitialFeatures, Set *NewFeatures, const std::string &CFPath, bool V /*Verbose*/) { + if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge. size_t NumAttempts = 0; if (FileSize(CFPath)) { VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n", -- cgit v1.2.1 From a35757418b2f100086f7c4ca811719af4d0b07b1 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Thu, 14 Feb 2019 23:12:33 +0000 Subject: [libFuzzer] when doing the merge, keep track of the coveraged edges, not just features git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@354076 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 556a231f1..55e5c9ad3 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -42,10 +42,12 @@ void Merger::ParseOrExit(std::istream &IS, bool ParseCoverage) { // file1 // file2 # One file name per line. // STARTED 0 123 # FileID, file size -// DONE 0 1 4 6 8 # FileID COV1 COV2 ... -// STARTED 1 456 # If DONE is missing, the input crashed while processing. +// FT 0 1 4 6 8 # FileID COV1 COV2 ... +// COV 0 7 8 9 # FileID COV1 COV1 +// STARTED 1 456 # If FT is missing, the input crashed while processing. // STARTED 2 567 -// DONE 2 8 9 +// FT 2 8 9 +// COV 2 11 12 bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastFailure.clear(); std::string Line; @@ -70,11 +72,12 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { if (!std::getline(IS, Files[i].Name, '\n')) return false; - // Parse STARTED and DONE lines. + // Parse STARTED, FT, and COV lines. size_t ExpectedStartMarker = 0; const size_t kInvalidStartMarker = -1; size_t LastSeenStartMarker = kInvalidStartMarker; Vector TmpFeatures; + Set PCs; while (std::getline(IS, Line, '\n')) { std::istringstream ISS1(Line); std::string Marker; @@ -89,8 +92,8 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastSeenStartMarker = ExpectedStartMarker; assert(ExpectedStartMarker < Files.size()); ExpectedStartMarker++; - } else if (Marker == "DONE") { - // DONE FILE_ID COV1 COV2 COV3 ... + } else if (Marker == "FT") { + // FT FILE_ID COV1 COV2 COV3 ... size_t CurrentFileIdx = N; if (CurrentFileIdx != LastSeenStartMarker) return false; @@ -102,6 +105,11 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { std::sort(TmpFeatures.begin(), TmpFeatures.end()); Files[CurrentFileIdx].Features = TmpFeatures; } + } else if (Marker == "COV") { + if (ParseCoverage) + while (ISS1 >> std::hex >> N) + if (PCs.insert(N).second) + NumCoveredPCs++; } else { return false; } @@ -199,6 +207,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app); Set AllFeatures; + Set AllPCs; for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { Fuzzer::MaybeExitGracefully(); auto U = FileToVector(M.Files[i].Name); @@ -223,16 +232,24 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { if (AllFeatures.insert(Feature).second) UniqFeatures.insert(Feature); }); + TPC.UpdateObservedPCs(); // Show stats. if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))) PrintStats("pulse "); // Write the post-run marker and the coverage. - OF << "DONE " << i; + OF << "FT " << i; for (size_t F : UniqFeatures) OF << " " << std::hex << F; OF << "\n"; + OF << "COV " << i; + TPC.ForEachObservedPC([&](const TracePC::PCTableEntry *TE) { + if (AllPCs.insert(TE).second) + OF << " " << TPC.PCTableEntryIdx(TE); + }); + OF << "\n"; OF.flush(); } + PrintStats("DONE "); } static void WriteNewControlFile(const std::string &CFPath, -- cgit v1.2.1 From 8d70b932e3dbd7218d31d28c2c5ed277018de7e9 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Fri, 15 Feb 2019 00:08:16 +0000 Subject: [libFuzzer] when doing the merge, keep track of the coveraged edges, not just features git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@354087 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 55e5c9ad3..549d180d4 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -77,7 +77,7 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { const size_t kInvalidStartMarker = -1; size_t LastSeenStartMarker = kInvalidStartMarker; Vector TmpFeatures; - Set PCs; + Set PCs; while (std::getline(IS, Line, '\n')) { std::istringstream ISS1(Line); std::string Marker; @@ -106,10 +106,11 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { Files[CurrentFileIdx].Features = TmpFeatures; } } else if (Marker == "COV") { + size_t CurrentFileIdx = N; if (ParseCoverage) while (ISS1 >> std::hex >> N) if (PCs.insert(N).second) - NumCoveredPCs++; + Files[CurrentFileIdx].Cov.push_back(N); } else { return false; } @@ -130,9 +131,9 @@ size_t Merger::ApproximateMemoryConsumption() const { // Decides which files need to be merged (add those to NewFiles). // Returns the number of new features added. -size_t Merger::Merge(const Set &InitialFeatures, - Set *NewFeatures, - Vector *NewFiles) { +void Merger::Merge(const Set &InitialFeatures, + Set *NewFeatures, const Set &InitialCov, + Set *NewCov, Vector *NewFiles) { NewFiles->clear(); assert(NumFilesInFirstCorpus <= Files.size()); Set AllFeatures = InitialFeatures; @@ -142,8 +143,6 @@ size_t Merger::Merge(const Set &InitialFeatures, auto &Cur = Files[i].Features; AllFeatures.insert(Cur.begin(), Cur.end()); } - size_t InitialNumFeatures = AllFeatures.size(); - // Remove all features that we already know from all other inputs. for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { auto &Cur = Files[i].Features; @@ -178,8 +177,10 @@ size_t Merger::Merge(const Set &InitialFeatures, } if (FoundNewFeatures) NewFiles->push_back(Files[i].Name); + for (auto Cov : Files[i].Cov) + if (InitialCov.find(Cov) == InitialCov.end()) + NewCov->insert(Cov); } - return AllFeatures.size() - InitialNumFeatures; } Set Merger::AllFeatures() const { @@ -241,7 +242,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { for (size_t F : UniqFeatures) OF << " " << std::hex << F; OF << "\n"; - OF << "COV " << i; + OF << "COV " << std::dec << i; TPC.ForEachObservedPC([&](const TracePC::PCTableEntry *TE) { if (AllPCs.insert(TE).second) OF << " " << TPC.PCTableEntryIdx(TE); @@ -276,7 +277,10 @@ void CrashResistantMerge(const Vector &Args, const Vector &NewCorpus, Vector *NewFiles, const Set &InitialFeatures, - Set *NewFeatures, const std::string &CFPath, + Set *NewFeatures, + const Set &InitialCov, + Set *NewCov, + const std::string &CFPath, bool V /*Verbose*/) { if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge. size_t NumAttempts = 0; @@ -346,9 +350,10 @@ void CrashResistantMerge(const Vector &Args, VPrintf(V, "MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); - size_t NumNewFeatures = M.Merge(InitialFeatures, NewFeatures, NewFiles); - VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added\n", - NewFiles->size(), NumNewFeatures); + M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); + VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; " + "%zd new coverage edges\n", + NewFiles->size(), NewFeatures->size(), NewCov->size()); } } // namespace fuzzer -- cgit v1.2.1 From 459210eed41fa9d37af2cb490161f73f64f4e632 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Fri, 15 Feb 2019 00:15:13 +0000 Subject: [libFuzzer] fix the unit tests git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@354088 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 549d180d4..870a87522 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -131,9 +131,10 @@ size_t Merger::ApproximateMemoryConsumption() const { // Decides which files need to be merged (add those to NewFiles). // Returns the number of new features added. -void Merger::Merge(const Set &InitialFeatures, - Set *NewFeatures, const Set &InitialCov, - Set *NewCov, Vector *NewFiles) { +size_t Merger::Merge(const Set &InitialFeatures, + Set *NewFeatures, + const Set &InitialCov, Set *NewCov, + Vector *NewFiles) { NewFiles->clear(); assert(NumFilesInFirstCorpus <= Files.size()); Set AllFeatures = InitialFeatures; @@ -181,6 +182,7 @@ void Merger::Merge(const Set &InitialFeatures, if (InitialCov.find(Cov) == InitialCov.end()) NewCov->insert(Cov); } + return NewFeatures->size(); } Set Merger::AllFeatures() const { -- cgit v1.2.1 From 3bca84ff3f580ed8f9724218fc9bff83c442f7a1 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Fri, 15 Feb 2019 01:22:00 +0000 Subject: [libFuzzer] print new functions as they are discovered in the fork mode git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@354092 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index 870a87522..d70aa3e90 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -100,7 +100,7 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastSeenStartMarker = kInvalidStartMarker; if (ParseCoverage) { TmpFeatures.clear(); // use a vector from outer scope to avoid resizes. - while (ISS1 >> std::hex >> N) + while (ISS1 >> N) TmpFeatures.push_back(N); std::sort(TmpFeatures.begin(), TmpFeatures.end()); Files[CurrentFileIdx].Features = TmpFeatures; @@ -108,7 +108,7 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { } else if (Marker == "COV") { size_t CurrentFileIdx = N; if (ParseCoverage) - while (ISS1 >> std::hex >> N) + while (ISS1 >> N) if (PCs.insert(N).second) Files[CurrentFileIdx].Cov.push_back(N); } else { @@ -220,7 +220,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { } std::ostringstream StartedLine; // Write the pre-run marker. - OF << "STARTED " << std::dec << i << " " << U.size() << "\n"; + OF << "STARTED " << i << " " << U.size() << "\n"; OF.flush(); // Flush is important since Command::Execute may crash. // Run. TPC.ResetMaps(); @@ -242,9 +242,9 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { // Write the post-run marker and the coverage. OF << "FT " << i; for (size_t F : UniqFeatures) - OF << " " << std::hex << F; + OF << " " << F; OF << "\n"; - OF << "COV " << std::dec << i; + OF << "COV " << i; TPC.ForEachObservedPC([&](const TracePC::PCTableEntry *TE) { if (AllPCs.insert(TE).second) OF << " " << TPC.PCTableEntryIdx(TE); -- cgit v1.2.1 From 8e4a50b7b72f4442fbe896778df6185218bd113a Mon Sep 17 00:00:00 2001 From: Jonathan Metzman Date: Wed, 27 Feb 2019 19:27:16 +0000 Subject: [libFuzzer][Windows] Port fork mode to Windows Summary: Port libFuzzer's fork mode to Windows. Implement Windows versions of MkDir, RmDir, and IterateDirRecursive to do this. Don't print error messages under new normal uses of FileSize (on a non-existent file). Implement portable way of piping output to /dev/null. Fix test for Windows and comment fork-sigusr.test on why it won't be ported to Win. Reviewers: zturner Reviewed By: zturner Subscribers: kcc, zturner, jdoerfert, #sanitizers, llvm-commits Tags: #sanitizers, #llvm Differential Revision: https://reviews.llvm.org/D58513 git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@355019 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/fuzzer/FuzzerMerge.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/fuzzer/FuzzerMerge.cpp') diff --git a/lib/fuzzer/FuzzerMerge.cpp b/lib/fuzzer/FuzzerMerge.cpp index d70aa3e90..dace45ece 100644 --- a/lib/fuzzer/FuzzerMerge.cpp +++ b/lib/fuzzer/FuzzerMerge.cpp @@ -331,7 +331,7 @@ void CrashResistantMerge(const Vector &Args, Cmd.addFlag("merge_control_file", CFPath); Cmd.addFlag("merge_inner", "1"); if (!V) { - Cmd.setOutputFile("/dev/null"); // TODO: need to handle this on Windows? + Cmd.setOutputFile(getDevNull()); Cmd.combineOutAndErr(); } auto ExitCode = ExecuteCommand(Cmd); -- cgit v1.2.1