From 7110dca575a8f1e6b5d83509cd878c0d46c957e3 Mon Sep 17 00:00:00 2001 From: Daniel Lowengrub Date: Mon, 6 Jul 2026 11:48:41 -0700 Subject: [PATCH] No public description PiperOrigin-RevId: 943413342 --- centipede/BUILD | 24 +- centipede/centipede_interface.cc | 7 +- centipede/crash_deduplication.cc | 661 +++++++++++++++------ centipede/crash_deduplication.h | 48 +- centipede/crash_deduplication_test.cc | 372 ++++++++---- centipede/crash_deduplication_test_util.cc | 42 ++ centipede/crash_deduplication_test_util.h | 55 ++ common/BUILD | 2 + common/CMakeLists.txt | 1 + common/remote_file.h | 29 +- common/remote_file_oss.cc | 14 + 11 files changed, 897 insertions(+), 358 deletions(-) create mode 100644 centipede/crash_deduplication_test_util.cc create mode 100644 centipede/crash_deduplication_test_util.h diff --git a/centipede/BUILD b/centipede/BUILD index 607811f8d..d0a8600c6 100644 --- a/centipede/BUILD +++ b/centipede/BUILD @@ -906,6 +906,7 @@ cc_library( "@abseil-cpp//absl/status:statusor", "@abseil-cpp//absl/strings", "@abseil-cpp//absl/time", + "@abseil-cpp//absl/time:clock_interface", "@com_google_fuzztest//common:crashing_input_filename", "@com_google_fuzztest//common:defs", "@com_google_fuzztest//common:hash", @@ -915,6 +916,22 @@ cc_library( ], ) +cc_library( + name = "crash_deduplication_test_util", + testonly = 1, + srcs = ["crash_deduplication_test_util.cc"], + hdrs = ["crash_deduplication_test_util.h"], + deps = [ + ":centipede_callbacks", + ":environment", + ":runner_result", + ":stop", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/types:span", + "@com_google_fuzztest//common:defs", + ], +) + cc_library( name = "crash_summary", srcs = ["crash_summary.cc"], @@ -1921,18 +1938,17 @@ cc_test( deps = [ ":centipede_callbacks", ":crash_deduplication", + ":crash_deduplication_test_util", ":crash_summary", ":environment", - ":runner_result", ":stop", ":util", ":workdir", "@abseil-cpp//absl/container:flat_hash_map", "@abseil-cpp//absl/strings:str_format", "@abseil-cpp//absl/time", - "@abseil-cpp//absl/types:span", - "@com_google_fuzztest//common:defs", - "@com_google_fuzztest//common:hash", + "@abseil-cpp//absl/time:clock_interface", + "@abseil-cpp//absl/time:simulated_clock", "@com_google_fuzztest//common:temp_dir", "@googletest//:gtest_main", ], diff --git a/centipede/centipede_interface.cc b/centipede/centipede_interface.cc index 99cace6b3..104ad050e 100644 --- a/centipede/centipede_interface.cc +++ b/centipede/centipede_interface.cc @@ -74,6 +74,8 @@ namespace fuzztest::internal { namespace { +constexpr absl::Duration kDefaultRegressionTtl = absl::Hours(24 * 7); + // Sets signal handler for SIGINT. // TODO(b/378532202): Replace this with a more generic mechanism that allows // the called or `CentipedeMain()` to indicate when to stop. @@ -557,9 +559,10 @@ int UpdateCorpusDatabase(Environment env, crash_summary.Report(&std::cerr); return exit_code; } - OrganizeCrashingInputs(regression_dir, fuzztest_db_path / "crashing", env, + OrganizeCrashingInputs(regression_dir, fuzztest_db_path / "crashing", + fuzztest_db_path / "incubating", env, callbacks_factory, crashes_by_signature, crash_summary, - stop_condition); + stop_condition, kDefaultRegressionTtl); if (env.report_crash_summary) crash_summary.Report(&std::cerr); // Distill and store the coverage corpus. diff --git a/centipede/crash_deduplication.cc b/centipede/crash_deduplication.cc index 268c13ce3..53174cd5b 100644 --- a/centipede/crash_deduplication.cc +++ b/centipede/crash_deduplication.cc @@ -17,6 +17,7 @@ #include #include #include // NOLINT +#include #include #include #include @@ -28,6 +29,7 @@ #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/time/clock.h" +#include "absl/time/clock_interface.h" #include "absl/time/time.h" #include "./centipede/centipede_callbacks.h" #include "./centipede/crash_summary.h" @@ -45,12 +47,477 @@ namespace fuzztest::internal { namespace { +constexpr size_t kMaxCrashInputCount = 10; + std::string GetInputFileName(std::string_view bug_id, std::string_view crash_signature, std::string_view input_signature) { return absl::StrCat(bug_id, "-", crash_signature, "-", input_signature); } +struct CrashReport { + CrashDetails details; + std::string signature; + std::string bug_id; +}; + +enum class ActionType { + kTouch, + kKeep, + // Replace the input but keep the bug and signature. + kReplaceInput, + // Incubate the input and then replace it. + kIncubateAndReplaceInput, + kDelete +}; + +struct ExistingCrash { + CrashReport crash_report; + std::string new_signature; +}; + +struct IncubatingCrash { + CrashDetails details; + std::string new_signature; +}; + +struct ExistingCrashAction { + ActionType action_type; + std::optional new_details; +}; + +absl::StatusOr> ReadExistingCrashes( + const std::filesystem::path& crashing_dir) { + std::vector existing_crashes; + std::vector input_files; + ASSIGN_OR_RETURN_IF_NOT_OK( + input_files, + RemoteListFiles(crashing_dir.c_str(), /*recursively=*/false)); + + for (const std::string& input_file : input_files) { + auto input_file_components = ParseCrashingInputFilename(input_file); + ExistingCrash existing; + existing.crash_report.details.input_path = input_file; + + if (input_file_components.ok()) { + existing.crash_report.bug_id = input_file_components->bug_id; + existing.crash_report.signature = input_file_components->crash_signature; + existing.crash_report.details.input_signature = + input_file_components->input_signature; + } else { + existing.crash_report.signature = ""; + existing.crash_report.bug_id = ""; + } + existing_crashes.push_back(std::move(existing)); + } + return existing_crashes; +} + +absl::StatusOr> ReadIncubatingCrashes( + const std::filesystem::path& incubating_dir) { + std::vector incubating_crashes; + if (!RemotePathExists(incubating_dir.string())) { + return incubating_crashes; + } + std::vector input_files; + ASSIGN_OR_RETURN_IF_NOT_OK( + input_files, + RemoteListFiles(incubating_dir.c_str(), /*recursively=*/false)); + + for (const std::string& input_file : input_files) { + IncubatingCrash incubating; + incubating.details.input_path = input_file; + incubating.details.input_signature = + std::filesystem::path(input_file).filename().string(); + incubating_crashes.push_back(std::move(incubating)); + } + return incubating_crashes; +} + +absl::Status ReplayCrash(CentipedeCallbacks& callbacks, const Environment& env, + const std::string& input_path, + std::string& out_signature, + std::string& out_description) { + ByteArray input_bytes; + RETURN_IF_NOT_OK(RemoteFileGetContents(input_path, input_bytes)); + + BatchResult batch_result; + const bool is_reproducible = + !callbacks.Execute(env.binary, {input_bytes}, batch_result) && + batch_result.IsInputFailure(); + + if (is_reproducible) { + out_signature = batch_result.failure_signature(); + out_description = batch_result.failure_description(); + } else { + out_signature = ""; + out_description = ""; + } + return absl::OkStatus(); +} + +absl::Status ReplayExistingCrashes( + CentipedeCallbacks& callbacks, const Environment& env, + std::vector& existing_crashes) { + for (auto& existing : existing_crashes) { + if (existing.crash_report.signature.empty()) { + continue; + } + RETURN_IF_NOT_OK(ReplayCrash( + callbacks, env, existing.crash_report.details.input_path, + existing.new_signature, existing.crash_report.details.description)); + } + return absl::OkStatus(); +} + +absl::Status ReplayIncubatingCrashes( + CentipedeCallbacks& callbacks, const Environment& env, + std::vector& incubating_crashes) { + for (auto& incubating : incubating_crashes) { + RETURN_IF_NOT_OK(ReplayCrash(callbacks, env, incubating.details.input_path, + incubating.new_signature, + incubating.details.description)); + } + return absl::OkStatus(); +} + +absl::flat_hash_map FindNewCrashes( + const absl::flat_hash_map& + new_crashes_by_signature, + const std::vector& incubating_crashes, + const std::vector& existing_crashes) { + absl::flat_hash_map new_crashes; + + // Add new crashes from the current run. + for (const auto& [signature, details] : new_crashes_by_signature) { + new_crashes[signature] = details; + } + + // Add reproducing incubating crashes. + for (const auto& incubating : incubating_crashes) { + if (!incubating.new_signature.empty()) { + new_crashes[incubating.new_signature] = incubating.details; + } + } + + // Add reproducing existing crashes (highest priority) + for (const auto& existing : existing_crashes) { + if (!existing.new_signature.empty()) { + new_crashes[existing.new_signature] = existing.crash_report.details; + } + } + + return new_crashes; +} + +ExistingCrashAction ComputeExistingCrashAction( + const ExistingCrash& existing, + const absl::flat_hash_map& new_crashes) { + const std::string& sig = existing.crash_report.signature; + const std::string& sig_new = existing.new_signature; + + // If the crash was malformed (no signature), we delete it immediately. + if (sig.empty()) { + return {ActionType::kDelete, std::nullopt}; + } + + if (sig == sig_new) { + return {ActionType::kTouch, std::nullopt}; + } + + // The signature changed or it no longer reproduces. + auto it = new_crashes.find(sig); + if (it == new_crashes.end()) { + // No input reproduces this signature anymore. Keep it on disk (subject to + // TTL). + return {ActionType::kKeep, std::nullopt}; + } + + // We have an input for this crash signature. + if (sig_new.empty()) { + if (it->second.input_signature == + existing.crash_report.details.input_signature) { + // The crashing input is the same which means that this input is flakey. + // Just touch it. + return {ActionType::kTouch, it->second}; + } + // The old crash did not reproduce at all. Move it to incubating and + // replace. + return {ActionType::kIncubateAndReplaceInput, it->second}; + } + + // The old crash reproduced with a different signature. Replace it. + return {ActionType::kReplaceInput, it->second}; +} + +std::vector ComputeExistingCrashActions( + const std::vector& existing_crashes, + const absl::flat_hash_map& new_crashes) { + std::vector actions; + actions.reserve(existing_crashes.size()); + for (const auto& existing : existing_crashes) { + actions.push_back(ComputeExistingCrashAction(existing, new_crashes)); + } + return actions; +} + +std::string GenerateBugId(std::string_view input_signature) { + return Hash(absl::StrCat(absl::FormatTime(absl::Now()), input_signature)); +} + +absl::Status WriteCrashToFile(const std::filesystem::path& crashing_dir, + std::string_view bug_id, + std::string_view crash_signature, + const CrashDetails& details, + CrashSummary& crash_summary) { + std::string new_input_file_name = + GetInputFileName(bug_id, crash_signature, details.input_signature); + std::filesystem::path new_input_path = crashing_dir / new_input_file_name; + + if (details.input_path != new_input_path.string()) { + RETURN_IF_NOT_OK( + RemoteFileCopy(details.input_path, new_input_path.string())); + } + + crash_summary.AddCrash({/*id=*/new_input_file_name, + /*category=*/details.description, + std::string(crash_signature), details.description}); + return absl::OkStatus(); +} + +absl::flat_hash_map GenerateNewCrashReports( + const absl::flat_hash_map& new_crashes, + const std::vector& existing_crashes) { + absl::flat_hash_set existing_sigs; + for (const auto& existing : existing_crashes) { + if (!existing.crash_report.signature.empty()) { + existing_sigs.insert(existing.crash_report.signature); + } + } + + absl::flat_hash_map new_crash_reports; + for (const auto& [signature, details] : new_crashes) { + if (existing_sigs.contains(signature)) { + continue; + } + std::string bug_id = GenerateBugId(details.input_signature); + new_crash_reports[signature] = CrashReport{details, signature, bug_id}; + } + return new_crash_reports; +} + +absl::Status TouchCrash(const ExistingCrash& existing, + const std::optional& new_details, + CrashSummary& crash_summary) { + RETURN_IF_NOT_OK( + RemotePathTouchExistingFile(existing.crash_report.details.input_path)); + + std::string description = existing.crash_report.details.description; + if (description.empty() && new_details.has_value()) { + description = new_details->description; + } + + crash_summary.AddCrash({ + /*id=*/std::filesystem::path(existing.crash_report.details.input_path) + .filename() + .string(), + /*category=*/description, + existing.crash_report.signature, + description, + }); + return absl::OkStatus(); +} + +absl::Status ExecuteExistingCrashPreservationActions( + const std::filesystem::path& crashing_dir, + const std::vector& existing_crashes, + const std::vector& actions, + CrashSummary& crash_summary) { + for (size_t i = 0; i < existing_crashes.size(); ++i) { + const auto& existing = existing_crashes[i]; + const auto& action = actions[i]; + + switch (action.action_type) { + case ActionType::kTouch: + RETURN_IF_NOT_OK( + TouchCrash(existing, action.new_details, crash_summary)); + break; + + case ActionType::kReplaceInput: + case ActionType::kIncubateAndReplaceInput: + RETURN_IF_NOT_OK(WriteCrashToFile(crashing_dir, + existing.crash_report.bug_id, + existing.crash_report.signature, + *action.new_details, crash_summary)); + break; + + case ActionType::kKeep: + case ActionType::kDelete: + break; + } + } + return absl::OkStatus(); +} + +absl::Status ExecuteExistingCrashDestructiveActions( + const std::filesystem::path& incubating_dir, + const std::vector& existing_crashes, + const std::vector& actions) { + for (size_t i = 0; i < existing_crashes.size(); ++i) { + const auto& existing = existing_crashes[i]; + const auto& action = actions[i]; + + switch (action.action_type) { + case ActionType::kDelete: + RETURN_IF_NOT_OK(RemotePathDelete( + existing.crash_report.details.input_path, /*recursively=*/false)); + break; + + case ActionType::kReplaceInput: + RETURN_IF_NOT_OK(RemotePathDelete( + existing.crash_report.details.input_path, /*recursively=*/false)); + break; + + case ActionType::kIncubateAndReplaceInput: { + std::filesystem::path dest_path = + incubating_dir / existing.crash_report.details.input_signature; + RETURN_IF_NOT_OK(RemoteFileRename( + existing.crash_report.details.input_path, dest_path.string())); + break; + } + + case ActionType::kKeep: + case ActionType::kTouch: + break; + } + } + return absl::OkStatus(); +} + +absl::Status WriteNewCrashes( + const std::filesystem::path& crashing_dir, + const absl::flat_hash_map& new_crash_reports, + size_t num_new_allowed, CrashSummary& crash_summary) { + size_t new_signatures_written = 0; + for (const auto& [signature, report] : new_crash_reports) { + if (new_signatures_written < num_new_allowed) { + RETURN_IF_NOT_OK(WriteCrashToFile(crashing_dir, report.bug_id, + report.signature, report.details, + crash_summary)); + ++new_signatures_written; + } else { + FUZZTEST_LOG(WARNING) + << "Reached the maximum number of crash inputs: " + << kMaxCrashInputCount + << ". Not storing new crash with signature: " << signature; + } + } + return absl::OkStatus(); +} + +absl::Status CleanUpIncubating( + const std::filesystem::path& incubating_dir, + const std::vector& incubating_crashes) { + for (const auto& incubating : incubating_crashes) { + if (!incubating.new_signature.empty()) { + RETURN_IF_NOT_OK(RemotePathDelete(incubating.details.input_path, + /*recursively=*/false)); + } + } + return absl::OkStatus(); +} + +absl::Status MoveExpiredCrashesToRegression( + const std::filesystem::path& source_dir, + const std::filesystem::path& regression_dir, absl::Duration ttl, + absl::Clock& clock) { + std::vector active_crash_files; + if (!RemotePathExists(source_dir.string())) { + return absl::OkStatus(); + } + ASSIGN_OR_RETURN_IF_NOT_OK( + active_crash_files, + RemoteListFiles(source_dir.c_str(), /*recursively=*/false)); + + absl::Time now = clock.TimeNow(); + + for (const std::string& file_path : active_crash_files) { + absl::Time mtime; + ASSIGN_OR_RETURN_IF_NOT_OK(mtime, RemoteFileGetMTime(file_path)); + + if (now - mtime > ttl) { + auto input_file_components = ParseCrashingInputFilename(file_path); + std::string dest_filename; + if (input_file_components.ok()) { + dest_filename = input_file_components->input_signature; + } else { + dest_filename = std::filesystem::path(file_path).filename().string(); + } + + std::filesystem::path dest_path = regression_dir / dest_filename; + RETURN_IF_NOT_OK(RemoteFileRename(file_path, dest_path.string())); + } + } + return absl::OkStatus(); +} + +absl::Status OrganizeCrashingInputsImpl( + const std::filesystem::path& regression_dir, + const std::filesystem::path& crashing_dir, + const std::filesystem::path& incubating_dir, const Environment& env, + CentipedeCallbacksFactory& callbacks_factory, + const absl::flat_hash_map& + new_crashes_by_signature, + CrashSummary& crash_summary, StopCondition& stop_condition, + absl::Duration regression_ttl, absl::Clock& clock) { + RETURN_IF_NOT_OK(RemoteMkdir(crashing_dir.c_str())); + RETURN_IF_NOT_OK(RemoteMkdir(regression_dir.c_str())); + RETURN_IF_NOT_OK(RemoteMkdir(incubating_dir.c_str())); + + ASSIGN_OR_RETURN_IF_NOT_OK(auto existing_crashes, + ReadExistingCrashes(crashing_dir)); + ASSIGN_OR_RETURN_IF_NOT_OK(auto incubating_crashes, + ReadIncubatingCrashes(incubating_dir)); + + ScopedCentipedeCallbacks scoped_callbacks(callbacks_factory, env, + stop_condition); + RETURN_IF_NOT_OK(ReplayExistingCrashes(*scoped_callbacks.callbacks(), env, + existing_crashes)); + RETURN_IF_NOT_OK(ReplayIncubatingCrashes(*scoped_callbacks.callbacks(), env, + incubating_crashes)); + + absl::flat_hash_map new_crashes = FindNewCrashes( + new_crashes_by_signature, incubating_crashes, existing_crashes); + + std::vector actions = + ComputeExistingCrashActions(existing_crashes, new_crashes); + + absl::flat_hash_map new_crash_reports = + GenerateNewCrashReports(new_crashes, existing_crashes); + + RETURN_IF_NOT_OK(ExecuteExistingCrashPreservationActions( + crashing_dir, existing_crashes, actions, crash_summary)); + + size_t num_new_allowed = kMaxCrashInputCount > existing_crashes.size() + ? kMaxCrashInputCount - existing_crashes.size() + : 0; + + RETURN_IF_NOT_OK(WriteNewCrashes(crashing_dir, new_crash_reports, + num_new_allowed, crash_summary)); + + RETURN_IF_NOT_OK(ExecuteExistingCrashDestructiveActions( + incubating_dir, existing_crashes, actions)); + + RETURN_IF_NOT_OK(CleanUpIncubating(incubating_dir, incubating_crashes)); + + RETURN_IF_NOT_OK(MoveExpiredCrashesToRegression(crashing_dir, regression_dir, + regression_ttl, clock)); + RETURN_IF_NOT_OK(MoveExpiredCrashesToRegression( + incubating_dir, regression_dir, regression_ttl, clock)); + + return absl::OkStatus(); +} + } // namespace absl::flat_hash_map GetCrashesFromWorkdir( @@ -129,193 +596,19 @@ absl::flat_hash_map GetCrashesFromWorkdir( void OrganizeCrashingInputs( const std::filesystem::path& regression_dir, - const std::filesystem::path& crashing_dir, const Environment& env, + const std::filesystem::path& crashing_dir, + const std::filesystem::path& incubating_dir, const Environment& env, CentipedeCallbacksFactory& callbacks_factory, const absl::flat_hash_map& new_crashes_by_signature, - CrashSummary& crash_summary, StopCondition& stop_condition) { - FUZZTEST_CHECK_OK(RemoteMkdir(crashing_dir.c_str())); - FUZZTEST_CHECK_OK(RemoteMkdir(regression_dir.c_str())); - - // The corpus database layout assumes the crash input files are located - // directly in the crashing directory, so we don't list recursively. - std::vector old_input_files = - ValueOrDie(RemoteListFiles(crashing_dir.c_str(), /*recursively=*/false)); - size_t crash_input_count = old_input_files.size(); - ScopedCentipedeCallbacks scoped_callbacks(callbacks_factory, env, - stop_condition); - BatchResult batch_result; - - absl::flat_hash_map reproduced_crashes; - for (const std::string& old_input_file : old_input_files) { - ByteArray old_input; - FUZZTEST_CHECK_OK(RemoteFileGetContents(old_input_file, old_input)); - const bool is_reproducible = !scoped_callbacks.callbacks()->Execute( - env.binary, {old_input}, batch_result) && - batch_result.IsInputFailure(); - auto input_file_components = ParseCrashingInputFilename(old_input_file); - FUZZTEST_LOG_IF(WARNING, !input_file_components.ok()) - << "Failed to get input file components for " << old_input_file - << ". Status: " << input_file_components.status(); - - if (is_reproducible) { - if (input_file_components.ok()) { - // Overwrite the old crash signature with the new one. - input_file_components->crash_signature = - batch_result.failure_signature(); - } else { - // We'll rename the input file to the new format using the input - // signature as the bug ID. - const std::string input_signature = Hash(old_input); - input_file_components = InputFileComponents{ - /*bug_id=*/input_signature, - /*crash_signature=*/batch_result.failure_signature(), - /*input_signature=*/input_signature, - }; - } - - std::string new_input_file_name = GetInputFileName( - input_file_components->bug_id, input_file_components->crash_signature, - input_file_components->input_signature); - std::string new_input_file = crashing_dir / new_input_file_name; - if (old_input_file == new_input_file) { - const auto status = RemotePathTouchExistingFile(new_input_file); - FUZZTEST_LOG_IF(ERROR, !status.ok()) - << "Failed to touch file " << new_input_file - << ". Status: " << status; - } else { - const auto status = RemoteFileRename(old_input_file, new_input_file); - if (!status.ok()) { - FUZZTEST_LOG(ERROR) - << "Failed to rename file " << old_input_file << " to " - << new_input_file << ". Status: " << status; - new_input_file_name = - std::filesystem::path(old_input_file).filename(); - new_input_file = old_input_file; - } - } - // In crash reports we report the full file name as the crash ID. This is - // what the user can use to replay or export the crash. - crash_summary.AddCrash({/*id=*/new_input_file_name, - /*category=*/batch_result.failure_description(), - batch_result.failure_signature(), - batch_result.failure_description()}); - reproduced_crashes.try_emplace( - batch_result.failure_signature(), - CrashDetails{ - /*input_signature=*/input_file_components->input_signature, - /*description=*/batch_result.failure_description(), - /*input_path=*/new_input_file, - }); - continue; - } - FUZZTEST_CHECK(!is_reproducible); - - if (!input_file_components.ok()) { - // Irreproducible, no bug ID, and no crash signature. Nothing to do with - // this input but move it to the regression directory. - const std::string regression_input_file = - regression_dir / Hash(old_input); - const auto status = - RemoteFileRename(old_input_file, regression_input_file); - if (status.ok()) { - --crash_input_count; - } else { - FUZZTEST_LOG(ERROR) - << "Failed to rename file " << old_input_file << " to " - << regression_input_file << ". Status: " << status; - } - continue; - } - - auto crash_it = - reproduced_crashes.find(input_file_components->crash_signature); - auto new_crash_it = crash_it == reproduced_crashes.end() - ? new_crashes_by_signature.find( - input_file_components->crash_signature) - : new_crashes_by_signature.end(); - if (crash_it != reproduced_crashes.end() || - new_crash_it == new_crashes_by_signature.end()) { - const std::string regression_input_file = - regression_dir / input_file_components->input_signature; - const auto status = RemoteFileCopy(old_input_file, regression_input_file); - FUZZTEST_LOG_IF(ERROR, !status.ok()) - << "Failed to copy file " << old_input_file << " to " - << regression_input_file << ". Status: " << status; - continue; - } - crash_it = reproduced_crashes.insert(*new_crash_it).first; - - const std::string new_input_file_name = GetInputFileName( - input_file_components->bug_id, input_file_components->crash_signature, - crash_it->second.input_signature); - const std::string new_input_file = crashing_dir / new_input_file_name; - absl::Status replace_status; - if (new_input_file == old_input_file) { - // For some reason, the old input couldn't reproduce the crash during - // reproduction, but it was re-discovered during fuzzing, so it is - // flaky. We keep the input and don't store it as a regression. - replace_status = RemotePathTouchExistingFile(new_input_file); - FUZZTEST_LOG_IF(ERROR, !replace_status.ok()) - << "Failed to touch file " << new_input_file - << ". Status: " << replace_status; - } else { - const std::string regression_input_file = - regression_dir / input_file_components->input_signature; - replace_status = RemoteFileRename(old_input_file, regression_input_file); - if (replace_status.ok()) { - --crash_input_count; - replace_status = - RemoteFileCopy(crash_it->second.input_path, new_input_file); - if (replace_status.ok()) { - ++crash_input_count; - } else { - FUZZTEST_LOG(ERROR) - << "Failed to copy file " << crash_it->second.input_path << " to " - << new_input_file << ". Status: " << replace_status; - } - } else { - FUZZTEST_LOG(ERROR) - << "Failed to rename file " << old_input_file << " to " - << regression_input_file << ". Status: " << replace_status; - } - } - if (replace_status.ok()) { - crash_summary.AddCrash({/*id=*/new_input_file_name, - /*category=*/crash_it->second.description, - input_file_components->crash_signature, - crash_it->second.description}); - } else { - reproduced_crashes.erase(crash_it); - } - } - - static constexpr int kMaxCrashInputCount = 10; - for (auto& [crash_signature, details] : new_crashes_by_signature) { - if (reproduced_crashes.contains(crash_signature)) continue; - if (crash_input_count >= kMaxCrashInputCount) { - FUZZTEST_LOG(WARNING) - << "Reached the maximum number of crash inputs: " - << kMaxCrashInputCount << ". Not storing any new crashes."; - break; - } - const std::string bug_id = Hash( - absl::StrCat(absl::FormatTime(absl::Now()), details.input_signature)); - const std::string new_input_file_name = - GetInputFileName(bug_id, crash_signature, details.input_signature); - const std::string new_input_file = crashing_dir / new_input_file_name; - const auto status = RemoteFileCopy(details.input_path, new_input_file); - if (!status.ok()) { - FUZZTEST_LOG(ERROR) << "Failed to copy file " << details.input_path - << " to " << new_input_file << ". Status: " << status; - continue; - } - crash_summary.AddCrash({/*id=*/new_input_file_name, - /*category=*/details.description, crash_signature, - details.description}); - reproduced_crashes.insert({std::move(crash_signature), std::move(details)}); - ++crash_input_count; - } + CrashSummary& crash_summary, StopCondition& stop_condition, + absl::Duration regression_ttl, absl::Clock& clock) { + auto status = OrganizeCrashingInputsImpl( + regression_dir, crashing_dir, incubating_dir, env, callbacks_factory, + new_crashes_by_signature, crash_summary, stop_condition, regression_ttl, + clock); + FUZZTEST_LOG_IF(ERROR, !status.ok()) + << "Failed to organize crashing inputs: " << status; } } // namespace fuzztest::internal diff --git a/centipede/crash_deduplication.h b/centipede/crash_deduplication.h index 936d12a7d..d22507396 100644 --- a/centipede/crash_deduplication.h +++ b/centipede/crash_deduplication.h @@ -20,9 +20,12 @@ #include #include "absl/container/flat_hash_map.h" +#include "absl/time/clock_interface.h" +#include "absl/time/time.h" #include "./centipede/centipede_callbacks.h" #include "./centipede/crash_summary.h" #include "./centipede/environment.h" +#include "./centipede/stop.h" #include "./centipede/workdir.h" namespace fuzztest::internal { @@ -41,51 +44,16 @@ absl::flat_hash_map GetCrashesFromWorkdir( // Organizes crashing inputs from `crashing_dir` by attempting to reproduce // them, and stores new crashes from `new_crashes_by_signature` that are not // duplicates of existing ones. -// -// The input files in `crashing_dir` are uniquely identified by `bug_id`s. The -// file names are in the format `--` -// or `` (legacy format, in which case `` is -// also considered to be the `bug_id`, and the crash signature is considered to -// be missing). -// -// 1. Inputs from `crashing_dir` are re-executed: -// - If an input is reproducible and causes an input failure (i.e., not a -// setup failure or other special cases): -// - It is kept in `crashing_dir` and reported to `crash_summary`. -// - If its crash signature has changed, it is renamed to reflect the new -// signature: `--`. -// - The file's modification time is updated. -// - If an input is not reproducible or doesn't cause an input failure: -// - If its parsed crash signature is found in `new_crashes_by_signature`, -// this signature hasn't been seen in a reproducible crash yet, and no -// other input has been processed for this signature in the current call: -// - The `bug_id` is reused for bug continuity: If the input from -// `new_crashes_by_signature` has the same input signature as the -// irreproducible input (flaky crash), the file is kept in -// `crashing_dir` and its modification time is updated. Otherwise, the -// irreproducible input is moved to `regression_dir`, and the input -// from `new_crashes_by_signature` is copied to `crashing_dir` using -// file name `--`. -// - The crash is reported to `crash_summary`. -// - Otherwise, if the input file has a valid name, it is copied to -// `regression_dir`. -// - If input file has an invalid name, it is moved to `regression_dir`. -// -// 2. New crashes from `new_crashes_by_signature` are stored: -// - If a new crash has a signature that was not observed among crashes -// from step 1, it is stored in `crashing_dir` with a newly generated -// `bug_id`: `--`, and -// reported to `crash_summary`. -// - If the total number of inputs in `crashing_dir` reaches a predefined -// limit, no more new crashes will be stored (unless they replace old -// irreproducible inputs as described in step 1). void OrganizeCrashingInputs( const std::filesystem::path& regression_dir, - const std::filesystem::path& crashing_dir, const Environment& env, + const std::filesystem::path& crashing_dir, + const std::filesystem::path& incubating_dir, const Environment& env, CentipedeCallbacksFactory& callbacks_factory, const absl::flat_hash_map& new_crashes_by_signature, - CrashSummary& crash_summary, StopCondition& stop_condition); + CrashSummary& crash_summary, StopCondition& stop_condition, + absl::Duration regression_ttl, + absl::Clock& clock = absl::Clock::GetRealClock()); } // namespace fuzztest::internal diff --git a/centipede/crash_deduplication_test.cc b/centipede/crash_deduplication_test.cc index cad5d3fde..afcd67259 100644 --- a/centipede/crash_deduplication_test.cc +++ b/centipede/crash_deduplication_test.cc @@ -14,7 +14,6 @@ #include "./centipede/crash_deduplication.h" -#include #include // NOLINT #include #include @@ -26,17 +25,16 @@ #include "absl/container/flat_hash_map.h" #include "absl/strings/str_format.h" #include "absl/time/clock.h" +#include "absl/time/clock_interface.h" +#include "absl/time/simulated_clock.h" #include "absl/time/time.h" -#include "absl/types/span.h" #include "./centipede/centipede_callbacks.h" +#include "./centipede/crash_deduplication_test_util.h" #include "./centipede/crash_summary.h" #include "./centipede/environment.h" -#include "./centipede/runner_result.h" #include "./centipede/stop.h" #include "./centipede/util.h" #include "./centipede/workdir.h" -#include "./common/defs.h" -#include "./common/hash.h" #include "./common/temp_dir.h" namespace fuzztest::internal { @@ -158,38 +156,6 @@ TEST(GetCrashesFromWorkdirTest, FailsOnEmptyCrashDescriptionIfEnvVarSet) { unsetenv("FUZZTEST_FAIL_ON_EMPTY_CRASH_METADATA"); } -class FakeCentipedeCallbacks : public CentipedeCallbacks { - public: - struct Crash { - std::string signature; - std::string description; - }; - - explicit FakeCentipedeCallbacks( - const Environment& env, - absl::flat_hash_map crashing_inputs) - : CentipedeCallbacks(env, internal_stop_condition_), - crashing_inputs_(std::move(crashing_inputs)) {} - - bool Execute(std::string_view binary, absl::Span inputs, - BatchResult& batch_result) override { - batch_result.ClearAndResize(inputs.size()); - for (ByteSpan input : inputs) { - auto it = crashing_inputs_.find(AsStringView(input)); - if (it == crashing_inputs_.end()) continue; - batch_result.exit_code() = EXIT_FAILURE; - batch_result.failure_signature() = it->second.signature; - batch_result.failure_description() = it->second.description; - return false; - } - return true; - } - - private: - absl::flat_hash_map crashing_inputs_; - StopCondition internal_stop_condition_; -}; - struct FileAndContents { std::string basename; std::string contents; @@ -216,9 +182,11 @@ class OrganizeCrashingInputsTest : public ::testing::Test { OrganizeCrashingInputsTest() : crashing_dir_(test_dir_.path() / "crashing"), regression_dir_(test_dir_.path() / "regression"), + incubating_dir_(test_dir_.path() / "incubating"), new_crashes_dir_(test_dir_.path() / "new_crashes") { std::filesystem::create_directories(crashing_dir_); std::filesystem::create_directories(regression_dir_); + std::filesystem::create_directories(incubating_dir_); std::filesystem::create_directories(new_crashes_dir_); } @@ -226,17 +194,35 @@ class OrganizeCrashingInputsTest : public ::testing::Test { const std::filesystem::path& regression_dir() const { return regression_dir_; } + const std::filesystem::path& incubating_dir() const { + return incubating_dir_; + } const std::filesystem::path& new_crashes_dir() const { return new_crashes_dir_; } const Environment& env() const { return env_; } CrashSummary& crash_summary() { return crash_summary_; } - StopCondition& stop_condition() { return stop_condition_; } + + void OrganizeCrashingInputs( + const std::filesystem::path& regression_dir, + const std::filesystem::path& crashing_dir, const Environment& env, + CentipedeCallbacksFactory& callbacks_factory, + const absl::flat_hash_map& + new_crashes_by_signature, + CrashSummary& crash_summary, + absl::Duration regression_ttl = absl::Hours(24 * 7), + absl::Clock& clock = absl::Clock::GetRealClock()) { + ::fuzztest::internal::OrganizeCrashingInputs( + regression_dir, crashing_dir, incubating_dir_, env, callbacks_factory, + new_crashes_by_signature, crash_summary, stop_condition_, + regression_ttl, clock); + } private: TempDir test_dir_; std::filesystem::path crashing_dir_; std::filesystem::path regression_dir_; + std::filesystem::path incubating_dir_; std::filesystem::path new_crashes_dir_; Environment env_; CrashSummary crash_summary_{"binary_id", "fuzz_test"}; @@ -251,8 +237,7 @@ TEST_F(OrganizeCrashingInputsTest, CreatesDirectoriesIfMissing) { NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir, crashing_dir, env(), factory, - /*new_crashes_by_signature=*/{}, crash_summary(), - stop_condition()); + /*new_crashes_by_signature=*/{}, crash_summary()); const std::filesystem::directory_entry crashing_dir_entry{crashing_dir}; const std::filesystem::directory_entry regression_dir_entry{regression_dir}; @@ -261,7 +246,7 @@ TEST_F(OrganizeCrashingInputsTest, CreatesDirectoriesIfMissing) { regression_dir_entry.exists() && regression_dir_entry.is_directory()); } -TEST_F(OrganizeCrashingInputsTest, RenamesOldStyleCrashFileToNewStyle) { +TEST_F(OrganizeCrashingInputsTest, DeletesOldStyleCrashes) { SetContentsAndGetPath(crashing_dir(), "isig", "input"); FakeCentipedeCallbacks callbacks(env(), /*crashing_inputs=*/{ {"input", {"csig", "desc"}}, @@ -269,19 +254,13 @@ TEST_F(OrganizeCrashingInputsTest, RenamesOldStyleCrashFileToNewStyle) { NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - /*new_crashes_by_signature=*/{}, crash_summary(), - stop_condition()); + /*new_crashes_by_signature=*/{}, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); - EXPECT_THAT(ReadFiles(crashing_dir()), - UnorderedElementsAre(FieldsAre("isig-csig-isig", "input"))); + EXPECT_THAT(ReadFiles(crashing_dir()), IsEmpty()); EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); - EXPECT_THAT(crash_report, AllOf(HasSubstr("Total crashes: 1"), - HasSubstr("Crash ID : isig-csig-isig"), - HasSubstr("Category : desc"), - HasSubstr("Signature : csig"), - HasSubstr("Description: desc"))); + EXPECT_THAT(crash_report, HasSubstr("Total crashes: 0")); } TEST_F(OrganizeCrashingInputsTest, KeepsNewStyleCrashFileIfSignatureUnchanged) { @@ -292,14 +271,14 @@ TEST_F(OrganizeCrashingInputsTest, KeepsNewStyleCrashFileIfSignatureUnchanged) { NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - /*new_crashes_by_signature=*/{}, crash_summary(), - stop_condition()); + /*new_crashes_by_signature=*/{}, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); EXPECT_THAT(ReadFiles(crashing_dir()), UnorderedElementsAre(FieldsAre("bug-csig-isig", "input"))); EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); + EXPECT_THAT(ReadFiles(incubating_dir()), IsEmpty()); EXPECT_THAT(crash_report, AllOf(HasSubstr("Total crashes: 1"), HasSubstr("Crash ID : bug-csig-isig"), HasSubstr("Category : desc"), @@ -307,7 +286,7 @@ TEST_F(OrganizeCrashingInputsTest, KeepsNewStyleCrashFileIfSignatureUnchanged) { HasSubstr("Description: desc"))); } -TEST_F(OrganizeCrashingInputsTest, UpdatesCrashSignatureInFileNameIfChanged) { +TEST_F(OrganizeCrashingInputsTest, AddsNewCrashIfCrashSignatureChanges) { SetContentsAndGetPath(crashing_dir(), "bug-csig_old-isig", "input"); FakeCentipedeCallbacks callbacks(env(), /*crashing_inputs=*/{ {"input", {"csig_new", "desc"}}, @@ -315,19 +294,23 @@ TEST_F(OrganizeCrashingInputsTest, UpdatesCrashSignatureInFileNameIfChanged) { NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - /*new_crashes_by_signature=*/{}, crash_summary(), - stop_condition()); + /*new_crashes_by_signature=*/{}, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); EXPECT_THAT(ReadFiles(crashing_dir()), - UnorderedElementsAre(FieldsAre("bug-csig_new-isig", "input"))); + UnorderedElementsAre( + FieldsAre("bug-csig_old-isig", "input"), + FieldsAre(testing::MatchesRegex("[a-f0-9]+-csig_new-isig"), + "input"))); EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); - EXPECT_THAT(crash_report, AllOf(HasSubstr("Total crashes: 1"), - HasSubstr("Crash ID : bug-csig_new-isig"), - HasSubstr("Category : desc"), - HasSubstr("Signature : csig_new"), - HasSubstr("Description: desc"))); + EXPECT_THAT(ReadFiles(incubating_dir()), IsEmpty()); + EXPECT_THAT( + crash_report, + AllOf(HasSubstr("Total crashes: 1"), + testing::ContainsRegex("Crash ID : [a-f0-9]+-csig_new-isig"), + HasSubstr("Category : desc"), HasSubstr("Signature : csig_new"), + HasSubstr("Description: desc"))); } TEST_F(OrganizeCrashingInputsTest, @@ -349,8 +332,7 @@ TEST_F(OrganizeCrashingInputsTest, NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - /*new_crashes_by_signature=*/{}, crash_summary(), - stop_condition()); + /*new_crashes_by_signature=*/{}, crash_summary()); EXPECT_GT(std::filesystem::last_write_time(reproducible_input_path), reproducible_mtime_before); @@ -359,7 +341,7 @@ TEST_F(OrganizeCrashingInputsTest, } TEST_F(OrganizeCrashingInputsTest, - KeepsReproducibleCrashesWithSameCrashSignature) { + KeepsOldFilesWhenDeduplicatingToSameSignature) { SetContentsAndGetPath(crashing_dir(), "bug1-csig1-isig1", "input1"); SetContentsAndGetPath(crashing_dir(), "bug2-csig2-isig2", "input2"); FakeCentipedeCallbacks callbacks(env(), /*crashing_inputs=*/{ @@ -369,21 +351,26 @@ TEST_F(OrganizeCrashingInputsTest, NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - /*new_crashes_by_signature=*/{}, crash_summary(), - stop_condition()); + /*new_crashes_by_signature=*/{}, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); EXPECT_THAT(ReadFiles(crashing_dir()), - UnorderedElementsAre(FieldsAre("bug1-csig-isig1", "input1"), - FieldsAre("bug2-csig-isig2", "input2"))); + AnyOf(UnorderedElementsAre( + FieldsAre("bug1-csig1-isig1", "input1"), + FieldsAre("bug2-csig2-isig2", "input2"), + FieldsAre(testing::MatchesRegex("[a-f0-9]+-csig-isig1"), + "input1")), + UnorderedElementsAre( + FieldsAre("bug1-csig1-isig1", "input1"), + FieldsAre("bug2-csig2-isig2", "input2"), + FieldsAre(testing::MatchesRegex("[a-f0-9]+-csig-isig2"), + "input2")))); EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); - EXPECT_THAT(crash_report, AllOf(HasSubstr("Total crashes: 2"), - HasSubstr("Crash ID : bug1-csig-isig1"), - HasSubstr("Crash ID : bug2-csig-isig2"), - HasSubstr("Category : desc"), - HasSubstr("Signature : csig"), - HasSubstr("Description: desc"))); + EXPECT_THAT( + crash_report, + AllOf(HasSubstr("Total crashes: 1"), HasSubstr("Category : desc"), + HasSubstr("Signature : csig"), HasSubstr("Description: desc"))); } TEST_F(OrganizeCrashingInputsTest, KeepsFlakyCrashAndUpdatesModificationTime) { @@ -402,8 +389,7 @@ TEST_F(OrganizeCrashingInputsTest, KeepsFlakyCrashAndUpdatesModificationTime) { new_crashes_by_signature["csig"] = {"isig", "desc", new_input_path}; OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - new_crashes_by_signature, crash_summary(), - stop_condition()); + new_crashes_by_signature, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); @@ -418,27 +404,45 @@ TEST_F(OrganizeCrashingInputsTest, KeepsFlakyCrashAndUpdatesModificationTime) { EXPECT_GT(std::filesystem::last_write_time(input_path), mtime_before); } -TEST_F(OrganizeCrashingInputsTest, - KeepsIrreproducibleCrashAndCopiesToRegressionDir) { +TEST_F(OrganizeCrashingInputsTest, KeepsIrreproducibleCrashIfTtlNotExpired) { SetContentsAndGetPath(crashing_dir(), "bug-csig-isig", "input"); FakeCentipedeCallbacks callbacks(env(), /*crashing_inputs=*/{}); NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - /*new_crashes_by_signature=*/{}, crash_summary(), - stop_condition()); + /*new_crashes_by_signature=*/{}, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); EXPECT_THAT(ReadFiles(crashing_dir()), UnorderedElementsAre(FieldsAre("bug-csig-isig", "input"))); + EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); + EXPECT_THAT(ReadFiles(incubating_dir()), IsEmpty()); + EXPECT_THAT(crash_report, HasSubstr("Total crashes: 0")); +} + +TEST_F(OrganizeCrashingInputsTest, + MovesIrreproducibleCrashToRegressionIfTtlExpired) { + absl::SimulatedClock clock(absl::Now()); + SetContentsAndGetPath(crashing_dir(), "bug-csig-isig", "input"); + clock.AdvanceTime(absl::Hours(25)); + + FakeCentipedeCallbacks callbacks(env(), /*crashing_inputs=*/{}); + NonOwningCallbacksFactory factory(callbacks); + + OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, + /*new_crashes_by_signature=*/{}, crash_summary(), + /*regression_ttl=*/absl::Hours(24), clock); + std::string crash_report; + crash_summary().Report(&crash_report); + + EXPECT_THAT(ReadFiles(crashing_dir()), IsEmpty()); EXPECT_THAT(ReadFiles(regression_dir()), UnorderedElementsAre(FieldsAre("isig", "input"))); EXPECT_THAT(crash_report, HasSubstr("Total crashes: 0")); } -TEST_F(OrganizeCrashingInputsTest, - KeepsSetupFailureCrashAndCopiesToRegressionDir) { +TEST_F(OrganizeCrashingInputsTest, KeepsSetupFailureCrashIfTtlNotExpired) { SetContentsAndGetPath(crashing_dir(), "bug-csig-isig", "input"); FakeCentipedeCallbacks callbacks( env(), /*crashing_inputs=*/{ @@ -447,33 +451,53 @@ TEST_F(OrganizeCrashingInputsTest, NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - /*new_crashes_by_signature=*/{}, crash_summary(), - stop_condition()); + /*new_crashes_by_signature=*/{}, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); EXPECT_THAT(ReadFiles(crashing_dir()), UnorderedElementsAre(FieldsAre("bug-csig-isig", "input"))); + EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); + EXPECT_THAT(crash_report, HasSubstr("Total crashes: 0")); +} + +TEST_F(OrganizeCrashingInputsTest, + MovesSetupFailureCrashToRegressionIfTtlExpired) { + absl::SimulatedClock clock(absl::Now()); + SetContentsAndGetPath(crashing_dir(), "bug-csig-isig", "input"); + clock.AdvanceTime(absl::Hours(25)); + + FakeCentipedeCallbacks callbacks( + env(), /*crashing_inputs=*/{ + {"input", {"csig", "SETUP FAILURE: desc"}}, + }); + NonOwningCallbacksFactory factory(callbacks); + + OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, + /*new_crashes_by_signature=*/{}, crash_summary(), + /*regression_ttl=*/absl::Hours(24), clock); + std::string crash_report; + crash_summary().Report(&crash_report); + + EXPECT_THAT(ReadFiles(crashing_dir()), IsEmpty()); EXPECT_THAT(ReadFiles(regression_dir()), UnorderedElementsAre(FieldsAre("isig", "input"))); EXPECT_THAT(crash_report, HasSubstr("Total crashes: 0")); } TEST_F(OrganizeCrashingInputsTest, - MovesIrreproducibleCrashWithMalformedFileNameToRegressionDir) { + DeletesIrreproducibleCrashWithMalformedFileName) { SetContentsAndGetPath(crashing_dir(), "invalid-name", "input"); FakeCentipedeCallbacks callbacks(env(), /*crashing_inputs=*/{}); NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - /*new_crashes_by_signature=*/{}, crash_summary(), - stop_condition()); + /*new_crashes_by_signature=*/{}, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); EXPECT_THAT(ReadFiles(crashing_dir()), IsEmpty()); - EXPECT_THAT(ReadFiles(regression_dir()), - UnorderedElementsAre(FieldsAre(Hash("input"), "input"))); + EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); EXPECT_THAT(crash_report, HasSubstr("Total crashes: 0")); } @@ -489,14 +513,14 @@ TEST_F(OrganizeCrashingInputsTest, new_crashes_by_signature["csig"] = {"isig2", "desc2", input2_path}; OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - new_crashes_by_signature, crash_summary(), - stop_condition()); + new_crashes_by_signature, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); EXPECT_THAT(ReadFiles(crashing_dir()), UnorderedElementsAre(FieldsAre("bug-csig-isig2", "input2"))); - EXPECT_THAT(ReadFiles(regression_dir()), + EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); + EXPECT_THAT(ReadFiles(incubating_dir()), UnorderedElementsAre(FieldsAre("isig1", "input1"))); EXPECT_THAT(crash_report, AllOf(HasSubstr("Total crashes: 1"), HasSubstr("Crash ID : bug-csig-isig2"), @@ -506,7 +530,7 @@ TEST_F(OrganizeCrashingInputsTest, } TEST_F(OrganizeCrashingInputsTest, - DoesNotReplaceIrreproducibleCrashIfReproducedByAnotherOldInput) { + ReplacesIrreproducibleCrashIfReproducedByAnotherOldInput) { SetContentsAndGetPath(crashing_dir(), "bug1-csig-isig1", "input1"); SetContentsAndGetPath(crashing_dir(), "bug2-csig-isig2", "input2"); FakeCentipedeCallbacks callbacks(env(), /*crashing_inputs=*/{ @@ -515,18 +539,17 @@ TEST_F(OrganizeCrashingInputsTest, NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - /*new_crashes_by_signature=*/{}, crash_summary(), - stop_condition()); + /*new_crashes_by_signature=*/{}, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); EXPECT_THAT(ReadFiles(crashing_dir()), UnorderedElementsAre(FieldsAre("bug1-csig-isig1", "input1"), - FieldsAre("bug2-csig-isig2", "input2"))); - EXPECT_THAT(ReadFiles(regression_dir()), - UnorderedElementsAre(FieldsAre("isig2", "input2"))); - EXPECT_THAT(crash_report, AllOf(HasSubstr("Total crashes: 1"), + FieldsAre("bug2-csig-isig1", "input1"))); + EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); + EXPECT_THAT(crash_report, AllOf(HasSubstr("Total crashes: 2"), HasSubstr("Crash ID : bug1-csig-isig1"), + HasSubstr("Crash ID : bug2-csig-isig1"), HasSubstr("Category : desc1"), HasSubstr("Signature : csig"), HasSubstr("Description: desc1"))); @@ -542,8 +565,7 @@ TEST_F(OrganizeCrashingInputsTest, StoresNewCrashWithUniqueCrashSignature) { new_crashes_by_signature["csig"] = {"isig", "desc", input_path}; OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - new_crashes_by_signature, crash_summary(), - stop_condition()); + new_crashes_by_signature, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); @@ -572,8 +594,7 @@ TEST_F(OrganizeCrashingInputsTest, new_crashes_by_signature["csig"] = {"isig2", "desc2", input2_path}; OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - new_crashes_by_signature, crash_summary(), - stop_condition()); + new_crashes_by_signature, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); @@ -594,8 +615,7 @@ TEST_F(OrganizeCrashingInputsTest, DoesNotProcessInputsInRegressionDir) { NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - /*new_crashes_by_signature=*/{}, crash_summary(), - stop_condition()); + /*new_crashes_by_signature=*/{}, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); @@ -634,8 +654,7 @@ TEST_F(OrganizeCrashingInputsTest, AddsNewCrashesUpToFileLimit) { NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - new_crashes_by_signature, crash_summary(), - stop_condition()); + new_crashes_by_signature, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); @@ -652,11 +671,7 @@ TEST_F(OrganizeCrashingInputsTest, AddsNewCrashesUpToFileLimit) { FieldsAre("bug9-csig9-isig9", "irrepro9"), AnyOf(FieldsAre(HasSubstr("-csig10-isig10"), "new10"), FieldsAre(HasSubstr("-csig11-isig11"), "new11")))); - EXPECT_THAT(ReadFiles(regression_dir()), - UnorderedElementsAre(FieldsAre("isig6", "irrepro6"), - FieldsAre("isig7", "irrepro7"), - FieldsAre("isig8", "irrepro8"), - FieldsAre("isig9", "irrepro9"))); + EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); EXPECT_THAT(crash_report, HasSubstr("Total crashes: 6")); } @@ -691,8 +706,7 @@ TEST_F(OrganizeCrashingInputsTest, ReplacesIrreproducibleCrashAtFileLimit) { NonOwningCallbacksFactory factory(callbacks); OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, - new_crashes_by_signature, crash_summary(), - stop_condition()); + new_crashes_by_signature, crash_summary()); std::string crash_report; crash_summary().Report(&crash_report); @@ -707,8 +721,7 @@ TEST_F(OrganizeCrashingInputsTest, ReplacesIrreproducibleCrashAtFileLimit) { FieldsAre("bug8-csig8-isig8", "repro8"), FieldsAre("bug9-csig9-isig9", "repro9"), FieldsAre("bug10-csig10-isig11", "new11"))); - EXPECT_THAT(ReadFiles(regression_dir()), - UnorderedElementsAre(FieldsAre("isig10", "irrepro10"))); + EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); EXPECT_THAT(crash_report, AllOf(HasSubstr("Total crashes: 10"), HasSubstr("Crash ID : bug10-csig10-isig11"), HasSubstr("Category : desc11"), @@ -716,5 +729,132 @@ TEST_F(OrganizeCrashingInputsTest, ReplacesIrreproducibleCrashAtFileLimit) { HasSubstr("Description: desc11"))); } +TEST_F(OrganizeCrashingInputsTest, + MovesReproducingIncubatingCrashToCrashingDir) { + SetContentsAndGetPath(incubating_dir(), "isig1", "input1"); + + FakeCentipedeCallbacks callbacks(env(), /*crashing_inputs=*/{ + {"input1", {"csig", "desc"}}, + }); + NonOwningCallbacksFactory factory(callbacks); + + OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, + /*new_crashes_by_signature=*/{}, crash_summary()); + std::string crash_report; + crash_summary().Report(&crash_report); + + EXPECT_THAT(ReadFiles(incubating_dir()), IsEmpty()); + EXPECT_THAT(ReadFiles(crashing_dir()), + UnorderedElementsAre(FieldsAre( + testing::MatchesRegex("[a-f0-9]+-csig-isig1"), "input1"))); + EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); + EXPECT_THAT( + crash_report, + AllOf(HasSubstr("Total crashes: 1"), + testing::ContainsRegex("Crash ID : [a-f0-9]+-csig-isig1"), + HasSubstr("Category : desc"), HasSubstr("Signature : csig"), + HasSubstr("Description: desc"))); +} + +TEST_F(OrganizeCrashingInputsTest, + KeepsIrreproducibleIncubatingCrashIfTtlNotExpired) { + SetContentsAndGetPath(incubating_dir(), "isig1", "input1"); + + FakeCentipedeCallbacks callbacks(env(), /*crashing_inputs=*/{}); + NonOwningCallbacksFactory factory(callbacks); + + OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, + /*new_crashes_by_signature=*/{}, crash_summary(), + /*regression_ttl=*/absl::Hours(24)); + std::string crash_report; + crash_summary().Report(&crash_report); + + EXPECT_THAT(ReadFiles(incubating_dir()), + UnorderedElementsAre(FieldsAre("isig1", "input1"))); + EXPECT_THAT(ReadFiles(crashing_dir()), IsEmpty()); + EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); + EXPECT_THAT(crash_report, HasSubstr("Total crashes: 0")); +} + +TEST_F(OrganizeCrashingInputsTest, MovesExpiredIncubatingCrashToRegressionDir) { + absl::SimulatedClock clock(absl::Now()); + SetContentsAndGetPath(incubating_dir(), "isig1", "input1"); + clock.AdvanceTime(absl::Hours(25)); + + FakeCentipedeCallbacks callbacks(env(), /*crashing_inputs=*/{}); + NonOwningCallbacksFactory factory(callbacks); + + OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, + /*new_crashes_by_signature=*/{}, crash_summary(), + /*regression_ttl=*/absl::Hours(24), clock); + std::string crash_report; + crash_summary().Report(&crash_report); + + EXPECT_THAT(ReadFiles(incubating_dir()), IsEmpty()); + EXPECT_THAT(ReadFiles(crashing_dir()), IsEmpty()); + EXPECT_THAT(ReadFiles(regression_dir()), + UnorderedElementsAre(FieldsAre("isig1", "input1"))); + EXPECT_THAT(crash_report, HasSubstr("Total crashes: 0")); +} + +TEST_F(OrganizeCrashingInputsTest, + ReplacesCrashInputIfSignatureChangesButNewCrashWithOldSignatureExists) { + SetContentsAndGetPath(crashing_dir(), "bug1-csig_old-isig1", "input1"); + + FakeCentipedeCallbacks callbacks(env(), /*crashing_inputs=*/{ + {"input1", {"csig_new", "desc_new"}}, + }); + NonOwningCallbacksFactory factory(callbacks); + + absl::flat_hash_map new_crashes_by_signature; + const auto input2_path = + SetContentsAndGetPath(new_crashes_dir(), "isig2", "input2"); + new_crashes_by_signature["csig_old"] = {"isig2", "desc_old", input2_path}; + + OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, + new_crashes_by_signature, crash_summary()); + std::string crash_report; + crash_summary().Report(&crash_report); + + EXPECT_THAT(ReadFiles(crashing_dir()), + UnorderedElementsAre( + FieldsAre("bug1-csig_old-isig2", "input2"), + FieldsAre(testing::MatchesRegex("[a-f0-9]+-csig_new-isig1"), + "input1"))); + EXPECT_THAT(ReadFiles(regression_dir()), IsEmpty()); + EXPECT_THAT(ReadFiles(incubating_dir()), IsEmpty()); + EXPECT_THAT( + crash_report, + AllOf(HasSubstr("Total crashes: 2"), + HasSubstr("Crash ID : bug1-csig_old-isig2"), + testing::ContainsRegex("Crash ID : [a-f0-9]+-csig_new-isig1"), + HasSubstr("Signature : csig_old"), + HasSubstr("Signature : csig_new"))); +} + +TEST_F(OrganizeCrashingInputsTest, ReplacesInputWithWinnerAlreadyOnDisk) { + // Setup two existing crashes for the same bug/signature. + SetContentsAndGetPath(crashing_dir(), "bug1-sig1-isig1", "input1"); + SetContentsAndGetPath(crashing_dir(), "bug1-sig1-isig2", "input2"); + + // input1 reproduces with sig1 (winner). + // input2 does not reproduce. + FakeCentipedeCallbacks callbacks(env(), /*crashing_inputs=*/{ + {"input1", {"sig1", "desc1"}}, + }); + NonOwningCallbacksFactory factory(callbacks); + + // This should not fail with "filesystem::copy() failed" (self-copy). + OrganizeCrashingInputs(regression_dir(), crashing_dir(), env(), factory, + /*new_crashes_by_signature=*/{}, crash_summary()); + + // bug1-sig1-isig1 should be kept. + // bug1-sig1-isig2 should be demoted to incubating. + EXPECT_THAT(ReadFiles(crashing_dir()), + UnorderedElementsAre(FieldsAre("bug1-sig1-isig1", "input1"))); + EXPECT_THAT(ReadFiles(incubating_dir()), + UnorderedElementsAre(FieldsAre("isig2", "input2"))); +} + } // namespace } // namespace fuzztest::internal diff --git a/centipede/crash_deduplication_test_util.cc b/centipede/crash_deduplication_test_util.cc new file mode 100644 index 000000000..5b881e1e0 --- /dev/null +++ b/centipede/crash_deduplication_test_util.cc @@ -0,0 +1,42 @@ +// Copyright 2026 The Centipede Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "./centipede/crash_deduplication_test_util.h" + +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/types/span.h" +#include "./centipede/runner_result.h" +#include "./common/defs.h" + +namespace fuzztest::internal { + +bool FakeCentipedeCallbacks::Execute(std::string_view binary, + absl::Span inputs, + BatchResult& batch_result) { + batch_result.ClearAndResize(inputs.size()); + for (ByteSpan input : inputs) { + auto it = crashing_inputs_.find(AsStringView(input)); + if (it == crashing_inputs_.end()) continue; + batch_result.exit_code() = EXIT_FAILURE; + batch_result.failure_signature() = it->second.signature; + batch_result.failure_description() = it->second.description; + return false; + } + return true; +} + +} // namespace fuzztest::internal diff --git a/centipede/crash_deduplication_test_util.h b/centipede/crash_deduplication_test_util.h new file mode 100644 index 000000000..490c64a21 --- /dev/null +++ b/centipede/crash_deduplication_test_util.h @@ -0,0 +1,55 @@ +// Copyright 2026 The Centipede Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef FUZZTEST_CENTIPEDE_CRASH_DEDUPLICATION_TEST_UTIL_H_ +#define FUZZTEST_CENTIPEDE_CRASH_DEDUPLICATION_TEST_UTIL_H_ + +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/types/span.h" +#include "./centipede/centipede_callbacks.h" +#include "./centipede/environment.h" +#include "./centipede/runner_result.h" +#include "./centipede/stop.h" +#include "./common/defs.h" + +namespace fuzztest::internal { + +class FakeCentipedeCallbacks : public CentipedeCallbacks { + public: + struct Crash { + std::string signature; + std::string description; + }; + + explicit FakeCentipedeCallbacks( + const Environment& env, + absl::flat_hash_map crashing_inputs) + : CentipedeCallbacks(env, internal_stop_condition_), + crashing_inputs_(std::move(crashing_inputs)) {} + + bool Execute(std::string_view binary, absl::Span inputs, + BatchResult& batch_result) override; + + private: + absl::flat_hash_map crashing_inputs_; + StopCondition internal_stop_condition_; +}; + +} // namespace fuzztest::internal + +#endif // FUZZTEST_CENTIPEDE_CRASH_DEDUPLICATION_TEST_UTIL_H_ diff --git a/common/BUILD b/common/BUILD index d64c48092..fc9c8097c 100644 --- a/common/BUILD +++ b/common/BUILD @@ -133,6 +133,7 @@ cc_library( "@abseil-cpp//absl/status", "@abseil-cpp//absl/status:statusor", "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/time", ] + select({ "//conditions:default": [":remote_file_oss"], }) + @@ -164,6 +165,7 @@ cc_library( "@abseil-cpp//absl/status", "@abseil-cpp//absl/status:statusor", "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/time", ] + select({ "@com_google_fuzztest//fuzztest:disable_riegeli": [], "//conditions:default": [ diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index f4516741f..671195e07 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -124,6 +124,7 @@ fuzztest_cc_library( absl::status absl::statusor absl::strings + absl::time ) fuzztest_cc_library( diff --git a/common/remote_file.h b/common/remote_file.h index 0b704444e..7a370f176 100644 --- a/common/remote_file.h +++ b/common/remote_file.h @@ -30,6 +30,7 @@ #include "absl/base/nullability.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/time/time.h" #include "./common/defs.h" #ifndef CENTIPEDE_DISABLE_RIEGELI #include "riegeli/bytes/reader.h" @@ -61,22 +62,22 @@ struct RemoteFile {}; // Opens a (potentially remote) file 'file_path' and returns a handle to it. // Supported modes: "r", "a", "w", same as in C FILE API. -absl::StatusOr RemoteFileOpen(std::string_view file_path, - const char *mode); +absl::StatusOr RemoteFileOpen(std::string_view file_path, + const char* mode); // Closes the file previously opened by RemoteFileOpen. -absl::Status RemoteFileClose(RemoteFile *absl_nonnull f); +absl::Status RemoteFileClose(RemoteFile* absl_nonnull f); // Adjusts the buffered I/O capacity for a file opened for writing. By default, // the internal buffer of size `BUFSIZ` is used. May only be used after opening // a file, but before performing any other operations on it. Violating this // requirement in general can cause undefined behavior. -absl::Status RemoteFileSetWriteBufferSize(RemoteFile *absl_nonnull f, +absl::Status RemoteFileSetWriteBufferSize(RemoteFile* absl_nonnull f, size_t size); // Appends characters from 'contents' to 'f'. -absl::Status RemoteFileAppend(RemoteFile *absl_nonnull f, - const std::string &contents); +absl::Status RemoteFileAppend(RemoteFile* absl_nonnull f, + const std::string& contents); // Appends bytes from 'contents' to 'f'. absl::Status RemoteFileAppend(RemoteFile* absl_nonnull f, ByteSpan contents); @@ -85,13 +86,13 @@ absl::Status RemoteFileAppend(RemoteFile* absl_nonnull f, ByteSpan contents); // pipeline are consumed by itself (e.g. shard cross-pollination) and can be // consumed by external processes (e.g. monitoring): for such files, call this // API after every write to ensure that they are in a valid state. -absl::Status RemoteFileFlush(RemoteFile *absl_nonnull f); +absl::Status RemoteFileFlush(RemoteFile* absl_nonnull f); // Reads all current contents of 'f' into 'ba'. -absl::Status RemoteFileRead(RemoteFile *absl_nonnull f, ByteArray &ba); +absl::Status RemoteFileRead(RemoteFile* absl_nonnull f, ByteArray& ba); // Reads all current contents of 'f' into 'contents'. -absl::Status RemoteFileRead(RemoteFile *absl_nonnull f, std::string &contents); +absl::Status RemoteFileRead(RemoteFile* absl_nonnull f, std::string& contents); // Creates a (potentially remote) directory 'dir_path', as well as any missing // parent directories. No-op if the directory already exists. @@ -105,11 +106,11 @@ absl::Status RemoteFileSetContents(std::string_view path, absl::Status RemoteFileSetContents(std::string_view path, ByteSpan contents); // Reads the contents of the file at 'path' into 'contents'. -absl::Status RemoteFileGetContents(std::string_view path, ByteArray &contents); +absl::Status RemoteFileGetContents(std::string_view path, ByteArray& contents); // Reads the contents of the file at 'path' into 'contents'. absl::Status RemoteFileGetContents(std::string_view path, - std::string &contents); + std::string& contents); // Returns true if `path` exists. bool RemotePathExists(std::string_view path); @@ -120,6 +121,10 @@ bool RemotePathIsDirectory(std::string_view path); // Returns the size of the file at `path` in bytes. The file must exist. absl::StatusOr RemoteFileGetSize(std::string_view path); +// Returns the last modification time of the file at `path`. +// The file must exist. +absl::StatusOr RemoteFileGetMTime(std::string_view path); + // Finds all files matching `glob` and appends them to `matches`. // // Returns Ok when matches were found, or NotFound when no matches were found. @@ -131,7 +136,7 @@ absl::StatusOr RemoteFileGetSize(std::string_view path); // filtering based on some criterion. [[deprecated("Do not use in new code.")]] absl::Status RemoteGlobMatch(std::string_view glob, - std::vector &matches); + std::vector& matches); // Lists all files within `path`, recursively expanding subdirectories if // `recursively` is true. Does not return any directories. Returns an empty diff --git a/common/remote_file_oss.cc b/common/remote_file_oss.cc index 10299f6b6..bf7569362 100644 --- a/common/remote_file_oss.cc +++ b/common/remote_file_oss.cc @@ -24,6 +24,9 @@ #include #endif // defined(_MSC_VER) +#include +#include + #include #include #include @@ -40,6 +43,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" +#include "absl/time/time.h" #include "./common/defs.h" #include "./common/logging.h" #include "./common/remote_file.h" @@ -361,6 +365,16 @@ absl::StatusOr RemoteFileGetSize(std::string_view path) { return sz; } +absl::StatusOr RemoteFileGetMTime(std::string_view path) { + struct ::stat st; + if (::stat(path.data(), &st) != 0) { + return absl::UnknownError( + absl::StrCat("stat() failed, path: ", std::string(path), + ", errno: ", std::strerror(errno))); + } + return absl::FromTimeT(st.st_mtime); +} + namespace { #if defined(FUZZTEST_HAS_OSS_GLOB)