diff --git a/centipede/BUILD b/centipede/BUILD index 607811f8..ca62dfb5 100644 --- a/centipede/BUILD +++ b/centipede/BUILD @@ -975,38 +975,6 @@ cc_library( ], ) -cc_library( - name = "engine_worker", - srcs = [ - "engine_worker.cc", - "runner_utils.cc", - "runner_utils.h", - ], - hdrs = ["engine_worker_abi.h"], - deps = [ - ":engine_abi", - ":execution_metadata", - ":feature", - ":runner_request", - ":runner_result", - ":shared_memory_blob_sequence", - "@abseil-cpp//absl/base:nullability", - "@abseil-cpp//absl/random", - "@abseil-cpp//absl/random:bit_gen_ref", - "@com_google_fuzztest//common:defs", - ], -) - -cc_library( - name = "engine_controller_with_subprocess", - srcs = ["engine_controller_with_subprocess.cc"], - hdrs = ["engine_controller_abi.h"], - deps = [ - "@com_google_fuzztest//centipede:engine_abi", - "@com_google_fuzztest//fuzztest/internal:escaping", - ], -) - # The runner library is special: # * It must not be instrumented with asan, sancov, etc. # * It must not have heavy dependencies, and ideally not at all. @@ -1027,8 +995,6 @@ RUNNER_SOURCES_NO_MAIN = [ "runner_dl_info.cc", "runner_dl_info.h", "runner_interface.h", - "runner_utils.cc", - "runner_utils.h", "sancov_callbacks.cc", "sancov_interceptors.cc", "sancov_object_array.cc", @@ -1076,11 +1042,13 @@ RUNNER_DEPS = [ ":knobs", ":mutation_data", ":rolling_hash", - ":engine_abi", ":runner_cmp_trace", ":runner_fork_server", ":runner_request", ":runner_result", + ":runner_utils", + ":engine_abi", + ":engine_worker", ":shared_memory_blob_sequence", "@com_google_fuzztest//common:defs", "@abseil-cpp//absl/base:core_headers", @@ -1089,6 +1057,46 @@ RUNNER_DEPS = [ "@abseil-cpp//absl/types:span", ] +cc_library( + name = "runner_utils", + srcs = ["runner_utils.cc"], + hdrs = ["runner_utils.h"], + copts = RUNNER_COPTS, + deps = ["@abseil-cpp//absl/base:nullability"], +) + +cc_library( + name = "engine_worker", + srcs = [ + "engine_worker.cc", + ], + hdrs = ["engine_worker_abi.h"], + copts = RUNNER_COPTS, + deps = [ + ":engine_abi", + ":execution_metadata", + ":feature", + ":runner_request", + ":runner_result", + ":runner_utils", + ":shared_memory_blob_sequence", + "@abseil-cpp//absl/base:nullability", + "@abseil-cpp//absl/random", + "@abseil-cpp//absl/random:bit_gen_ref", + "@com_google_fuzztest//common:defs", + ], +) + +cc_library( + name = "engine_controller_with_subprocess", + srcs = ["engine_controller_with_subprocess.cc"], + hdrs = ["engine_controller_abi.h"], + deps = [ + "@com_google_fuzztest//centipede:engine_abi", + "@com_google_fuzztest//fuzztest/internal:escaping", + ], +) + # A fuzz target needs to link with this library in order to run with Centipede. # The fuzz target must provide its own main(). # @@ -1217,8 +1225,6 @@ cc_library( "reverse_pc_table.h", "runner_dl_info.cc", "runner_dl_info.h", - "runner_utils.cc", - "runner_utils.h", "sancov_callbacks.cc", "sancov_interceptors.cc", "sancov_object_array.cc", @@ -1240,6 +1246,7 @@ cc_library( ":foreach_nonzero", ":int_utils", ":runner_cmp_trace", + ":runner_utils", "@abseil-cpp//absl/base:core_headers", "@abseil-cpp//absl/base:nullability", "@abseil-cpp//absl/numeric:bits", diff --git a/centipede/centipede_test.cc b/centipede/centipede_test.cc index 17383e8f..84215a39 100644 --- a/centipede/centipede_test.cc +++ b/centipede/centipede_test.cc @@ -963,14 +963,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, GetsSeedInputs) { CentipedeDefaultCallbacks callbacks(env, stop_condition); std::vector seeds; - EXPECT_EQ(callbacks.GetSeeds(10, seeds), 10); - EXPECT_THAT(seeds, testing::ContainerEq(std::vector{ - {0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}})); - EXPECT_EQ(callbacks.GetSeeds(5, seeds), 10); - EXPECT_THAT(seeds, testing::ContainerEq( - std::vector{{0}, {1}, {2}, {3}, {4}})); - EXPECT_EQ(callbacks.GetSeeds(100, seeds), 10); - EXPECT_THAT(seeds, testing::ContainerEq(std::vector{ + callbacks.GetSeeds(10, seeds); + EXPECT_THAT(seeds, testing::IsSupersetOf(std::vector{ {0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}})); } @@ -1400,8 +1394,9 @@ TEST_F(CentipedeWithTemporaryLocalDir, EngineWorksInWorkerMode) { env.test_name = "some_test"; env.populate_binary_info = false; env.fork_server = false; - env.persistent_mode = false; + env.persistent_mode = true; env.exit_on_crash = true; + env.stop_at = absl::Now() + absl::Seconds(10); fuzztest::internal::DefaultCallbacksFactory< fuzztest::internal::CentipedeDefaultCallbacks> callbacks; diff --git a/centipede/engine_worker.cc b/centipede/engine_worker.cc index 01030d1e..3683e5bf 100644 --- a/centipede/engine_worker.cc +++ b/centipede/engine_worker.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. #include +#include +#include #include #include @@ -105,7 +107,7 @@ struct WorkerFlags { }; // The first call of this function must be outside of signal handlers since it -// allocates memory (enforced by `GetWorkerFlagsEarly`). After that it would be +// allocates memory (enforced by `WorkerInitEarly`). After that it would be // signal-safe. // // The worker flags format is `:(NAME=VALUE|SWITCH:)+`. `GetWorkerFlags` @@ -137,10 +139,6 @@ const WorkerFlags& GetWorkerFlags() { return worker_flags; } -__attribute__((constructor(200))) void GetWorkerFlagsEarly() { - (void)GetWorkerFlags(); -} - // `header` should be in the form of `FLAG_NAME=`. // // Extracts "value" as a null-terminated string from "\0FLAG_NAME=value\0" in @@ -237,6 +235,9 @@ constexpr std::string_view kWorkerInputsBlobSequencePathFlagHeader = "arg1="; // TODO: Use better flag names when standardizing the protocol. constexpr std::string_view kWorkerOutputsBlobSequencePathFlagHeader = "arg2="; // TODO: Use better flag names when standardizing the protocol. +constexpr std::string_view kWorkerPersistentModeSocketPathFlagHeader = + "persistent_mode_socket="; // TODO: Use better flag names when + // standardizing the protocol. struct WorkerState { std::atomic has_failure_output = false; @@ -323,6 +324,64 @@ inline std::string_view ToStringView(const std::vector& bytes) { return {reinterpret_cast(bytes.data()), bytes.size()}; } +// Zero initialized. +static int persistent_mode_socket; + +__attribute__((constructor(200))) void WorkerInitEarly() { + const char* persistent_mode_socket_path = + GetWorkerFlag(kWorkerPersistentModeSocketPathFlagHeader); + if (persistent_mode_socket_path == nullptr) return; + persistent_mode_socket = socket(AF_UNIX, SOCK_STREAM, 0); + if (persistent_mode_socket < 0) { + WorkerLog( + "Failed to create persistent mode socket - not running persistent " + "mode.", + LogLnSync{}); + return; + } + + struct sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + const size_t socket_path_len = strlen(persistent_mode_socket_path); + WorkerCheck( + socket_path_len < sizeof(addr.sun_path), + "persistent mode socket path string must be fit in sockaddr_un.sun_path"); + std::memcpy(addr.sun_path, persistent_mode_socket_path, socket_path_len); + + int connect_ret = 0; + do { + connect_ret = + connect(persistent_mode_socket, (struct sockaddr*)&addr, sizeof(addr)); + } while (connect_ret == -1 && errno == EINTR); + if (connect_ret == -1) { + WorkerLog("Failed to connect the persistent mode socket to ", + persistent_mode_socket_path, LogLnSync{}); + (void)close(persistent_mode_socket); + persistent_mode_socket = -1; + } + + int flags = fcntl(persistent_mode_socket, F_GETFD); + if (flags == -1) { + WorkerLog( + "fcntl(F_GETFD) failed on the persistent mode socket - exiting " + "persistent mode", + LogLnSync{}); + (void)close(persistent_mode_socket); + persistent_mode_socket = -1; + } + flags |= FD_CLOEXEC; + if (fcntl(persistent_mode_socket, F_SETFD, flags) == -1) { + WorkerLog( + "fcntl(F_SETFD) failed on the persistent mode socket - exiting " + "persistent mode", + LogLnSync{}); + (void)close(persistent_mode_socket); + persistent_mode_socket = -1; + } + WorkerLog("Persistent mode: connected to ", persistent_mode_socket_path, + LogLnSync{}); +} + BlobSequence* GetInputsBlobSequence() { static auto result = []() -> BlobSequence* { if (!HasWorkerSwitchFlag("shmem")) { @@ -695,6 +754,70 @@ const char* FuzzTestWorkerGetTestName() { return test_name; } +void HandlePersistentMode(const FuzzTestAdapter& adapter) { + auto* inputs_blobseq = GetInputsBlobSequence(); + auto* outputs_blobseq = GetOutputsBlobSequence(); + bool first = true; + while (true) { + PersistentModeRequest req; + if (!ReadAll(persistent_mode_socket, reinterpret_cast(&req), 1)) { + WorkerLog("Failed to read request from persistent mode socket: ", + LogErrNo{}, LogLnSync{}); + return; + } + if (first) { + first = false; + WorkerLog("FuzzTest engine worker enter persistent mode", LogLnSync{}); + } else { + // Reset stdout/stderr. + for (int fd = 1; fd <= 2; fd++) { + lseek(fd, 0, SEEK_SET); + // NOTE: Allow ftruncate() to fail by ignoring its return; that's okay + // to happen when the stdout/stderr are not redirected to a file. + (void)ftruncate(fd, 0); + } + WorkerLog( + "FuzzTest engine worker (", + req == PersistentModeRequest::kExit ? "exiting persistent mode" + : "persistent mode batch", + "); flags: ", + GetWorkerFlags().present + ? std::string_view{GetWorkerFlags().str, GetWorkerFlags().len} + : "", + LogLnSync{}); + } + if (req == PersistentModeRequest::kExit) break; + WorkerCheck(req == PersistentModeRequest::kRunBatch, + "Unknown persistent mode request"); + + inputs_blobseq->Reset(); + outputs_blobseq->Reset(); + + // Read the first blob. It indicates what further actions to take. + auto request_type_blob = inputs_blobseq->Read(); + if (IsMutationRequest(request_type_blob)) { + inputs_blobseq->Reset(); + WorkerDoMutate(adapter); + } else if (IsExecutionRequest(request_type_blob)) { + inputs_blobseq->Reset(); + WorkerDoExecute(adapter); + } else { + WorkerCheck(false, "Unknown shmem request"); + } + + const int result = + GetWorkerState().has_finding.load(std::memory_order_relaxed) + ? EXIT_FAILURE + : EXIT_SUCCESS; + if (!WriteAll(persistent_mode_socket, + reinterpret_cast(&result), sizeof(result))) { + WorkerLog("Failed to write response to the persistent mode socket: ", + LogErrNo{}, LogLnSync{}); + return; + } + } +} + FuzzTestWorkerStatus WorkerRun(const FuzzTestAdapterManager& manager) { const auto& flags = GetWorkerFlags(); WorkerCheck(flags.present, "worker flags must present"); @@ -764,7 +887,9 @@ FuzzTestWorkerStatus WorkerRun(const FuzzTestAdapterManager& manager) { WorkerCheck(adapter.FreeInput != nullptr, "FreeInput must be defined"); WorkerCheck(adapter.FreeCtx != nullptr, "FreeCtx must be defined"); - if (action == WorkerAction::kTestGetSeeds) { + if (persistent_mode_socket > 0) { + HandlePersistentMode(adapter); + } else if (action == WorkerAction::kTestGetSeeds) { WorkerDoGetSeeds(adapter); } else if (action == WorkerAction::kTestMutate) { WorkerDoMutate(adapter); diff --git a/centipede/runner.cc b/centipede/runner.cc index e085b54e..8d553759 100644 --- a/centipede/runner.cc +++ b/centipede/runner.cc @@ -52,6 +52,8 @@ #include "absl/types/span.h" #include "./centipede/byte_array_mutator.h" #include "./centipede/dispatcher_flag_helper.h" +#include "./centipede/engine_abi.h" +#include "./centipede/engine_worker_abi.h" #include "./centipede/execution_metadata.h" #include "./centipede/feature.h" #include "./centipede/mutation_data.h" @@ -78,6 +80,21 @@ struct GlobalRunnerStateManager { GlobalRunnerStateManager state_manager __attribute__((init_priority(200))); +class SpinlockGuard { + public: + SpinlockGuard(std::atomic& lock, bool acquire = true) : lock_(lock) { + if (!acquire) return; + while (lock.exchange(true)) { + pthread_yield(); + } + } + + ~SpinlockGuard() { lock_ = false; } + + private: + std::atomic& lock_; +}; + } // namespace static size_t GetPeakRSSMb() { @@ -322,7 +339,11 @@ static void PrepareCoverage(bool full_clear) { PrepareSancov(full_clear); } -void RunnerCallbacks::GetSeeds(std::function seed_callback) { +void RunnerCallbacks::GetPresetSeedInputs( + std::function seed_callback) {} + +void RunnerCallbacks::GetRandomSeedInput( + std::function seed_callback) { seed_callback({0}); } @@ -488,66 +509,6 @@ static bool FinishSendingOutputsToEngine(BlobSequence &outputs_blobseq) { return true; } -// Handles an ExecutionRequest, see RequestExecution(). Reads inputs from -// `inputs_blobseq`, runs them, saves coverage features to `outputs_blobseq`. -// Returns EXIT_SUCCESS on success and EXIT_FAILURE otherwise. -static int ExecuteInputsFromShmem(BlobSequence &inputs_blobseq, - BlobSequence &outputs_blobseq, - RunnerCallbacks &callbacks) { - size_t num_inputs = 0; - if (!IsExecutionRequest(inputs_blobseq.Read())) return EXIT_FAILURE; - if (!IsNumInputs(inputs_blobseq.Read(), num_inputs)) return EXIT_FAILURE; - - CentipedeBeginExecutionBatch(); - - for (size_t i = 0; i < num_inputs; i++) { - auto blob = inputs_blobseq.Read(); - // TODO(kcc): distinguish bad input from end of stream. - if (!blob.IsValid()) return EXIT_SUCCESS; // no more blobs to read. - if (!IsDataInput(blob)) return EXIT_FAILURE; - - // TODO(kcc): [impl] handle sizes larger than kMaxDataSize. - size_t size = std::min(kMaxDataSize, blob.size); - // Copy from blob to data so that to not pass the shared memory further. - std::vector data(blob.data, blob.data + size); - - // Starting execution of one more input. - if (!StartSendingOutputsToEngine(outputs_blobseq)) break; - - RunOneInput(data.data(), data.size(), callbacks); - - if (state->has_failure_description.load()) break; - - if (!FinishSendingOutputsToEngine(outputs_blobseq)) break; - } - - CentipedeEndExecutionBatch(); - - return state->has_failure_description.load() ? EXIT_FAILURE : EXIT_SUCCESS; -} - -// Dumps seed inputs to `output_dir`. Also see `GetSeedsViaExternalBinary()`. -static void DumpSeedsToDir(RunnerCallbacks &callbacks, const char *output_dir) { - size_t seed_index = 0; - callbacks.GetSeeds([&](ByteSpan seed) { - // Cap seed index within 9 digits. If this was triggered, the dumping would - // take forever.. - if (seed_index >= 1000000000) return; - char seed_path_buf[PATH_MAX]; - const size_t num_path_chars = - snprintf(seed_path_buf, PATH_MAX, "%s/%09lu", output_dir, seed_index); - PrintErrorAndExitIf(num_path_chars >= PATH_MAX, - "seed path reaches PATH_MAX"); - FILE *output_file = fopen(seed_path_buf, "w"); - const size_t num_bytes_written = - fwrite(seed.data(), 1, seed.size(), output_file); - PrintErrorAndExitIf(num_bytes_written != seed.size(), - "wrong number of bytes written for cf table"); - fclose(output_file); - ++seed_index; - }); -} - // Dumps serialized target config to `output_file_path`. Also see // `GetSerializedTargetConfigViaExternalBinary()`. static void DumpSerializedTargetConfigToFile(RunnerCallbacks &callbacks, @@ -566,73 +527,6 @@ static void DumpSerializedTargetConfigToFile(RunnerCallbacks &callbacks, // TODO(kcc): [as-needed] optionally pass an external seed. static unsigned GetRandomSeed() { return time(nullptr); } -// Handles a Mutation Request, see RequestMutation(). -// Mutates inputs read from `inputs_blobseq`, -// writes the mutants to `outputs_blobseq` -// Returns EXIT_SUCCESS on success and EXIT_FAILURE on failure -// so that main() can return its result. -// If both `custom_mutator_cb` and `custom_crossover_cb` are nullptr, -// returns EXIT_FAILURE. -// -// TODO(kcc): [impl] make use of custom_crossover_cb, if available. -static int MutateInputsFromShmem(BlobSequence &inputs_blobseq, - BlobSequence &outputs_blobseq, - RunnerCallbacks &callbacks) { - // Read max_num_mutants. - size_t num_mutants = 0; - size_t num_inputs = 0; - if (!IsMutationRequest(inputs_blobseq.Read())) return EXIT_FAILURE; - if (!IsNumMutants(inputs_blobseq.Read(), num_mutants)) return EXIT_FAILURE; - if (!IsNumInputs(inputs_blobseq.Read(), num_inputs)) return EXIT_FAILURE; - - // Mutation input with ownership. - struct MutationInput { - ByteArray data; - ExecutionMetadata metadata; - }; - // TODO(kcc): unclear if we can continue using std::vector (or other STL) - // in the runner. But for now use std::vector. - // Collect the inputs into a vector. We copy them instead of using pointers - // into shared memory so that the user code doesn't touch the shared memory. - std::vector inputs; - inputs.reserve(num_inputs); - std::vector input_refs; - input_refs.reserve(num_inputs); - for (size_t i = 0; i < num_inputs; ++i) { - // If inputs_blobseq have overflown in the engine, we still want to - // handle the first few inputs. - ExecutionMetadata metadata; - if (!IsExecutionMetadata(inputs_blobseq.Read(), metadata)) { - break; - } - auto blob = inputs_blobseq.Read(); - if (!IsDataInput(blob)) break; - inputs.push_back( - MutationInput{/*data=*/ByteArray{blob.data, blob.data + blob.size}, - /*metadata=*/std::move(metadata)}); - input_refs.push_back( - MutationInputRef{/*data=*/inputs.back().data, - /*metadata=*/&inputs.back().metadata}); - } - - if (!inputs.empty()) { - state->byte_array_mutator->SetMetadata(inputs[0].metadata); - } - - if (!MutationResult::WriteHasCustomMutator(callbacks.HasCustomMutator(), - outputs_blobseq)) { - return EXIT_FAILURE; - } - if (!callbacks.HasCustomMutator()) return EXIT_SUCCESS; - - if (!callbacks.Mutate(input_refs, num_mutants, [&](MutantRef mutant) { - MutationResult::WriteMutant(mutant, outputs_blobseq); - })) { - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} - bool LegacyRunnerCallbacks::Mutate( absl::Span inputs, size_t num_mutants, std::function new_mutant_callback) { @@ -732,57 +626,10 @@ extern void ForkServerCallMeVeryEarly(); [[maybe_unused]] auto fake_reference_for_fork_server = &ForkServerCallMeVeryEarly; -void MaybeConnectToPersistentMode() { - if (state->persistent_mode_socket_path == nullptr) { - return; - } - state->persistent_mode_socket = socket(AF_UNIX, SOCK_STREAM, 0); - if (state->persistent_mode_socket < 0) { - fprintf(stderr, "Failed to create persistent mode socket\n"); - } - - struct sockaddr_un addr{}; - addr.sun_family = AF_UNIX; - const size_t socket_path_len = strlen(state->persistent_mode_socket_path); - RunnerCheck( - socket_path_len < sizeof(addr.sun_path), - "persistent mode socket path string must be fit in sockaddr_un.sun_path"); - std::memcpy(addr.sun_path, state->persistent_mode_socket_path, - socket_path_len); - - int connect_ret = 0; - do { - connect_ret = connect(state->persistent_mode_socket, - (struct sockaddr*)&addr, sizeof(addr)); - } while (connect_ret == -1 && errno == EINTR); - if (connect_ret == -1) { - fprintf(stderr, "Failed to connect the persistent mode socket to %s\n", - state->persistent_mode_socket_path); - (void)close(state->persistent_mode_socket); - state->persistent_mode_socket = -1; - } - - int flags = fcntl(state->persistent_mode_socket, F_GETFD); - if (flags == -1) { - fprintf(stderr, "fcntl(F_GETFD) failed\n"); - (void)close(state->persistent_mode_socket); - state->persistent_mode_socket = -1; - } - flags |= FD_CLOEXEC; - if (fcntl(state->persistent_mode_socket, F_SETFD, flags) == -1) { - fprintf(stderr, "fcntl(F_SETFD) failed\n"); - (void)close(state->persistent_mode_socket); - state->persistent_mode_socket = -1; - } -} - GlobalRunnerState::GlobalRunnerState() { // Make sure fork server is started if needed. ForkServerCallMeVeryEarly(); - // Connecting to the persistent mode socket should be immediately after. - MaybeConnectToPersistentMode(); - SancovRuntimeInitialize(); // TODO(kcc): move some code from CentipedeRunnerMain() here so that it works @@ -812,76 +659,6 @@ void GlobalRunnerState::OnTermination() { } } -static int HandleSharedMemoryRequest(RunnerCallbacks& callbacks, - BlobSequence& inputs_blobseq, - BlobSequence& outputs_blobseq) { - state->has_failure_description = false; - // Read the first blob. It indicates what further actions to take. - auto request_type_blob = inputs_blobseq.Read(); - if (IsMutationRequest(request_type_blob)) { - // Mutation request. - inputs_blobseq.Reset(); - static auto mutator = new ByteArrayMutator(state->knobs, GetRandomSeed()); - state->byte_array_mutator = mutator; - // Since we are mutating, no need to spend time collecting the coverage. - // We still pay for executing the coverage callbacks, but those will - // return immediately. - const int old_traced = CentipedeSetCurrentThreadTraced(/*traced=*/0); - const int result = - MutateInputsFromShmem(inputs_blobseq, outputs_blobseq, callbacks); - CentipedeSetCurrentThreadTraced(old_traced); - return result; - } - if (IsExecutionRequest(request_type_blob)) { - // Execution request. - inputs_blobseq.Reset(); - return ExecuteInputsFromShmem(inputs_blobseq, outputs_blobseq, callbacks); - } - return EXIT_FAILURE; -} - -static int HandlePersistentMode(RunnerCallbacks& callbacks, - BlobSequence& inputs_blobseq, - BlobSequence& outputs_blobseq) { - bool first = true; - while (true) { - PersistentModeRequest req; - if (!ReadAll(state->persistent_mode_socket, reinterpret_cast(&req), - 1)) { - perror("Failed to read request from persistent mode socket"); - return EXIT_FAILURE; - } - if (first) { - first = false; - } else { - // Reset stdout/stderr. - for (int fd = 1; fd <= 2; fd++) { - lseek(fd, 0, SEEK_SET); - // NOTE: Allow ftruncate() to fail by ignoring its return; that's okay - // to happen when the stdout/stderr are not redirected to a file. - (void)ftruncate(fd, 0); - } - fprintf(stderr, "Centipede fuzz target runner (%s); flags: %s\n", - req == PersistentModeRequest::kExit ? "exiting persistent mode" - : "persistent mode batch", - state->flag_helper.flags); - } - if (req == PersistentModeRequest::kExit) break; - RunnerCheck(req == PersistentModeRequest::kRunBatch, - "Unknown persistent mode request"); - const int result = - HandleSharedMemoryRequest(callbacks, inputs_blobseq, outputs_blobseq); - inputs_blobseq.Reset(); - outputs_blobseq.Reset(); - if (!WriteAll(state->persistent_mode_socket, - reinterpret_cast(&result), sizeof(result))) { - perror("Failed to write response to the persistent mode socket"); - return EXIT_FAILURE; - } - } - return EXIT_SUCCESS; -} - // If HasFlag(:shmem:), state->arg1 and state->arg2 are the names // of in/out shared memory locations. // Read inputs and write outputs via shared memory. @@ -901,30 +678,184 @@ int RunnerMain(int argc, char **argv, RunnerCallbacks &callbacks) { return EXIT_SUCCESS; } - if (state->flag_helper.HasFlag(":dump_seed_inputs:")) { - // Seed request. - DumpSeedsToDir(callbacks, /*output_dir=*/sancov_state->arg1); - return EXIT_SUCCESS; - } - - // Inputs / outputs from shmem. - if (state->flag_helper.HasFlag(":shmem:")) { - if (!sancov_state->arg1 || !sancov_state->arg2) return EXIT_FAILURE; - SharedMemoryBlobSequence inputs_blobseq(sancov_state->arg1); - SharedMemoryBlobSequence outputs_blobseq(sancov_state->arg2); - // Persistent mode loop. - if (state->persistent_mode_socket > 0) { - return HandlePersistentMode(callbacks, inputs_blobseq, outputs_blobseq); + struct Input { + ByteArray content; + ExecutionMetadata metadata; + }; + // TODO: move it to ctx. + static bool need_full_cleanup = true; + FuzzTestAdapterManager manager = { + /*ctx=*/reinterpret_cast(&callbacks), + /*GetBinaryId=*/nullptr, + /*GetTestName=*/ + [](FuzzTestAdapterManagerCtx* ctx, const FuzzTestBytesSink* sink) { + // Provide the test name exactly specified from the flag. This should be + // fine as the user of runner should call RunnerMain at most once. + static const char* test_name = + state->flag_helper.GetStringFlag(":test="); + if (test_name == nullptr) return; + static size_t len = strlen(test_name); + const auto bytes = FuzzTestBytesView{ + reinterpret_cast(test_name), + len, + }; + sink->Emit(sink->ctx, &bytes); + }, + /*ConstructAdapter=*/ + [](FuzzTestAdapterManagerCtx* ctx, + const FuzzTestDiagnosticSink* diagnostic_sink, + FuzzTestAdapter* adapter_out) { + { + SpinlockGuard guard(state->diagnostic_sink_spinlock); + state->diagnostic_sink = diagnostic_sink; + } + adapter_out->ctx = reinterpret_cast(ctx); + adapter_out->SetUpCoverageDomains = + [](FuzzTestAdapterCtx* ctx, + const FuzzTestCoverageDomainRegistry* registry) { + SanCovRuntimeSetUpCoverageDomains(registry); + }; + adapter_out->GetPresetSeedInputs = [](FuzzTestAdapterCtx* ctx, + const FuzzTestInputSink* sink) { + auto* callbacks = reinterpret_cast(ctx); + callbacks->GetPresetSeedInputs([&](ByteSpan input) { + sink->Emit( + sink->ctx, + reinterpret_cast(new Input{ + /*content=*/{input.data(), input.data() + input.size()}, + /*metadata=*/{}})); + }); + }; + adapter_out->GetRandomSeedInput = [](FuzzTestAdapterCtx* ctx, + const FuzzTestInputSink* sink) { + auto* callbacks = reinterpret_cast(ctx); + callbacks->GetRandomSeedInput([&](ByteSpan input) { + sink->Emit( + sink->ctx, + reinterpret_cast(new Input{ + /*content=*/{input.data(), input.data() + input.size()}, + /*metadata=*/{}})); + }); + }; + adapter_out->Mutate = [](FuzzTestAdapterCtx* ctx, + FuzzTestInputHandle origin_handle, int shrink, + const FuzzTestInputSink* sink) { + need_full_cleanup = true; + auto* callbacks = reinterpret_cast(ctx); + const auto* origin = reinterpret_cast(origin_handle); + const auto mutation_input = MutationInputRef{ + origin->content, + &origin->metadata, + }; + callbacks->Mutate({&mutation_input, 1}, 1, [&](MutantRef mutant_ref) { + auto* mutant = new Input{ + /*content=*/{mutant_ref.data.data(), + mutant_ref.data.data() + mutant_ref.data.size()}, + /*metadata=*/{}, + }; + sink->Emit(sink->ctx, + reinterpret_cast(mutant)); + }); + }; + adapter_out->Execute = [](FuzzTestAdapterCtx* ctx, + FuzzTestInputHandle handle, + const FuzzTestFeedbackSink* sink) { + auto* callbacks = reinterpret_cast(ctx); + auto* input = reinterpret_cast(handle); + if (need_full_cleanup) { + SanCovRuntimeClearCoverage(true); + need_full_cleanup = false; + } + const int old_traced = CentipedeSetCurrentThreadTraced(/*traced=*/1); + RunOneInput(input->content.data(), input->content.size(), *callbacks); + CentipedeSetCurrentThreadTraced(old_traced); + { + LockGuard lock(state->execution_result_override_mu); + bool has_overridden_execution_result = false; + if (state->execution_result_override != nullptr) { + RunnerCheck( + state->execution_result_override->results().size() <= 1, + "unexpected number of overridden execution results"); + has_overridden_execution_result = + state->execution_result_override->results().size() == 1; + } + if (has_overridden_execution_result) { + auto& result = state->execution_result_override->results()[0]; + SanCovRuntimeConvertToEngineFeatures( + result.mutable_features().data(), + result.mutable_features().size()); + const FuzzTestUint64sView features = { + result.features().data(), + result.features().size(), + }; + sink->EmitCoverageFeatures(sink->ctx, &features); + input->metadata = result.metadata(); + return; + } + } + SanCovRuntimeEmitFeatures(sink); + input->metadata = SanCovRuntimeGetExecutionMetadata(); + }; + adapter_out->DeserializeInputContent = + [](FuzzTestAdapterCtx* ctx, const FuzzTestBytesView* view, + const FuzzTestInputSink* sink) { + auto* input = new Input{ + /*content=*/{view->data, view->data + view->size}, + /*metadata=*/{}, + }; + sink->Emit(sink->ctx, + reinterpret_cast(input)); + }; + adapter_out->UpdateInputMetadata = [](FuzzTestAdapterCtx* ctx, + const FuzzTestBytesView* view, + FuzzTestInputHandle handle) { + auto* input = reinterpret_cast(handle); + input->metadata.cmp_data = {view->data, view->data + view->size}; + }; + adapter_out->SerializeInputContent = [](FuzzTestAdapterCtx* ctx, + FuzzTestInputHandle handle, + const FuzzTestBytesSink* sink) { + auto* input = reinterpret_cast(handle); + const auto input_bytes = FuzzTestBytesView{ + reinterpret_cast(input->content.data()), + input->content.size(), + }; + sink->Emit(sink->ctx, &input_bytes); + }; + adapter_out->SerializeInputMetadata = + [](FuzzTestAdapterCtx* ctx, FuzzTestInputHandle handle, + const FuzzTestBytesSink* sink) { + auto* input = reinterpret_cast(handle); + const auto input_bytes = FuzzTestBytesView{ + reinterpret_cast( + input->metadata.cmp_data.data()), + input->metadata.cmp_data.size(), + }; + sink->Emit(sink->ctx, &input_bytes); + }; + adapter_out->FreeInput = [](FuzzTestAdapterCtx* ctx, + FuzzTestInputHandle handle) { + delete reinterpret_cast(handle); + }; + adapter_out->FreeCtx = [](FuzzTestAdapterCtx* ctx) { + { + SpinlockGuard guard(state->diagnostic_sink_spinlock); + state->diagnostic_sink = nullptr; + } + }; + }, + }; + const int old_traced = CentipedeSetCurrentThreadTraced(/*traced=*/0); + const auto s = FuzzTestWorkerMaybeRun(&manager); + CentipedeSetCurrentThreadTraced(old_traced); + if (s == kFuzzTestWorkerNotRequired) { + // By default, run every input file one-by-one. + for (int i = 1; i < argc; i++) { + ReadOneInputExecuteItAndDumpCoverage(argv[i], callbacks); } - return HandleSharedMemoryRequest(callbacks, inputs_blobseq, - outputs_blobseq); - } - - // By default, run every input file one-by-one. - for (int i = 1; i < argc; i++) { - ReadOneInputExecuteItAndDumpCoverage(argv[i], callbacks); + return EXIT_SUCCESS; } - return EXIT_SUCCESS; + return s == kFuzzTestWorkerSuccess ? EXIT_SUCCESS : EXIT_FAILURE; } } // namespace fuzztest::internal @@ -1044,22 +975,52 @@ extern "C" void CentipedeSetExecutionResult(const uint8_t *data, size_t size) { } extern "C" void CentipedeSetFailureDescription(const char *description) { - using fuzztest::internal::state; - if (state->failure_description_path == nullptr) return; - if (state->has_failure_description.exchange(true)) return; - FILE* f = fopen(state->failure_description_path, "w"); - if (f == nullptr) { - perror("FAILURE: fopen()"); - return; - } - const auto len = strlen(description); - if (fwrite(description, 1, len, f) != len) { - perror("FAILURE: fwrite()"); - } - if (fflush(f) != 0) { - perror("FAILURE: fflush()"); - } - if (fclose(f) != 0) { - perror("FAILURE: fclose()"); + std::string_view desc_sv = description; + static constexpr std::string_view kInputFailurePrefix = "INPUT FAILURE:"; + static constexpr std::string_view kIgnoredFailurePrefix = "IGNORED FAILURE:"; + static constexpr std::string_view kSetupFailurePrefix = "SETUP FAILURE:"; + static constexpr std::string_view kSkippedTestPrefix = "SKIPPED TEST:"; + using ::fuzztest::internal::SpinlockGuard; + using ::fuzztest::internal::state; + if (desc_sv.substr(0, kIgnoredFailurePrefix.size()) == + kIgnoredFailurePrefix || + desc_sv.substr(0, kSkippedTestPrefix.size()) == kSkippedTestPrefix) { + // Nothing to do. + } else if (desc_sv.substr(0, kSetupFailurePrefix.size()) == + kSetupFailurePrefix) { + const bool try_lock_result = + !state->diagnostic_sink_spinlock.exchange(true); + if (try_lock_result) { + SpinlockGuard guard(state->diagnostic_sink_spinlock, /*acquire=*/false); + const auto* diagnostic_sink = state->diagnostic_sink; + const FuzzTestBytesView error = { + reinterpret_cast(desc_sv.data() + + kSetupFailurePrefix.size()), + desc_sv.size() - kSetupFailurePrefix.size(), + }; + diagnostic_sink->EmitError(diagnostic_sink->ctx, &error); + return; + } + std::_Exit(EXIT_FAILURE); + } else { + if (desc_sv.substr(0, kInputFailurePrefix.size()) == kInputFailurePrefix) { + desc_sv = desc_sv.substr(kInputFailurePrefix.size()); + } + const bool try_lock_result = + !state->diagnostic_sink_spinlock.exchange(true); + if (try_lock_result) { + SpinlockGuard guard(state->diagnostic_sink_spinlock, /*acquire=*/false); + const auto* diagnostic_sink = state->diagnostic_sink; + if (diagnostic_sink != nullptr) { + const FuzzTestBytesView finding_desc = { + reinterpret_cast(desc_sv.data()), + desc_sv.size(), + }; + diagnostic_sink->EmitFinding(diagnostic_sink->ctx, &finding_desc, + &finding_desc); + return; + } + } + std::_Exit(EXIT_FAILURE); } } diff --git a/centipede/runner.h b/centipede/runner.h index fbe09416..ae626b51 100644 --- a/centipede/runner.h +++ b/centipede/runner.h @@ -23,6 +23,7 @@ #include "./centipede/byte_array_mutator.h" #include "./centipede/dispatcher_flag_helper.h" +#include "./centipede/engine_abi.h" #include "./centipede/knobs.h" #include "./centipede/runner_interface.h" #include "./centipede/runner_result.h" @@ -118,6 +119,10 @@ struct GlobalRunnerState { // The Watchdog thread sets this to true. std::atomic watchdog_thread_started; + + // Engine diagnostic sink, protected by a spinlock + std::atomic diagnostic_sink_spinlock; + const FuzzTestDiagnosticSink* diagnostic_sink; }; extern ExplicitLifetime state; diff --git a/centipede/runner_interface.h b/centipede/runner_interface.h index c622daf2..28373a56 100644 --- a/centipede/runner_interface.h +++ b/centipede/runner_interface.h @@ -141,9 +141,12 @@ class RunnerCallbacks { // Attempts to execute the test logic using `input`, and returns false if the // input should be ignored from the corpus, true otherwise. virtual bool Execute(ByteSpan input) = 0; - // Generates seed inputs by calling `seed_callback` for each input. + // Provide the preset seed inputs by calling `seed_callback` on each of them. + // The default implementation provides no inputs. + virtual void GetPresetSeedInputs(std::function seed_callback); + // Generates a random seed input by calling `seed_callback` on it. // The default implementation generates a single-byte input {0}. - virtual void GetSeeds(std::function seed_callback); + virtual void GetRandomSeedInput(std::function seed_callback); // Returns the serialized configuration from the test target. The default // implementation returns the empty string. virtual std::string GetSerializedTargetConfig(); diff --git a/centipede/testing/seeded_fuzz_target.cc b/centipede/testing/seeded_fuzz_target.cc index 0530109d..645b0734 100644 --- a/centipede/testing/seeded_fuzz_target.cc +++ b/centipede/testing/seeded_fuzz_target.cc @@ -29,7 +29,8 @@ class SeededRunnerCallbacks : public fuzztest::internal::RunnerCallbacks { return true; } - void GetSeeds(std::function seed_callback) override { + void GetPresetSeedInputs( + std::function seed_callback) override { constexpr size_t kNumAvailSeeds = 10; for (size_t i = 0; i < kNumAvailSeeds; ++i) seed_callback({static_cast(i)}); diff --git a/centipede/testing/test_binary_for_engine_testing.cc b/centipede/testing/test_binary_for_engine_testing.cc index 75c1a42a..071f2232 100644 --- a/centipede/testing/test_binary_for_engine_testing.cc +++ b/centipede/testing/test_binary_for_engine_testing.cc @@ -202,10 +202,10 @@ int main(int argc, char** argv) { return worker_status == kFuzzTestWorkerSuccess ? EXIT_SUCCESS : EXIT_FAILURE; } - return ControllerRun(&manager, {absl::StrCat("--binary=", argv[0]), - "--test_name=some_test", - "--populate_binary_info=0", "--fork_server=0", - "--persistent_mode=0", "--exit_on_crash"}) == + return ControllerRun(&manager, + {absl::StrCat("--binary=", argv[0]), + "--test_name=some_test", "--populate_binary_info=0", + "--fork_server=0", "--exit_on_crash"}) == kFuzzTestControllerSuccess ? EXIT_SUCCESS : EXIT_FAILURE; diff --git a/fuzztest/internal/centipede_adaptor.cc b/fuzztest/internal/centipede_adaptor.cc index 7c23407e..448a37c3 100644 --- a/fuzztest/internal/centipede_adaptor.cc +++ b/fuzztest/internal/centipede_adaptor.cc @@ -531,15 +531,10 @@ class CentipedeAdaptorRunnerCallbacks return false; } - void GetSeeds(std::function seed_callback) - override { + void GetPresetSeedInputs(std::function + seed_callback) override { std::vector seeds = fuzzer_impl_.fixture_driver_->GetSeeds(); - constexpr int kInitialValuesInSeeds = 32; - for (int i = 0; i < kInitialValuesInSeeds; ++i) { - seeds.push_back(fuzzer_impl_.params_domain_.Init(prng_)); - } - absl::c_shuffle(seeds, prng_); for (const auto& seed : seeds) { const auto seed_serialized = SerializeIRObject(fuzzer_impl_.params_domain_.SerializeCorpus(seed)); @@ -547,6 +542,14 @@ class CentipedeAdaptorRunnerCallbacks } } + void GetRandomSeedInput(std::function + seed_callback) override { + auto input = fuzzer_impl_.params_domain_.Init(prng_); + const auto serialized = + SerializeIRObject(fuzzer_impl_.params_domain_.SerializeCorpus(input)); + seed_callback(fuzztest::internal::AsByteSpan(serialized)); + } + std::string GetSerializedTargetConfig() override { return configuration_.Serialize(); }