diff --git a/README.md b/README.md index 4c07853..7a232b3 100644 --- a/README.md +++ b/README.md @@ -6,21 +6,14 @@ This repository hosts all fuzzing efforts for OpenPrinting projects, including f See the `/projects` folder for details on existing integrations of OpenPrinting projects with OSS-Fuzz. -## Parser Fuzzers +## cups-filters libFuzzer Targets -The `/parser-fuzzers` directory contains a standalone parser/filter fuzzing -prototype for OpenPrinting targets. It includes format-aware seed generation, -SMT-assisted template repair, AFL++ handoff scripts, crash triage helpers, and -clone-only Python smoke tests. +The `/parser-fuzzers` directory contains an OSS-Fuzz project for the current +OpenPrinting cups-filters stack. It builds 12 in-process libFuzzer targets with +matching seed corpora, dictionaries, and runtime options. -Start with: - -```bash -cd parser-fuzzers -python3 -m pip install -e . -scripts/run_smoke.sh -scripts/setup_tui.sh --commands -``` +See [`parser-fuzzers/README.md`](parser-fuzzers/README.md) for the target list +and OSS-Fuzz validation commands. ## Contributing diff --git a/parser-fuzzers/.gitignore b/parser-fuzzers/.gitignore deleted file mode 100644 index faee4cc..0000000 --- a/parser-fuzzers/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -__pycache__/ -*.py[cod] -.pytest_cache/ -.mypy_cache/ -.venv/ -*.core -/core -core.* -*.profraw -*.profdata -*.gcda -*.gcno -crashes/ -hangs/ -queue/ -findings/ -work-archives/ -work/* -!work/.gitkeep -!work/corpus/ -work/corpus/* -!work/corpus/smt/ -work/corpus/smt/* -!work/corpus/smt/.gitkeep diff --git a/parser-fuzzers/Dockerfile b/parser-fuzzers/Dockerfile new file mode 100644 index 0000000..a3ed808 --- /dev/null +++ b/parser-fuzzers/Dockerfile @@ -0,0 +1,29 @@ +FROM gcr.io/oss-fuzz-base/base-builder + +RUN sed -i 's|http://|https://|g' /etc/apt/sources.list && \ + apt-get -o Acquire::Retries=8 -o Acquire::https::Timeout=60 update && \ + apt-get -o Acquire::Retries=8 -o Acquire::https::Timeout=60 \ + install -y --no-install-recommends \ + autoconf automake autopoint build-essential cups-common fontconfig \ + fonts-dejavu-core gettext git libavahi-client-dev libcap-dev \ + libcups2-dev libdbus-1-dev libexif-dev libfontconfig1-dev \ + libglib2.0-dev libjpeg-dev libgnutls28-dev liblcms2-dev libnss-mdns \ + libpng-dev libqpdf-dev libsystemd-dev libtiff-dev libtool \ + libwebp-dev libzstd-dev pkg-config patchelf poppler-utils zlib1g-dev && \ + rm -rf /var/lib/apt/lists/* + +RUN git clone --depth 1 https://github.com/OpenPrinting/cups.git \ + $SRC/cups && \ + git clone --depth 1 https://github.com/OpenPrinting/cups-filters.git \ + $SRC/cups-filters && \ + git clone --depth 1 https://github.com/OpenPrinting/libcupsfilters.git \ + $SRC/libcupsfilters && \ + git clone --depth 1 https://github.com/OpenPrinting/libppd.git \ + $SRC/libppd && \ + git clone --depth 1 --recurse-submodules \ + https://github.com/michaelrsweet/pdfio.git $SRC/pdfio + +COPY . $SRC/cupsfilters-core12 +COPY build.sh $SRC/build.sh + +WORKDIR $SRC/cups-filters diff --git a/parser-fuzzers/Makefile b/parser-fuzzers/Makefile deleted file mode 100644 index d57153b..0000000 --- a/parser-fuzzers/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -.PHONY: check-env validate smoke test plan-experiment afl-build-env afl-plan multitarget-monitor explore-monitor arithmetic-explore - -PYTHON ?= python3 - -check-env: - scripts/check_env.sh - -validate: - PYTHONPATH=src $(PYTHON) -m parser_fuzzers.cli validate --bugs bugs --configs configs - -smoke: - scripts/run_smoke.sh - -test: - $(PYTHON) -m unittest discover -s tests - -plan-experiment: - PYTHONPATH=src $(PYTHON) -m parser_fuzzers.cli plan-experiment --configs configs --targets 3 --trials 10 --hours 24 - -afl-build-env: - PYTHONPATH=src $(PYTHON) -m parser_fuzzers.cli afl-build-env --configs configs - -afl-plan: - PYTHONPATH=src $(PYTHON) -m parser_fuzzers.cli afl-prepare --target ppd_ipp_parser --config A1 --binary harnesses/bin/ppd_ipp_parser - -multitarget-monitor: - scripts/run_multitarget_ppd_fuzz.sh 1 4 - -explore-monitor: - scripts/run_explore_ppd_fuzz.sh "" 4 - -arithmetic-explore: - scripts/run_arithmetic_explore.sh 600 4 5 diff --git a/parser-fuzzers/README.md b/parser-fuzzers/README.md index 386d65d..a3d248c 100644 --- a/parser-fuzzers/README.md +++ b/parser-fuzzers/README.md @@ -1,404 +1,53 @@ -# parser-fuzzers +# cups-filters libFuzzer targets -`parser-fuzzers` is a runnable fuzzing toolkit for OpenPrinting parser and -filter paths. It combines format-aware seed/template generation, lightweight -SMT-based field repair, standard AFL++ execution, crash triage, and metrics -export. +This directory is an OSS-Fuzz project for the current OpenPrinting split +stack: CUPS, PDFio, libcupsfilters, libppd, and cups-filters. It contains 12 +in-process libFuzzer targets selected from the earlier parser-fuzzing work. -The current focus is CUPS, cups-filters, libcupsfilters, and libppd parser -paths where random bytes alone often fail early. A useful test case may need a -valid PPD, a compatible document format, coherent raster geometry, and job -options that let the target reach deeper state. +## Layout -```text -format templates + public seeds - -> SMT/fallback field repair - -> generated PPD + document cases - -> runner or standard AFL++ - -> queue/hangs/crash candidates + retained corpus - -> dedup, replay, metrics, optional LLVM coverage -``` - -Private reproducers, issue reports, and minimized crash inputs are not bundled -with this public toolkit. - -## Quick Start - -This path checks the project without AFL++, CUPS, private reproducers, or a local -OpenPrinting build. - -```bash -git clone openprinting-fuzzing -cd openprinting-fuzzing/parser-fuzzers - -python3 -m venv .venv -. .venv/bin/activate -python3 -m pip install -U pip -python3 -m pip install -e . - -scripts/check_env.sh -python3 -m parser_fuzzers.cli validate --bugs bugs --configs configs --allow-missing-local-artifacts -scripts/run_smoke.sh -python3 -m unittest discover -s tests -``` - -Expected result: - -- `validate` exits with zero errors. -- `run_smoke.sh` solves a synthetic branch event, patches one input byte, and - verifies the patched input. -- The unit test suite passes without external CUPS artifacts. - -For a menu-driven local setup: - -```bash -scripts/setup_tui.sh -``` - -To print the same setup commands without entering the menu: - -```bash -scripts/setup_tui.sh --commands -``` - -## Ubuntu Dependencies - -Minimal Python smoke path: - -```bash -scripts/install_ubuntu_deps.sh --minimal -y -``` - -AFL++ and triage tools: - -```bash -scripts/install_ubuntu_deps.sh --afl -y -``` - -System CUPS filter probing: - -```bash -scripts/install_ubuntu_deps.sh --system-filters -y -``` - -The helper reports missing tools clearly. AFL++ is optional for the smoke path. - -## What SMT Does - -SMT is not the fuzzer and it is not symbolic execution of the target program. -It is a constraint-solving helper around the fuzzing pipeline. - -Current roles: - -- Fill typed template slots such as width, height, bits-per-pixel, - bytes-per-line, color model, resolution, page count, and option values. -- Repair related fields after structural mutation so inputs survive shallow - parser checks more often. -- Solve small branch-event JSON constraints and write byte patches for the - `solve-event` / `patch-input` smoke workflow. - -In short: - -```text -template/mutator -> proposes structure and boundary values -SMT -> keeps dependent fields coherent -AFL++ -> evolves bytes from seed directories -runner -> executes real targets and records feedback -``` - -A crash is therefore not automatically "caused by SMT". SMT mainly improves -reachability by making generated inputs coherent enough to reach parser and -filter code that plain random mutation may miss. - -## Repository Layout - -```text -bugs/ optional local-only bug metadata interface -configs/ target and campaign configurations -dictionaries/ AFL++ dictionaries -docs/ architecture, evaluation, and status notes -harnesses/ AFL++ probe and C harness templates -scripts/ runnable workflows -seeds/public/ weak public seeds only -src/ Python package -tests/ unit tests -work/ generated local outputs, ignored by git -``` - -The Python package is named `parser_fuzzers`; the project name and console -script are `parser-fuzzers`. - -## Main Modes - -### 1. Clone-only smoke - -```bash -scripts/run_smoke.sh -python3 -m unittest discover -s tests -``` - -Use this to prove the solver, patcher, validation, and tests are wired. - -### 2. Local system filters - -This uses filters already installed on the system. - -```bash -scripts/check_cups_filters_targets.sh /usr/lib/cups/filter -scripts/run_local_cups_filters_campaign.sh /usr/lib/cups/filter 60 2 5 configs/parser_targets_general.yaml -``` - -Arguments are: - -```text - -``` - -### 3. Template/runner exploration - -This mode generates structured PPD/document cases and executes configured -targets directly. The wrapper asks for the filter root before running so the -campaign does not silently use a hard-coded local path. - -Quick reproducibility check: - -```bash -scripts/run_template_runner_campaign.sh 10 1 2 configs/parser_targets_auto_hybrid.yaml -``` - -Arguments are: - -```text - -``` - -For non-interactive runs, set `SMT_TEMPLATE_FILTER_ROOT=/path/to/cups-filters`. -Increase the duration, workers, and `SMT_TEMPLATE_MAX_RUN_GB` for longer local -campaigns. The runner writes retained cases, timeline data, triage summaries, -and metrics under `work/`. - -### 4. Standard AFL++ from generated seeds - -Build the clone-only AFL++ probe harness: - -```bash -scripts/build_afl_template_probe.sh work/afl/bin/template_probe -``` +- `Dockerfile`, `build.sh`, and `project.yaml` define the OSS-Fuzz project. +- `targets.sh` is the authoritative target and resource-limit list. +- `harnesses/` contains harnesses, shared support, custom mutators, and oracles. +- `corpus/` and `dictionaries/` contain one matching asset per target. -Generate structured seeds without executing any OpenPrinting filter: +## Target set -```bash -PYTHONPATH=src python3 -m parser_fuzzers.cli generate-template-seeds \ - --document-kind pwg_raster_feedback_sweep \ - --target-id template_probe_pwg \ - --output-dir work/afl/seeds \ - --count 64 \ - --extension .pwg -``` - -The seed directory contains AFL++ inputs only. Metadata is written next to it, -for example `work/afl/seeds-template_seed_manifest.json`. - -Alternatively, after a real template/runner campaign, export retained generated -documents into an AFL++ seed directory: - -```bash -template_run="$(find work/template-generate -mindepth 1 -maxdepth 1 -type d | sort | tail -n 1)" -PYTHONPATH=src python3 -m parser_fuzzers.cli export-template-seeds \ - --run-dir "$template_run" \ - --target-id pwg_to_pdf_afl_feedback \ - --extension .pwg \ - --output-dir work/afl/seeds \ - --limit 512 -``` - -Retained-corpus export metadata is also written next to the seed directory, for -example `work/afl/seeds-seed_export_manifest.json`. - -Run AFL++ in the standard way: - -```bash -PYTHONPATH=src python3 -m parser_fuzzers.cli afl-run \ - --target template_probe_pwg \ - --config A1 \ - --binary work/afl/bin/template_probe \ - --input-dir work/afl/seeds \ - --output-dir work/afl/out \ - --duration-sec 60 \ - --timeout-ms 1000 \ - --memory-mb 1024 \ - --execute -``` - -Record standard metrics from the AFL++ output: - -```bash -PYTHONPATH=src python3 -m parser_fuzzers.cli summarize-run-metrics \ - --run-dir work/afl \ - --afl-output-dir work/afl/out \ - --output work/afl/standard_metrics.json -``` - -The generated AFL++ command uses normal `afl-fuzz` arguments: - -```text -afl-fuzz -i -o -t -m [-x dict] [-c cmplog] -- @@ -``` - -Dumb mode is used only when explicitly requested for a non-instrumented target. - -### 5. Template -> AFL++ -> feedback loop - -This combines a real template/runner pass, a standard AFL++ run, and a feedback -template pass. By default it expects the local filter targets from -`configs/parser_targets_afl_pwg_feedback.yaml`; use mode 4 for the clone-only -template seed plus AFL++ probe flow. The script builds the clone-local probe -automatically if it is missing and asks for the template filter root before -starting an interactive run. - -```bash -SMT_TEMPLATE_AFL_WORK_ROOT=work/template-afl-loop \ - PYTHONPATH=src scripts/run_template_afl_loop.sh 180 pwg_to_pdf_afl_feedback .pwg -``` - -Use a larger first argument for real runs; each phase has a 60-second minimum -so very small values still take about three minutes. - -The loop is file-backed: - -```text -template run - -> export AFL++ seeds - -> standard AFL++ run - -> import queue/crashes/hangs/fuzzer_stats - -> build feedback profile - -> next template round -``` - -Outputs include `loop_manifest.json`, per-phase `standard_metrics.json`, and -`loop_standard_metrics.json`. - -### 6. AFL++/ASan cups-filters build - -For isolated AFL++ and ASan runs, build into `work/` instead of a system prefix. -By default this expects local source trees under `/data/pre-gsoc`; override the -paths with `SMT_AFL_*_SRC` variables when needed. - -```bash -scripts/build_afl_cupsfilters_stack.sh -``` - -Then run a real filter loop, for example: - -```bash -bash scripts/run_template_real_afl_loop.sh pwgtopdf 1200 300 -``` - -The build output is written under: - -```text -work/afl-src/ -work/afl-builds/ -work/afl-install/ -work/build-afl-cupsfilters/ -``` - -## Outputs - -A retained generated case usually contains: - -```text -candidate.ppd -document.* -command.txt -meta.json -stderr.txt -stdout.bin -``` - -Common run outputs: - -```text -timeline.jsonl -summary.json -dedup.json -dedup.md -standard_metrics.json -coverage reports, when enabled -``` - -AFL++ outputs are read from standard AFL++ directories such as `queue/`, -`crashes/`, `hangs/`, and `fuzzer_stats`. - -## Metrics - -The project records both fuzzing and research metrics: - -- executions and execs/sec -- AFL++ bitmap coverage and corpus counts -- retained corpus size and semantic feature count -- features per minute/hour -- crash-candidate counts and deduplicated signatures -- hangs/timeouts/skipped cases -- disk usage -- optional LLVM function/line/branch/region coverage - -Crash count alone is not treated as success. A candidate is useful after replay, -deduplication, and source-level triage. - -Representative local measurements from clone-local and local-filter runs: - -| Run | Result | +| Layer | Targets | | --- | --- | -| 30-minute LLVM metrics run | 193369 executions, 7287 semantic features, 5516 retained cases, 31.68% line coverage, 22.58% branch coverage | -| Standard AFL++ PWG-to-PDF-style run | 659225 executions, 245.81 execs/sec, 5.01% AFL++ bitmap coverage | -| Standard AFL++ PWG-to-PCLm-style run | 1299718 executions, 483.39 execs/sec, 3.86% AFL++ bitmap coverage | - -Crash-candidate details and private triage logs should stay out of public pull -requests until they are minimized and reported through the appropriate security -channel. - -## Triage - -Deduplicate and summarize a run: - -```bash -PYTHONPATH=src python3 -m parser_fuzzers.cli dedup-crashes --run-dir -PYTHONPATH=src python3 -m parser_fuzzers.cli summarize-run-metrics --run-dir -``` - -Replay a retained case with an ASan-built filter: - -```bash -scripts/replay_asan_filter.sh imagetops -scripts/replay_asan_filter.sh imagetoraster -scripts/replay_asan_filter.sh pdftoraster -``` - -Run a case under GDB: - -```bash -scripts/gdb_crash_filter.sh -``` - -## Safety And Seed Policy - -- Private reproducers are not placed in `seeds/public/`. -- Generated work stays under `work/`, which is ignored by git. -- Large run artifacts should be archived before deletion. -- Public issue reports should include minimized reproducers and upstream replay - evidence, not raw fuzzing directories. -- The clone-only smoke path should work without private local reports or reproducer - files. - -## Further Reading - -- `docs/architecture.md`: layered codebase map and data-flow boundaries -- `src/parser_fuzzers/README.md`: Python package layout -- `scripts/README.md`: script map -- `configs/README.md`: config map -- `docs/evaluation.md`: experiment metrics and oracle semantics -- `docs/coverage-discovery.md`: coverage-discovery mode -- `docs/branch-events.md`: SMT branch-event and patch schemas -- `docs/oss-fuzz-comparison.md`: comparison notes -- `docs/next-stage-evaluation.md`: next evaluation steps +| Raw formats | CUPS Raster, bounded JPEG, bounded PNG, bounded TIFF | +| State machines | PWG scale up/down, Raster to Apple Raster/PWG Raster | +| Algorithms | PCLX Mode 3 and Mode 10 codecs | +| Semantic/oracle | Text layout and page-selection output oracle | + +The binaries are: + +- `fuzz_cupsfilters_format_cups_raster` +- `fuzz_cupsfilters_format_image_jpeg_bounded` +- `fuzz_cupsfilters_format_image_png_bounded` +- `fuzz_cupsfilters_format_image_tiff_bounded` +- `fuzz_cupsfilters_state_pwg_to_raster_scale_down` +- `fuzz_cupsfilters_state_pwg_to_raster_scale_up` +- `fuzz_cupsfilters_state_raster_to_apple` +- `fuzz_cupsfilters_state_raster_to_pwg` +- `fuzz_cupsfilters_raster_to_pclx_mode3_codec` +- `fuzz_cupsfilters_raster_to_pclx_mode10_codec` +- `fuzz_cupsfilters_state_text_to_text_layout` +- `fuzz_cupsfilters_text_to_text_selection_oracle` + +The resource limits are defined in `targets.sh`. Every binary has one seed +corpus, one dictionary, and one `.options` file generated by `build.sh`. + +## OSS-Fuzz check + +Copy this directory to `projects/cups-filters` in an OSS-Fuzz checkout, then +run: + +```sh +python3 infra/helper.py build_image cups-filters +python3 infra/helper.py build_fuzzers --sanitizer address cups-filters +python3 infra/helper.py check_build --sanitizer address cups-filters +``` + +The harnesses execute in process and do not invoke standalone filters, +Ghostscript, Poppler, or MuPDF. diff --git a/parser-fuzzers/bugs/README.md b/parser-fuzzers/bugs/README.md deleted file mode 100644 index 0677737..0000000 --- a/parser-fuzzers/bugs/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Optional Local Bug Metadata - -This directory is intentionally empty in the public tree. - -Private reproducers, issue reports, minimized crashing inputs, and local -triage notes should not be committed here. If a local evaluation needs -ground-truth metadata, create private `bugs//meta.yaml` files outside the -public review flow and keep `known_poc_allowed_in_seed: false`. diff --git a/parser-fuzzers/build.sh b/parser-fuzzers/build.sh new file mode 100755 index 0000000..2742ad8 --- /dev/null +++ b/parser-fuzzers/build.sh @@ -0,0 +1,261 @@ +#!/bin/bash +set -euxo pipefail + +ROOT="$SRC/cupsfilters-core12" +HARNESS_ROOT="$ROOT/harnesses" +source "$ROOT/targets.sh" + +PREFIX="$WORK/cupsfilters-core12-prefix" +FUZZERS="$WORK/cupsfilters-core12-fuzzers" +JOBS="${JOBS:-$(nproc)}" +CUPS_PREFIX="$PREFIX/cups" +PDFIO_PREFIX="$PREFIX/pdfio" +LIBCUPSFILTERS_PREFIX="$PREFIX/libcupsfilters" +LIBPPD_PREFIX="$PREFIX/libppd" + +export CFLAGS="${CFLAGS} -O1 -g -fno-omit-frame-pointer -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION" +export CXXFLAGS="${CXXFLAGS} -O1 -g -fno-omit-frame-pointer -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION" +case "$CFLAGS" in + *-fsanitize=fuzzer-no-link*) ;; + *) + export CFLAGS="${CFLAGS} -fsanitize=fuzzer-no-link" + export CXXFLAGS="${CXXFLAGS} -fsanitize=fuzzer-no-link" + ;; +esac +ORIGINAL_LDFLAGS="${LDFLAGS:-}" +FUZZER_CFLAGS="$CFLAGS" +FUZZER_CXXFLAGS="$CXXFLAGS" +export LDFLAGS="${ORIGINAL_LDFLAGS} ${CFLAGS}" +export CPPFLAGS="${CPPFLAGS:-} -I$CUPS_PREFIX/include -I$PDFIO_PREFIX/include" +mkdir -p "$PREFIX" "$FUZZERS" "$OUT" + +build_autotools() { + local source="$1" prefix="$2" + shift 2 + cd "$source" + [[ -x ./configure ]] || ./autogen.sh + make distclean >/dev/null 2>&1 || true + ./configure --prefix="$prefix" --enable-static --disable-shared "$@" + make -j"$JOBS" + make install +} + +cd "$SRC/cups" +make distclean >/dev/null 2>&1 || true +./configure --prefix="$CUPS_PREFIX" --libdir="$CUPS_PREFIX/lib" \ + --enable-static --disable-shared +make -j1 -C cups libcups.a +make install-headers +make -C cups install-libs +mkdir -p "$CUPS_PREFIX/lib/pkgconfig" +install -m 0644 cups.pc "$CUPS_PREFIX/lib/pkgconfig/cups.pc" + +export PKG_CONFIG_PATH="$CUPS_PREFIX/lib/pkgconfig" +[[ "$(pkg-config --variable=prefix cups)" == "$CUPS_PREFIX" ]] + +cd "$SRC/pdfio" +./configure --prefix="$PDFIO_PREFIX" --enable-static --disable-shared +make -j"$JOBS" +make install + +export PKG_CONFIG_PATH="$CUPS_PREFIX/lib/pkgconfig:$PDFIO_PREFIX/lib/pkgconfig" +build_autotools "$SRC/libcupsfilters" "$LIBCUPSFILTERS_PREFIX" \ + --without-jpegxl --disable-exif --disable-poppler --disable-dbus \ + --disable-ghostscript --disable-mutool + +export PKG_CONFIG_PATH="$LIBCUPSFILTERS_PREFIX/lib/pkgconfig:$CUPS_PREFIX/lib/pkgconfig:$PDFIO_PREFIX/lib/pkgconfig" +build_autotools "$SRC/libppd" "$LIBPPD_PREFIX" \ + --disable-ghostscript --disable-pdftops --disable-mutool \ + --disable-acroread --with-pdftops=pdftocairo + +export PKG_CONFIG_PATH="$LIBPPD_PREFIX/lib/pkgconfig:$LIBCUPSFILTERS_PREFIX/lib/pkgconfig:$CUPS_PREFIX/lib/pkgconfig:$PDFIO_PREFIX/lib/pkgconfig" + +cd "$SRC/cups-filters" +[[ -x ./configure ]] || ./autogen.sh +make distclean >/dev/null 2>&1 || true +./configure --enable-individual-cups-filters --disable-universal-cups-filter \ + --disable-foomatic --disable-driverless --disable-ghostscript \ + --disable-mutool --enable-static --disable-shared + +export CFLAGS="$FUZZER_CFLAGS" +export CXXFLAGS="$FUZZER_CXXFLAGS" +export LDFLAGS="$ORIGINAL_LDFLAGS" + +read -r -a PKG_CFLAGS <<< "$(pkg-config --cflags libcupsfilters libppd pdfio cups)" +read -r -a DEP_LIBS <<< "$(pkg-config --libs --static libcupsfilters pdfio cups)" +read -r -a EXTRA_LIBS <<< "$(pkg-config --libs --static libjpeg libexif libqpdf libtiff-4 libpng fontconfig lcms2)" + +FILTERED_DEP_LIBS=() +for library in "${DEP_LIBS[@]}"; do + [[ "$library" == -lcupsfilters ]] || FILTERED_DEP_LIBS+=("$library") +done + +COMMON_CFLAGS=( + -I"$HARNESS_ROOT" + -I"$SRC/cups-filters" + -I"$SRC/cups-filters/filter" + -I"$SRC/libcupsfilters" + -I"$SRC/libcupsfilters/cupsfilters" + -I"$SRC/libppd" + -I"$LIBPPD_PREFIX/include" + "${PKG_CFLAGS[@]}" +) +COMMON_LIBS=( + "$LIBPPD_PREFIX/lib/libppd.a" + "$LIBCUPSFILTERS_PREFIX/lib/libcupsfilters.a" + "${FILTERED_DEP_LIBS[@]}" + "${EXTRA_LIBS[@]}" + -lstdc++ + -Wl,--allow-multiple-definition +) + +if [[ "${SANITIZER:-}" == coverage ]]; then + LSAN_COVERAGE_STUB="$FUZZERS/lsan_coverage_stubs.o" + "$CC" $CFLAGS -c "$HARNESS_ROOT/lsan_coverage_stubs.c" \ + -o "$LSAN_COVERAGE_STUB" + COMMON_LIBS+=("$LSAN_COVERAGE_STUB") +fi + +build_harness() { + local output="$1" source="$2" + shift 2 + "$CC" $CFLAGS "${COMMON_CFLAGS[@]}" "$@" "$source" \ + "${COMMON_LIBS[@]}" $LIB_FUZZING_ENGINE -o "$FUZZERS/$output" +} + +compact_state_mutator() { + local prefix_size="$1" selector_size="$2" min_payload="$3" + printf '%s\n' \ + "-DCF_FUZZ_STATE_PREFIX_SIZE=$prefix_size" \ + "-DCF_FUZZ_STATE_SELECTOR_SIZE=$selector_size" \ + "-DCF_FUZZ_STATE_MIN_PAYLOAD=$min_payload" \ + "$HARNESS_ROOT/compact_state_mutator.c" +} + +mapfile -t PWG_SCALE_MUTATOR < <(compact_state_mutator 7 8 0) +mapfile -t RASTER_OUTPUT_MUTATOR < <(compact_state_mutator 8 8 0) +mapfile -t PCLX_CODEC_MUTATOR < <(compact_state_mutator 8 12 1) +mapfile -t TEXT_LAYOUT_MUTATOR < <(compact_state_mutator 8 16 1) + +build_harness fuzz_cupsfilters_format_cups_raster \ + "$HARNESS_ROOT/fuzz_cups_raster_reader.c" \ + -DCUPS_RASTER_READER_MAX_INPUT=4194304 + +build_harness fuzz_cupsfilters_format_image_jpeg_bounded \ + "$HARNESS_ROOT/fuzz_cupsfilters_image_codec.c" \ + -DCUPSFILTERS_IMAGE_CODEC_JPEG -DCUPSFILTERS_IMAGE_CODEC_MAX_INPUT=2097152 \ + -Wl,--wrap=jpeg_std_error + +build_harness fuzz_cupsfilters_format_image_png_bounded \ + "$HARNESS_ROOT/fuzz_png_bounded_format.c" \ + -DCUPSFILTERS_IMAGE_CODEC_PNG -DCUPSFILTERS_IMAGE_CODEC_MAX_INPUT=2097152 + +build_harness fuzz_cupsfilters_format_image_tiff_bounded \ + "$HARNESS_ROOT/fuzz_cupsfilters_image_codec.c" \ + -DCUPSFILTERS_IMAGE_CODEC_TIFF -DCUPSFILTERS_IMAGE_CODEC_MAX_INPUT=2097152 \ + -Wl,--wrap=TIFFFdOpen -Wl,--wrap=TIFFReadScanline + +for direction in down up; do + build_harness "fuzz_cupsfilters_state_pwg_to_raster_scale_$direction" \ + "$HARNESS_ROOT/fuzz_pwg_scale_state.c" \ + "-DCF_FUZZ_PWG_SCALE_${direction^^}" "${PWG_SCALE_MUTATOR[@]}" +done + +for dialect in apple pwg; do + if [[ "$dialect" == apple ]]; then + output_mime=image/urf + else + output_mime=image/pwg-raster + fi + build_harness "fuzz_cupsfilters_state_raster_to_$dialect" \ + "$HARNESS_ROOT/fuzz_raster_output_state.c" \ + -DCF_FUZZ_FILTER_FUNCTION=cfFilterRasterToPWG \ + -DCF_FUZZ_TARGET_NAME="\"fuzz_cupsfilters_state_raster_to_$dialect\"" \ + -DCF_FUZZ_INPUT_MIME="\"application/vnd.cups-raster\"" \ + -DCF_FUZZ_OUTPUT_MIME="\"$output_mime\"" \ + "${RASTER_OUTPUT_MUTATOR[@]}" +done + +for mode in 3 10; do + build_harness "fuzz_cupsfilters_raster_to_pclx_mode${mode}_codec" \ + "$HARNESS_ROOT/fuzz_rastertopclx_compress_state.c" \ + -DCF_FUZZ_RASTERTOPCLX_SOURCE="\"$SRC/cups-filters/filter/rastertopclx.c\"" \ + "-DCF_FUZZ_PCLX_MODE${mode}_CODEC" \ + "$SRC/cups-filters/filter/pcl-common.c" \ + "${PCLX_CODEC_MUTATOR[@]}" +done + +build_harness fuzz_cupsfilters_state_text_to_text_layout \ + "$HARNESS_ROOT/fuzz_text_to_text_state.c" \ + -DCF_FUZZ_FILTER_FUNCTION=cfFilterTextToText \ + -DCF_FUZZ_TARGET_NAME='"fuzz_cupsfilters_state_text_to_text_layout"' \ + -DCF_FUZZ_INPUT_MIME='"text/plain"' \ + -DCF_FUZZ_OUTPUT_MIME='"text/plain"' \ + -DCF_FUZZ_TEXTTOTEXT_STATE_OPTIONS \ + "${TEXT_LAYOUT_MUTATOR[@]}" + +build_harness fuzz_cupsfilters_text_to_text_selection_oracle \ + "$HARNESS_ROOT/fuzz_text_to_text_page_order.c" \ + -DCF_FUZZ_FILTER_FUNCTION=cfFilterTextToText \ + -DCF_FUZZ_TARGET_NAME='"fuzz_cupsfilters_text_to_text_selection_oracle"' \ + -DCF_FUZZ_INPUT_MIME='"text/plain"' \ + -DCF_FUZZ_OUTPUT_MIME='"text/plain"' \ + -DCF_FUZZ_TEXTTOTEXT_STATE_OPTIONS \ + "${TEXT_LAYOUT_MUTATOR[@]}" + +for target in "${CF_CORE12_TARGETS[@]}"; do + [[ -x "$FUZZERS/$target" ]] || { + echo "core12 build did not produce $target" >&2 + exit 2 + } +done + +# helper.py preserves $OUT between builds. Remove this project's prior target +# artifacts so a renamed or removed target cannot survive an incremental build. +find "$OUT" -mindepth 1 -maxdepth 1 -name 'fuzz_*cupsfilters_*' \ + -exec rm -rf -- {} + +rm -f "$OUT"/*.so "$OUT"/*.so.* "$OUT/fonts.conf" +rm -rf "$OUT/cups-data" "$OUT/fonts" +mkdir -p "$OUT/cups-data/data" "$OUT/fonts" +cp -a /usr/share/cups/. "$OUT/cups-data/" 2>/dev/null || true +cp -a "$SRC/libcupsfilters/data/"*.pdf "$OUT/cups-data/data/" +cp -a /usr/share/fonts/truetype/dejavu/. "$OUT/fonts/" +cp "$ROOT/fonts.conf" "$OUT/fonts.conf" + +for target in "${CF_CORE12_TARGETS[@]}"; do + install -m 0755 "$FUZZERS/$target" "$OUT/$target" + cp "$ROOT/corpus/${target}_seed_corpus.zip" "$OUT/" + cp "$ROOT/dictionaries/${target}.dict" "$OUT/" + printf '[libfuzzer]\nmax_len=%s\ntimeout=%s\nrss_limit_mb=%s\ndetect_leaks=%s\n' \ + "$(core12_target_max_len "$target")" \ + "$(core12_target_timeout_sec "$target")" \ + "$(core12_target_rss_limit_mb "$target")" \ + "$(core12_target_detect_leaks "$target")" \ + >"$OUT/${target}.options" + patchelf --set-rpath '$ORIGIN' "$OUT/$target" +done + +is_system_runtime() { + case "$(basename "$1")" in + ld-linux*|libanl.so.*|libc.so.*|libdl.so.*|libgcc_s.so.*|libm.so.*|\ + libmemusage.so.*|libnsl.so.*|libnss_*.so.*|libpthread.so.*|\ + libresolv.so.*|librt.so.*|libstdc++.so.*|libthread_db.so.*|libutil.so.*) + return 0 ;; + *) return 1 ;; + esac +} + +for target in "${CF_CORE12_TARGETS[@]}"; do + while read -r library; do + [[ -z "$library" || "$library" == "$OUT/"* ]] && continue + is_system_runtime "$library" && continue + cp -L "$library" "$OUT/" + done < <(ldd "$OUT/$target" | awk '/=> \/[^ ]+/ { print $3 }') +done + +for library in "$OUT"/*.so "$OUT"/*.so.*; do + [[ -f "$library" ]] || continue + patchelf --set-rpath '$ORIGIN' "$library" +done + +echo "Exported ${#CF_CORE12_TARGETS[@]} cups-filters core targets to $OUT" diff --git a/parser-fuzzers/configs/README.md b/parser-fuzzers/configs/README.md deleted file mode 100644 index d21a695..0000000 --- a/parser-fuzzers/configs/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Config Map - -Configs define targets, experiment rows, and campaign profiles. - -## Core Configs - -- `experiment.yaml`: A0-A4 experiment matrix. -- `targets.yaml`: generic target metadata and dictionaries. -- `afl.yaml`: AFL++ command defaults. -- `parser_targets.yaml`: initial multi-target PPD/document config. - -## General Parser Campaigns - -- `parser_targets_general.yaml`: broad low-direction parser targets. -- `parser_targets_coverage.yaml`: coverage-discovery targets. -- `parser_targets_auto_hybrid.yaml`: current hybrid automatic exploration. -- `parser_targets_cold_semantic.yaml`: cold parser semantic expansion. - -## Image Campaigns - -- `parser_targets_image_feedback.yaml`: broad image feedback targets. -- `parser_targets_image_imagetopdf_feedback.yaml`: imagetopdf focus. -- `parser_targets_image_imagetops_feedback.yaml`: imagetops focus. -- `parser_targets_image_imagetoraster_feedback.yaml`: imagetoraster focus. - -## AFL++ Feedback Campaigns - -- `parser_targets_afl_pwg_feedback.yaml`: PWG AFL++ feedback targets. -- `parser_targets_afl_pwg_pdf_deep.yaml`: AFL++-seeded PDF/PWG deep target. - -## Underexplored Parser Campaigns - -- `parser_targets_underexplored.yaml`: underexplored parser sweep. -- `parser_targets_underexplored_feedback30.yaml`: 30-minute feedback variant. -- `parser_targets_underexplored_semantic.yaml`: semantic variant. - -## Older Or Focused Profiles - -- `parser_targets_explore.yaml`: earlier exploration profile. -- `parser_targets_feedback.yaml`: earlier feedback profile. -- `parser_targets_structural.yaml`: structural template profile. - -Generated local coverage configs under `work/` are intentionally not tracked. diff --git a/parser-fuzzers/configs/afl.yaml b/parser-fuzzers/configs/afl.yaml deleted file mode 100644 index 6a2f25f..0000000 --- a/parser-fuzzers/configs/afl.yaml +++ /dev/null @@ -1,16 +0,0 @@ -afl: - fuzzer: afl-fuzz - compiler_cc: afl-clang-fast - compiler_cxx: afl-clang-fast++ - input_seed_dir: seeds/public - smt_corpus_dir: work/corpus/smt - work_dir: work/afl - output_dir: work/afl/out - timeout_ms: 3000 - memory_mb: 1024 - cmplog_binary_suffix: .cmplog - custom_mutator_library: "" - env: - AFL_NO_UI: "1" - AFL_SKIP_CPUFREQ: "1" - AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: "1" diff --git a/parser-fuzzers/configs/experiment.yaml b/parser-fuzzers/configs/experiment.yaml deleted file mode 100644 index b91c4f4..0000000 --- a/parser-fuzzers/configs/experiment.yaml +++ /dev/null @@ -1,46 +0,0 @@ -experiment: - name: cups-smt-fuzz-evaluation - default_targets: 3 - smoke: - repetitions: 5 - hours: 6 - main: - repetitions: 10 - hours: 24 - -fuzz_configs: - - id: A0 - name: vanilla - description: AFL++ baseline with no dictionaries, grammar, CmpLog, or SMT patching. - dictionary: false - cmplog: false - grammar: false - smt: false - - id: A1 - name: dictionary - description: Baseline plus CUPS/IPP/PPD/MIME/image dictionaries. - dictionary: true - cmplog: false - grammar: false - smt: false - - id: A2 - name: dictionary+cmplog - description: Dictionary baseline plus AFL++ CmpLog/Redqueen support. - dictionary: true - cmplog: true - grammar: false - smt: false - - id: A3 - name: dictionary+grammar - description: Dictionary baseline plus structured grammar/custom mutator. - dictionary: true - cmplog: false - grammar: true - smt: false - - id: A4 - name: dictionary+grammar+smt - description: Grammar baseline plus SMT-generated patch inputs. - dictionary: true - cmplog: false - grammar: true - smt: true diff --git a/parser-fuzzers/configs/parser_targets.yaml b/parser-fuzzers/configs/parser_targets.yaml deleted file mode 100644 index 6e10c97..0000000 --- a/parser-fuzzers/configs/parser_targets.yaml +++ /dev/null @@ -1,49 +0,0 @@ -targets: - - id: ppd_text_to_rastertopclx_smoke - description: Generic PPD string-value variation through cupsfilter into rastertopclx. - ppd_kind: rastertopclx_general_strings - document_kind: text - executor: cupsfilter - input_mime: text/plain - output_mime: printer/foo - expected_filters: - - universal - - rastertopclx - cases: 8 - oracle: crash_or_signal - - - id: cups_raster_to_rastertoescpx_smoke - description: Generic CUPS Raster smoke path into rastertoescpx. - ppd_kind: rastertoescpx_single_pagesize - document_kind: cups_raster_basic - executor: cupsfilter - input_mime: application/vnd.cups-raster - output_mime: printer/foo - expected_filters: - - rastertoescpx - cases: 2 - oracle: reached_only - - - id: cups_raster_to_rastertopclx_smoke - description: CUPS Raster RGB path reaches rastertopclx parser code. - ppd_kind: rastertopclx_plain - document_kind: cups_raster_mode10 - executor: direct_filter - filter_binary: /usr/lib/cups/filter/rastertopclx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertopclx - cases: 2 - oracle: reached_only - - - id: pwg_to_raster_resolution_smoke - description: PWG Raster resolution and geometry stress path into pwgtoraster. - ppd_kind: pwgtoraster_1dpi - document_kind: pwg_raster_resolution_stress - executor: direct_filter - filter_binary: /usr/lib/cups/filter/pwgtoraster - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtoraster - cases: 2 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_afl_pwg_feedback.yaml b/parser-fuzzers/configs/parser_targets_afl_pwg_feedback.yaml deleted file mode 100644 index cb75ac1..0000000 --- a/parser-fuzzers/configs/parser_targets_afl_pwg_feedback.yaml +++ /dev/null @@ -1,24 +0,0 @@ -targets: - - id: pwg_to_pdf_afl_feedback - description: SMT PWG Raster feedback sweep seeded from AFL++ queue/crash artifacts into ASan-built pwgtopdf. - ppd_kind: pwgtopdf_coverage_options - document_kind: pwg_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopdf - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopdf - cases: 128 - oracle: crash_or_signal - - - id: pwg_to_pclm_afl_feedback - description: SMT PWG Raster feedback sweep seeded from AFL++ queue/crash artifacts into ASan-built pwgtopclm. - ppd_kind: pwgtopclm_coverage_options - document_kind: pwg_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopclm - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopclm - cases: 128 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_afl_pwg_pdf_deep.yaml b/parser-fuzzers/configs/parser_targets_afl_pwg_pdf_deep.yaml deleted file mode 100644 index 4c29735..0000000 --- a/parser-fuzzers/configs/parser_targets_afl_pwg_pdf_deep.yaml +++ /dev/null @@ -1,13 +0,0 @@ -targets: - - id: pwg_to_pdf_afl_deep - description: Depth-oriented SMT PWG Raster feedback sweep seeded from AFL++ artifacts into ASan-built pwgtopdf with captured PDF output. - ppd_kind: pwgtopdf_coverage_options - document_kind: pwg_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopdf - input_mime: application/vnd.cups-pwg - output_mime: application/pdf - expected_filters: - - pwgtopdf - cases: 160 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_auto_hybrid.yaml b/parser-fuzzers/configs/parser_targets_auto_hybrid.yaml deleted file mode 100644 index 0be9f8c..0000000 --- a/parser-fuzzers/configs/parser_targets_auto_hybrid.yaml +++ /dev/null @@ -1,240 +0,0 @@ -targets: - - id: cups_raster_to_rastertopclx_feedback - description: Feedback-driven SMT CUPS Raster sweep into ASan-built rastertopclx. - ppd_kind: raster_coverage_options - document_kind: cups_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertopclx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertopclx - cases: 24 - oracle: crash_or_signal - - - id: cups_raster_to_rastertoescpx_feedback - description: Feedback-driven SMT CUPS Raster sweep into ASan-built rastertoescpx. - ppd_kind: raster_coverage_options - document_kind: cups_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertoescpx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertoescpx - cases: 24 - oracle: crash_or_signal - - - id: cups_raster_to_rastertops_feedback - description: Feedback-driven SMT CUPS Raster sweep into ASan-built rastertops. - ppd_kind: raster_coverage_options - document_kind: cups_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertops - input_mime: application/vnd.cups-raster - expected_filters: - - rastertops - cases: 24 - oracle: crash_or_signal - - - id: pwg_to_raster_feedback - description: Feedback-driven SMT PWG Raster sweep into ASan-built pwgtoraster. - ppd_kind: pwg_resolution_coverage - document_kind: pwg_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtoraster - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtoraster - cases: 24 - oracle: crash_or_signal - - - id: pwg_to_pdf_feedback - description: Feedback-driven SMT PWG Raster sweep into ASan-built pwgtopdf. - ppd_kind: pwgtopdf_coverage_options - document_kind: pwg_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopdf - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopdf - cases: 24 - oracle: crash_or_signal - - - id: pwg_to_pclm_feedback - description: Feedback-driven SMT PWG Raster sweep into ASan-built pwgtopclm. - ppd_kind: pwgtopclm_coverage_options - document_kind: pwg_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopclm - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopclm - cases: 24 - oracle: crash_or_signal - - - id: pdf_to_pdftopdf_coverage - description: Coverage-discovery PDF parser sweep into ASan-built pdftopdf. - ppd_kind: pdftopdf_coverage_options - document_kind: pdf_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftopdf - input_mime: application/pdf - expected_filters: - - pdftopdf - cases: 8 - oracle: crash_or_signal - - - id: pdf_to_pdftops_coverage - description: Coverage-discovery PDF parser sweep into ASan-built pdftops. - ppd_kind: pdftops_coverage_options - document_kind: pdf_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftops - input_mime: application/pdf - expected_filters: - - pdftops - cases: 8 - oracle: crash_or_signal - - - id: pdf_to_pdftoraster_coverage - description: Coverage-discovery PDF parser sweep into ASan-built pdftoraster. - ppd_kind: pdftoraster_coverage_options - document_kind: pdf_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftoraster - input_mime: application/pdf - expected_filters: - - pdftoraster - cases: 8 - oracle: crash_or_signal - - - id: pdf_to_mupdftopwg_coverage - description: Coverage-discovery PDF parser sweep into ASan-built mupdftopwg. - ppd_kind: mupdftopwg_coverage_options - document_kind: pdf_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/mupdftopwg - input_mime: application/pdf - expected_filters: - - mupdftopwg - cases: 8 - oracle: crash_or_signal - - - id: image_to_imagetoraster_coverage - description: Coverage-discovery image parser sweep into ASan-built imagetoraster. - ppd_kind: imagetoraster_coverage_options - document_kind: image_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetoraster - input_mime: image/x-portable-anymap - expected_filters: - - imagetoraster - cases: 12 - oracle: crash_or_signal - - - id: image_to_imagetopdf_coverage - description: Coverage-discovery image parser sweep into ASan-built imagetopdf. - ppd_kind: imagetopdf_coverage_options - document_kind: image_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetopdf - input_mime: image/x-portable-anymap - expected_filters: - - imagetopdf - cases: 12 - oracle: crash_or_signal - - - id: image_to_imagetops_coverage - description: Coverage-discovery image parser sweep into ASan-built imagetops. - ppd_kind: imagetops_coverage_options - document_kind: image_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetops - input_mime: image/x-portable-anymap - expected_filters: - - imagetops - cases: 12 - oracle: crash_or_signal - - - id: text_to_texttopdf_coverage - description: Coverage-discovery text parser sweep into ASan-built texttopdf. - ppd_kind: texttopdf_coverage_options - document_kind: text_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/texttopdf - input_mime: text/plain - expected_filters: - - texttopdf - cases: 10 - oracle: crash_or_signal - - - id: text_to_texttotext_coverage - description: Coverage-discovery text parser sweep into ASan-built texttotext. - ppd_kind: texttotext_coverage_options - document_kind: text_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/texttotext - input_mime: text/plain - expected_filters: - - texttotext - cases: 10 - oracle: crash_or_signal - - - id: postscript_to_gstoraster_coverage - description: Coverage-discovery PostScript/Ghostscript parser sweep into ASan-built gstoraster wrapper. - ppd_kind: gstoraster_coverage_options - document_kind: postscript_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstoraster - input_mime: application/postscript - expected_filters: - - gstoraster - cases: 8 - oracle: crash_or_signal - - - id: postscript_to_gstopdf_coverage - description: Coverage-discovery PostScript/Ghostscript parser sweep into ASan-built gstopdf wrapper. - ppd_kind: gstopdf_coverage_options - document_kind: postscript_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstopdf - input_mime: application/postscript - expected_filters: - - gstopdf - cases: 8 - oracle: crash_or_signal - - - id: postscript_to_gstopxl_coverage - description: Coverage-discovery PostScript/Ghostscript parser sweep into ASan-built gstopxl wrapper. - ppd_kind: gstopxl_coverage_options - document_kind: postscript_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstopxl - input_mime: application/postscript - expected_filters: - - gstopxl - cases: 8 - oracle: crash_or_signal - - - id: command_to_escpx_coverage - description: Coverage-discovery CUPS command parser sweep into ASan-built commandtoescpx. - ppd_kind: commandtoescpx_coverage_options - document_kind: command_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/commandtoescpx - input_mime: application/vnd.cups-command - expected_filters: - - commandtoescpx - cases: 10 - oracle: crash_or_signal - - - id: command_to_pclx_coverage - description: Coverage-discovery CUPS command parser sweep into ASan-built commandtopclx. - ppd_kind: commandtopclx_coverage_options - document_kind: command_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/commandtopclx - input_mime: application/vnd.cups-command - expected_filters: - - commandtopclx - cases: 10 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_cold_semantic.yaml b/parser-fuzzers/configs/parser_targets_cold_semantic.yaml deleted file mode 100644 index 79a209f..0000000 --- a/parser-fuzzers/configs/parser_targets_cold_semantic.yaml +++ /dev/null @@ -1,168 +0,0 @@ -targets: - - id: pdf_to_pdftopdf_semantic - description: Semantic PDF parser sweep into ASan-built pdftopdf. - ppd_kind: pdftopdf_coverage_options - document_kind: pdf_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftopdf - input_mime: application/pdf - expected_filters: - - pdftopdf - cases: 64 - oracle: crash_or_signal - - - id: pdf_to_pdftops_semantic - description: Semantic PDF parser sweep into ASan-built pdftops. - ppd_kind: pdftops_coverage_options - document_kind: pdf_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftops - input_mime: application/pdf - expected_filters: - - pdftops - cases: 64 - oracle: crash_or_signal - - - id: pdf_to_pdftoraster_semantic - description: Semantic PDF parser sweep into ASan-built pdftoraster. - ppd_kind: pdftoraster_coverage_options - document_kind: pdf_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftoraster - input_mime: application/pdf - expected_filters: - - pdftoraster - cases: 64 - oracle: crash_or_signal - - - id: pdf_to_mupdftopwg_semantic - description: Semantic PDF parser sweep into ASan-built mupdftopwg. - ppd_kind: mupdftopwg_coverage_options - document_kind: pdf_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/mupdftopwg - input_mime: application/pdf - expected_filters: - - mupdftopwg - cases: 64 - oracle: crash_or_signal - - - id: image_to_imagetoraster_feedback_semantic - description: Output-goal image structure sweep into ASan-built imagetoraster. - ppd_kind: imagetoraster_coverage_options - document_kind: image_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetoraster - input_mime: image/x-portable-anymap - expected_filters: - - imagetoraster - cases: 96 - oracle: crash_or_signal - - - id: image_to_imagetopdf_feedback_semantic - description: Output-goal image structure sweep into ASan-built imagetopdf. - ppd_kind: imagetopdf_coverage_options - document_kind: image_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetopdf - input_mime: image/x-portable-anymap - expected_filters: - - imagetopdf - cases: 96 - oracle: crash_or_signal - - - id: image_to_imagetops_feedback_semantic - description: Output-goal image structure sweep into ASan-built imagetops. - ppd_kind: imagetops_coverage_options - document_kind: image_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetops - input_mime: image/x-portable-anymap - expected_filters: - - imagetops - cases: 96 - oracle: crash_or_signal - - - id: text_to_texttopdf_semantic - description: Semantic text parser sweep into ASan-built texttopdf. - ppd_kind: texttopdf_coverage_options - document_kind: text_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/texttopdf - input_mime: text/plain - expected_filters: - - texttopdf - cases: 64 - oracle: crash_or_signal - - - id: text_to_texttotext_semantic - description: Semantic text parser sweep into ASan-built texttotext. - ppd_kind: texttotext_coverage_options - document_kind: text_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/texttotext - input_mime: text/plain - expected_filters: - - texttotext - cases: 64 - oracle: crash_or_signal - - - id: postscript_to_gstoraster_semantic - description: Semantic PostScript parser sweep into ASan-built gstoraster wrapper. - ppd_kind: gstoraster_coverage_options - document_kind: postscript_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstoraster - input_mime: application/postscript - expected_filters: - - gstoraster - cases: 64 - oracle: crash_or_signal - - - id: postscript_to_gstopdf_semantic - description: Semantic PostScript parser sweep into ASan-built gstopdf wrapper. - ppd_kind: gstopdf_coverage_options - document_kind: postscript_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstopdf - input_mime: application/postscript - expected_filters: - - gstopdf - cases: 64 - oracle: crash_or_signal - - - id: postscript_to_gstopxl_semantic - description: Semantic PostScript parser sweep into ASan-built gstopxl wrapper. - ppd_kind: gstopxl_coverage_options - document_kind: postscript_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstopxl - input_mime: application/postscript - expected_filters: - - gstopxl - cases: 64 - oracle: crash_or_signal - - - id: command_to_escpx_semantic - description: Semantic CUPS command parser sweep into ASan-built commandtoescpx. - ppd_kind: commandtoescpx_coverage_options - document_kind: command_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/commandtoescpx - input_mime: application/vnd.cups-command - expected_filters: - - commandtoescpx - cases: 64 - oracle: crash_or_signal - - - id: command_to_pclx_semantic - description: Semantic CUPS command parser sweep into ASan-built commandtopclx. - ppd_kind: commandtopclx_coverage_options - document_kind: command_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/commandtopclx - input_mime: application/vnd.cups-command - expected_filters: - - commandtopclx - cases: 64 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_coverage.yaml b/parser-fuzzers/configs/parser_targets_coverage.yaml deleted file mode 100644 index 88eaf35..0000000 --- a/parser-fuzzers/configs/parser_targets_coverage.yaml +++ /dev/null @@ -1,240 +0,0 @@ -targets: - - id: cups_raster_to_rastertopclx_coverage - description: Coverage-discovery CUPS Raster sweep into ASan-built rastertopclx with varied PPD option groups. - ppd_kind: raster_coverage_options - document_kind: cups_raster_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertopclx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertopclx - cases: 14 - oracle: crash_or_signal - - - id: cups_raster_to_rastertoescpx_coverage - description: Coverage-discovery CUPS Raster sweep into ASan-built rastertoescpx with multi-page/color-space variation. - ppd_kind: raster_coverage_options - document_kind: cups_raster_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertoescpx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertoescpx - cases: 14 - oracle: crash_or_signal - - - id: cups_raster_to_rastertops_coverage - description: Coverage-discovery CUPS Raster sweep into ASan-built rastertops with varied PPD option groups. - ppd_kind: raster_coverage_options - document_kind: cups_raster_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertops - input_mime: application/vnd.cups-raster - expected_filters: - - rastertops - cases: 14 - oracle: crash_or_signal - - - id: pwg_to_raster_coverage - description: Coverage-discovery PWG Raster sweep into ASan-built pwgtoraster with avoid-constrained resolution options. - ppd_kind: pwg_resolution_coverage - document_kind: pwg_raster_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtoraster - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtoraster - cases: 16 - oracle: crash_or_signal - - - id: pwg_to_pdf_coverage - description: Coverage-discovery PWG Raster sweep into ASan-built pwgtopdf with varied PPD option groups. - ppd_kind: pwgtopdf_coverage_options - document_kind: pwg_raster_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopdf - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopdf - cases: 16 - oracle: crash_or_signal - - - id: pdf_to_pdftopdf_coverage - description: Coverage-discovery PDF parser sweep into ASan-built pdftopdf. - ppd_kind: pdftopdf_coverage_options - document_kind: pdf_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftopdf - input_mime: application/pdf - expected_filters: - - pdftopdf - cases: 8 - oracle: crash_or_signal - - - id: pdf_to_pdftops_coverage - description: Coverage-discovery PDF parser sweep into ASan-built pdftops. - ppd_kind: pdftops_coverage_options - document_kind: pdf_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftops - input_mime: application/pdf - expected_filters: - - pdftops - cases: 8 - oracle: crash_or_signal - - - id: pdf_to_pdftoraster_coverage - description: Coverage-discovery PDF parser sweep into ASan-built pdftoraster. - ppd_kind: pdftoraster_coverage_options - document_kind: pdf_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftoraster - input_mime: application/pdf - expected_filters: - - pdftoraster - cases: 8 - oracle: crash_or_signal - - - id: pdf_to_mupdftopwg_coverage - description: Coverage-discovery PDF parser sweep into ASan-built mupdftopwg. - ppd_kind: mupdftopwg_coverage_options - document_kind: pdf_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/mupdftopwg - input_mime: application/pdf - expected_filters: - - mupdftopwg - cases: 8 - oracle: crash_or_signal - - - id: image_to_imagetoraster_coverage - description: Coverage-discovery image parser sweep into ASan-built imagetoraster. - ppd_kind: imagetoraster_coverage_options - document_kind: image_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetoraster - input_mime: image/x-portable-anymap - expected_filters: - - imagetoraster - cases: 12 - oracle: crash_or_signal - - - id: image_to_imagetopdf_coverage - description: Coverage-discovery image parser sweep into ASan-built imagetopdf. - ppd_kind: imagetopdf_coverage_options - document_kind: image_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetopdf - input_mime: image/x-portable-anymap - expected_filters: - - imagetopdf - cases: 12 - oracle: crash_or_signal - - - id: image_to_imagetops_coverage - description: Coverage-discovery image parser sweep into ASan-built imagetops. - ppd_kind: imagetops_coverage_options - document_kind: image_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetops - input_mime: image/x-portable-anymap - expected_filters: - - imagetops - cases: 12 - oracle: crash_or_signal - - - id: text_to_texttopdf_coverage - description: Coverage-discovery text parser sweep into ASan-built texttopdf. - ppd_kind: texttopdf_coverage_options - document_kind: text_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/texttopdf - input_mime: text/plain - expected_filters: - - texttopdf - cases: 10 - oracle: crash_or_signal - - - id: text_to_texttotext_coverage - description: Coverage-discovery text parser sweep into ASan-built texttotext. - ppd_kind: texttotext_coverage_options - document_kind: text_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/texttotext - input_mime: text/plain - expected_filters: - - texttotext - cases: 10 - oracle: crash_or_signal - - - id: postscript_to_gstoraster_coverage - description: Coverage-discovery PostScript/Ghostscript parser sweep into ASan-built gstoraster wrapper. - ppd_kind: gstoraster_coverage_options - document_kind: postscript_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstoraster - input_mime: application/postscript - expected_filters: - - gstoraster - cases: 8 - oracle: crash_or_signal - - - id: postscript_to_gstopdf_coverage - description: Coverage-discovery PostScript/Ghostscript parser sweep into ASan-built gstopdf wrapper. - ppd_kind: gstopdf_coverage_options - document_kind: postscript_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstopdf - input_mime: application/postscript - expected_filters: - - gstopdf - cases: 8 - oracle: crash_or_signal - - - id: postscript_to_gstopxl_coverage - description: Coverage-discovery PostScript/Ghostscript parser sweep into ASan-built gstopxl wrapper. - ppd_kind: gstopxl_coverage_options - document_kind: postscript_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstopxl - input_mime: application/postscript - expected_filters: - - gstopxl - cases: 8 - oracle: crash_or_signal - - - id: pwg_to_pclm_coverage - description: Coverage-discovery PWG Raster parser sweep into ASan-built pwgtopclm. - ppd_kind: pwgtopclm_coverage_options - document_kind: pwg_raster_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopclm - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopclm - cases: 16 - oracle: crash_or_signal - - - id: command_to_escpx_coverage - description: Coverage-discovery CUPS command parser sweep into ASan-built commandtoescpx. - ppd_kind: commandtoescpx_coverage_options - document_kind: command_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/commandtoescpx - input_mime: application/vnd.cups-command - expected_filters: - - commandtoescpx - cases: 10 - oracle: crash_or_signal - - - id: command_to_pclx_coverage - description: Coverage-discovery CUPS command parser sweep into ASan-built commandtopclx. - ppd_kind: commandtopclx_coverage_options - document_kind: command_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/commandtopclx - input_mime: application/vnd.cups-command - expected_filters: - - commandtopclx - cases: 10 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_explore.yaml b/parser-fuzzers/configs/parser_targets_explore.yaml deleted file mode 100644 index b82dee3..0000000 --- a/parser-fuzzers/configs/parser_targets_explore.yaml +++ /dev/null @@ -1,49 +0,0 @@ -targets: - - id: ppd_text_to_rastertopclx_explore - description: Generic PPD string-value sweep through cupsfilter into rastertopclx. - ppd_kind: rastertopclx_string_sweep - document_kind: text - executor: cupsfilter - input_mime: text/plain - output_mime: printer/foo - expected_filters: - - universal - - rastertopclx - cases: 8 - oracle: crash_or_signal - - - id: cups_raster_to_rastertopclx_explore - description: Generic CUPS Raster header boundary sweep into rastertopclx. - ppd_kind: rastertopclx_plain - document_kind: cups_raster_boundary_sweep - executor: direct_filter - filter_binary: /usr/lib/cups/filter/rastertopclx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertopclx - cases: 8 - oracle: crash_or_signal - - - id: cups_raster_to_rastertoescpx_explore - description: Generic single-size PPD plus CUPS Raster boundary sweep into rastertoescpx. - ppd_kind: rastertoescpx_size_sweep - document_kind: cups_raster_boundary_sweep - executor: direct_filter - filter_binary: /usr/lib/cups/filter/rastertoescpx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertoescpx - cases: 8 - oracle: reached_only - - - id: pwg_to_raster_resolution_explore - description: Generic output-resolution and PWG Raster header boundary sweep into pwgtoraster. - ppd_kind: pwg_resolution_sweep - document_kind: pwg_raster_boundary_sweep - executor: direct_filter - filter_binary: /usr/lib/cups/filter/pwgtoraster - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtoraster - cases: 8 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_feedback.yaml b/parser-fuzzers/configs/parser_targets_feedback.yaml deleted file mode 100644 index b72eb0a..0000000 --- a/parser-fuzzers/configs/parser_targets_feedback.yaml +++ /dev/null @@ -1,72 +0,0 @@ -targets: - - id: cups_raster_to_rastertopclx_feedback - description: Feedback-driven SMT CUPS Raster field-relation sweep into ASan-built rastertopclx. - ppd_kind: raster_coverage_options - document_kind: cups_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertopclx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertopclx - cases: 24 - oracle: crash_or_signal - - - id: cups_raster_to_rastertoescpx_feedback - description: Feedback-driven SMT CUPS Raster field-relation sweep into ASan-built rastertoescpx. - ppd_kind: raster_coverage_options - document_kind: cups_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertoescpx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertoescpx - cases: 24 - oracle: crash_or_signal - - - id: cups_raster_to_rastertops_feedback - description: Feedback-driven SMT CUPS Raster field-relation sweep into ASan-built rastertops. - ppd_kind: raster_coverage_options - document_kind: cups_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertops - input_mime: application/vnd.cups-raster - expected_filters: - - rastertops - cases: 24 - oracle: crash_or_signal - - - id: pwg_to_raster_feedback - description: Feedback-driven SMT PWG Raster field-relation sweep into ASan-built pwgtoraster. - ppd_kind: pwg_resolution_coverage - document_kind: pwg_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtoraster - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtoraster - cases: 24 - oracle: crash_or_signal - - - id: pwg_to_pdf_feedback - description: Feedback-driven SMT PWG Raster field-relation sweep into ASan-built pwgtopdf. - ppd_kind: pwgtopdf_coverage_options - document_kind: pwg_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopdf - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopdf - cases: 24 - oracle: crash_or_signal - - - id: pwg_to_pclm_feedback - description: Feedback-driven SMT PWG Raster field-relation sweep into ASan-built pwgtopclm. - ppd_kind: pwgtopclm_coverage_options - document_kind: pwg_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopclm - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopclm - cases: 24 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_general.yaml b/parser-fuzzers/configs/parser_targets_general.yaml deleted file mode 100644 index b263180..0000000 --- a/parser-fuzzers/configs/parser_targets_general.yaml +++ /dev/null @@ -1,73 +0,0 @@ -targets: - - id: ppd_text_to_rastertopclx_general - description: General PPD string-value variation through cupsfilter into rastertopclx using safe literal strings. - ppd_kind: rastertopclx_general_strings - document_kind: text - executor: cupsfilter - input_mime: text/plain - output_mime: printer/foo - expected_filters: - - universal - - rastertopclx - cases: 12 - oracle: crash_or_signal - - - id: cups_raster_to_rastertopclx_general - description: General valid CUPS Raster sweep into ASan-built rastertopclx. - ppd_kind: rastertopclx_plain - document_kind: cups_raster_general_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertopclx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertopclx - cases: 12 - oracle: crash_or_signal - - - id: cups_raster_to_rastertoescpx_general - description: General valid CUPS Raster sweep into ASan-built rastertoescpx. - ppd_kind: rastertoescpx_size_sweep - document_kind: cups_raster_general_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertoescpx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertoescpx - cases: 12 - oracle: crash_or_signal - - - id: cups_raster_to_rastertops_general - description: General valid CUPS Raster sweep into ASan-built rastertops. - ppd_kind: rastertops_plain - document_kind: cups_raster_general_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertops - input_mime: application/vnd.cups-raster - expected_filters: - - rastertops - cases: 12 - oracle: crash_or_signal - - - id: pwg_to_raster_general - description: General valid PWG Raster and PPD resolution sweep into ASan-built pwgtoraster; excludes 2^31 stress value. - ppd_kind: pwg_resolution_general - document_kind: pwg_raster_general_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtoraster - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtoraster - cases: 15 - oracle: crash_or_signal - - - id: pwg_to_pdf_general - description: General valid PWG Raster sweep into ASan-built pwgtopdf. - ppd_kind: pwgtopdf_plain - document_kind: pwg_raster_general_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopdf - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopdf - cases: 15 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_image_feedback.yaml b/parser-fuzzers/configs/parser_targets_image_feedback.yaml deleted file mode 100644 index ac49ded..0000000 --- a/parser-fuzzers/configs/parser_targets_image_feedback.yaml +++ /dev/null @@ -1,36 +0,0 @@ -targets: - - id: image_to_imagetoraster_feedback - description: Feedback-driven SMT image parser sweep into ASan-built imagetoraster. - ppd_kind: imagetoraster_coverage_options - document_kind: image_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetoraster - input_mime: image/x-portable-anymap - expected_filters: - - imagetoraster - cases: 24 - oracle: crash_or_signal - - - id: image_to_imagetopdf_feedback - description: Feedback-driven SMT image parser sweep into ASan-built imagetopdf. - ppd_kind: imagetopdf_coverage_options - document_kind: image_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetopdf - input_mime: image/x-portable-anymap - expected_filters: - - imagetopdf - cases: 24 - oracle: crash_or_signal - - - id: image_to_imagetops_feedback - description: Feedback-driven SMT image parser sweep into ASan-built imagetops. - ppd_kind: imagetops_coverage_options - document_kind: image_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetops - input_mime: image/x-portable-anymap - expected_filters: - - imagetops - cases: 24 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_image_imagetopdf_feedback.yaml b/parser-fuzzers/configs/parser_targets_image_imagetopdf_feedback.yaml deleted file mode 100644 index 311a836..0000000 --- a/parser-fuzzers/configs/parser_targets_image_imagetopdf_feedback.yaml +++ /dev/null @@ -1,12 +0,0 @@ -targets: - - id: image_to_imagetopdf_feedback - description: Feedback-driven SMT image parser sweep into ASan-built imagetopdf. - ppd_kind: imagetopdf_coverage_options - document_kind: image_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetopdf - input_mime: image/x-portable-anymap - expected_filters: - - imagetopdf - cases: 24 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_image_imagetops_feedback.yaml b/parser-fuzzers/configs/parser_targets_image_imagetops_feedback.yaml deleted file mode 100644 index 8b1dc58..0000000 --- a/parser-fuzzers/configs/parser_targets_image_imagetops_feedback.yaml +++ /dev/null @@ -1,12 +0,0 @@ -targets: - - id: image_to_imagetops_feedback - description: Feedback-driven SMT image parser sweep into ASan-built imagetops. - ppd_kind: imagetops_coverage_options - document_kind: image_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetops - input_mime: image/x-portable-anymap - expected_filters: - - imagetops - cases: 24 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_image_imagetoraster_feedback.yaml b/parser-fuzzers/configs/parser_targets_image_imagetoraster_feedback.yaml deleted file mode 100644 index b0a39ed..0000000 --- a/parser-fuzzers/configs/parser_targets_image_imagetoraster_feedback.yaml +++ /dev/null @@ -1,12 +0,0 @@ -targets: - - id: image_to_imagetoraster_feedback - description: Feedback-driven SMT image parser sweep into ASan-built imagetoraster. - ppd_kind: imagetoraster_coverage_options - document_kind: image_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/imagetoraster - input_mime: image/x-portable-anymap - expected_filters: - - imagetoraster - cases: 24 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_structural.yaml b/parser-fuzzers/configs/parser_targets_structural.yaml deleted file mode 100644 index cf62c4f..0000000 --- a/parser-fuzzers/configs/parser_targets_structural.yaml +++ /dev/null @@ -1,72 +0,0 @@ -targets: - - id: cups_raster_to_rastertopclx_structural - description: SMT structural CUPS Raster field-relation sweep into ASan-built rastertopclx. - ppd_kind: raster_coverage_options - document_kind: cups_raster_structural_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertopclx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertopclx - cases: 24 - oracle: crash_or_signal - - - id: cups_raster_to_rastertoescpx_structural - description: SMT structural CUPS Raster field-relation sweep into ASan-built rastertoescpx. - ppd_kind: raster_coverage_options - document_kind: cups_raster_structural_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertoescpx - input_mime: application/vnd.cups-raster - expected_filters: - - rastertoescpx - cases: 24 - oracle: crash_or_signal - - - id: cups_raster_to_rastertops_structural - description: SMT structural CUPS Raster field-relation sweep into ASan-built rastertops. - ppd_kind: raster_coverage_options - document_kind: cups_raster_structural_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/rastertops - input_mime: application/vnd.cups-raster - expected_filters: - - rastertops - cases: 24 - oracle: crash_or_signal - - - id: pwg_to_raster_structural - description: SMT structural PWG Raster field-relation sweep into ASan-built pwgtoraster. - ppd_kind: pwg_resolution_coverage - document_kind: pwg_raster_structural_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtoraster - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtoraster - cases: 24 - oracle: crash_or_signal - - - id: pwg_to_pdf_structural - description: SMT structural PWG Raster field-relation sweep into ASan-built pwgtopdf. - ppd_kind: pwgtopdf_coverage_options - document_kind: pwg_raster_structural_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopdf - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopdf - cases: 24 - oracle: crash_or_signal - - - id: pwg_to_pclm_structural - description: SMT structural PWG Raster field-relation sweep into ASan-built pwgtopclm. - ppd_kind: pwgtopclm_coverage_options - document_kind: pwg_raster_structural_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopclm - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopclm - cases: 24 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_underexplored.yaml b/parser-fuzzers/configs/parser_targets_underexplored.yaml deleted file mode 100644 index b262615..0000000 --- a/parser-fuzzers/configs/parser_targets_underexplored.yaml +++ /dev/null @@ -1,144 +0,0 @@ -targets: - - id: pdf_to_pdftopdf_underexplored - description: Underexplored PDF parser sweep into ASan-built pdftopdf. - ppd_kind: pdftopdf_coverage_options - document_kind: pdf_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftopdf - input_mime: application/pdf - expected_filters: - - pdftopdf - cases: 16 - oracle: crash_or_signal - - - id: pdf_to_pdftops_underexplored - description: Underexplored PDF parser sweep into ASan-built pdftops. - ppd_kind: pdftops_coverage_options - document_kind: pdf_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftops - input_mime: application/pdf - expected_filters: - - pdftops - cases: 16 - oracle: crash_or_signal - - - id: pdf_to_mupdftopwg_underexplored - description: Underexplored PDF parser sweep into ASan-built mupdftopwg. - ppd_kind: mupdftopwg_coverage_options - document_kind: pdf_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/mupdftopwg - input_mime: application/pdf - expected_filters: - - mupdftopwg - cases: 16 - oracle: crash_or_signal - - - id: text_to_texttopdf_underexplored - description: Underexplored text parser sweep into ASan-built texttopdf. - ppd_kind: texttopdf_coverage_options - document_kind: text_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/texttopdf - input_mime: text/plain - expected_filters: - - texttopdf - cases: 20 - oracle: crash_or_signal - - - id: text_to_texttotext_underexplored - description: Underexplored text parser sweep into ASan-built texttotext. - ppd_kind: texttotext_coverage_options - document_kind: text_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/texttotext - input_mime: text/plain - expected_filters: - - texttotext - cases: 20 - oracle: crash_or_signal - - - id: postscript_to_gstoraster_underexplored - description: Underexplored PostScript parser sweep into ASan-built gstoraster wrapper. - ppd_kind: gstoraster_coverage_options - document_kind: postscript_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstoraster - input_mime: application/postscript - expected_filters: - - gstoraster - cases: 16 - oracle: crash_or_signal - - - id: postscript_to_gstopdf_underexplored - description: Underexplored PostScript parser sweep into ASan-built gstopdf wrapper. - ppd_kind: gstopdf_coverage_options - document_kind: postscript_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstopdf - input_mime: application/postscript - expected_filters: - - gstopdf - cases: 16 - oracle: crash_or_signal - - - id: postscript_to_gstopxl_underexplored - description: Underexplored PostScript parser sweep into ASan-built gstopxl wrapper. - ppd_kind: gstopxl_coverage_options - document_kind: postscript_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstopxl - input_mime: application/postscript - expected_filters: - - gstopxl - cases: 16 - oracle: crash_or_signal - - - id: pwg_to_pdf_underexplored - description: Underexplored PWG Raster parser sweep into ASan-built pwgtopdf. - ppd_kind: pwgtopdf_coverage_options - document_kind: pwg_raster_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopdf - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopdf - cases: 24 - oracle: crash_or_signal - - - id: pwg_to_pclm_underexplored - description: Underexplored PWG Raster parser sweep into ASan-built pwgtopclm. - ppd_kind: pwgtopclm_coverage_options - document_kind: pwg_raster_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopclm - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopclm - cases: 24 - oracle: crash_or_signal - - - id: command_to_escpx_underexplored - description: Underexplored CUPS command parser sweep into ASan-built commandtoescpx. - ppd_kind: commandtoescpx_coverage_options - document_kind: command_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/commandtoescpx - input_mime: application/vnd.cups-command - expected_filters: - - commandtoescpx - cases: 20 - oracle: crash_or_signal - - - id: command_to_pclx_underexplored - description: Underexplored CUPS command parser sweep into ASan-built commandtopclx. - ppd_kind: commandtopclx_coverage_options - document_kind: command_coverage_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/commandtopclx - input_mime: application/vnd.cups-command - expected_filters: - - commandtopclx - cases: 20 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_underexplored_feedback30.yaml b/parser-fuzzers/configs/parser_targets_underexplored_feedback30.yaml deleted file mode 100644 index 70e0726..0000000 --- a/parser-fuzzers/configs/parser_targets_underexplored_feedback30.yaml +++ /dev/null @@ -1,144 +0,0 @@ -targets: - - id: pdf_to_pdftopdf_semantic_f30 - description: Semantic PDF parser sweep into ASan-built pdftopdf for feedback-guided 30 minute runs. - ppd_kind: pdftopdf_coverage_options - document_kind: pdf_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftopdf - input_mime: application/pdf - expected_filters: - - pdftopdf - cases: 96 - oracle: crash_or_signal - - - id: pdf_to_pdftops_semantic_f30 - description: Semantic PDF parser sweep into ASan-built pdftops for feedback-guided 30 minute runs. - ppd_kind: pdftops_coverage_options - document_kind: pdf_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftops - input_mime: application/pdf - expected_filters: - - pdftops - cases: 96 - oracle: crash_or_signal - - - id: pdf_to_mupdftopwg_semantic_f30 - description: Semantic PDF parser sweep into ASan-built mupdftopwg for feedback-guided 30 minute runs. - ppd_kind: mupdftopwg_coverage_options - document_kind: pdf_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/mupdftopwg - input_mime: application/pdf - expected_filters: - - mupdftopwg - cases: 96 - oracle: crash_or_signal - - - id: text_to_texttopdf_semantic_f30 - description: Semantic text parser sweep into ASan-built texttopdf for feedback-guided 30 minute runs. - ppd_kind: texttopdf_coverage_options - document_kind: text_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/texttopdf - input_mime: text/plain - expected_filters: - - texttopdf - cases: 96 - oracle: crash_or_signal - - - id: text_to_texttotext_semantic_f30 - description: Semantic text parser sweep into ASan-built texttotext for feedback-guided 30 minute runs. - ppd_kind: texttotext_coverage_options - document_kind: text_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/texttotext - input_mime: text/plain - expected_filters: - - texttotext - cases: 96 - oracle: crash_or_signal - - - id: postscript_to_gstoraster_semantic_f30 - description: Semantic PostScript parser sweep into ASan-built gstoraster wrapper for feedback-guided 30 minute runs. - ppd_kind: gstoraster_coverage_options - document_kind: postscript_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstoraster - input_mime: application/postscript - expected_filters: - - gstoraster - cases: 72 - oracle: crash_or_signal - - - id: postscript_to_gstopdf_semantic_f30 - description: Semantic PostScript parser sweep into ASan-built gstopdf wrapper for feedback-guided 30 minute runs. - ppd_kind: gstopdf_coverage_options - document_kind: postscript_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstopdf - input_mime: application/postscript - expected_filters: - - gstopdf - cases: 72 - oracle: crash_or_signal - - - id: postscript_to_gstopxl_semantic_f30 - description: Semantic PostScript parser sweep into ASan-built gstopxl wrapper for feedback-guided 30 minute runs. - ppd_kind: gstopxl_coverage_options - document_kind: postscript_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstopxl - input_mime: application/postscript - expected_filters: - - gstopxl - cases: 72 - oracle: crash_or_signal - - - id: pwg_to_pdf_feedback_f30 - description: Frontier feedback PWG Raster parser sweep into ASan-built pwgtopdf. - ppd_kind: pwgtopdf_coverage_options - document_kind: pwg_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopdf - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopdf - cases: 128 - oracle: crash_or_signal - - - id: pwg_to_pclm_feedback_f30 - description: Frontier feedback PWG Raster parser sweep into ASan-built pwgtopclm. - ppd_kind: pwgtopclm_coverage_options - document_kind: pwg_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopclm - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopclm - cases: 128 - oracle: crash_or_signal - - - id: command_to_escpx_semantic_f30 - description: Semantic CUPS command parser sweep into ASan-built commandtoescpx for feedback-guided 30 minute runs. - ppd_kind: commandtoescpx_coverage_options - document_kind: command_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/commandtoescpx - input_mime: application/vnd.cups-command - expected_filters: - - commandtoescpx - cases: 64 - oracle: crash_or_signal - - - id: command_to_pclx_semantic_f30 - description: Semantic CUPS command parser sweep into ASan-built commandtopclx for feedback-guided 30 minute runs. - ppd_kind: commandtopclx_coverage_options - document_kind: command_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/commandtopclx - input_mime: application/vnd.cups-command - expected_filters: - - commandtopclx - cases: 64 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/parser_targets_underexplored_semantic.yaml b/parser-fuzzers/configs/parser_targets_underexplored_semantic.yaml deleted file mode 100644 index 36baad6..0000000 --- a/parser-fuzzers/configs/parser_targets_underexplored_semantic.yaml +++ /dev/null @@ -1,144 +0,0 @@ -targets: - - id: pdf_to_pdftopdf_semantic - description: Semantic PDF parser sweep into ASan-built pdftopdf. - ppd_kind: pdftopdf_coverage_options - document_kind: pdf_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftopdf - input_mime: application/pdf - expected_filters: - - pdftopdf - cases: 64 - oracle: crash_or_signal - - - id: pdf_to_pdftops_semantic - description: Semantic PDF parser sweep into ASan-built pdftops. - ppd_kind: pdftops_coverage_options - document_kind: pdf_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pdftops - input_mime: application/pdf - expected_filters: - - pdftops - cases: 64 - oracle: crash_or_signal - - - id: pdf_to_mupdftopwg_semantic - description: Semantic PDF parser sweep into ASan-built mupdftopwg. - ppd_kind: mupdftopwg_coverage_options - document_kind: pdf_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/mupdftopwg - input_mime: application/pdf - expected_filters: - - mupdftopwg - cases: 64 - oracle: crash_or_signal - - - id: text_to_texttopdf_semantic - description: Semantic text parser sweep into ASan-built texttopdf. - ppd_kind: texttopdf_coverage_options - document_kind: text_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/texttopdf - input_mime: text/plain - expected_filters: - - texttopdf - cases: 64 - oracle: crash_or_signal - - - id: text_to_texttotext_semantic - description: Semantic text parser sweep into ASan-built texttotext. - ppd_kind: texttotext_coverage_options - document_kind: text_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/texttotext - input_mime: text/plain - expected_filters: - - texttotext - cases: 64 - oracle: crash_or_signal - - - id: postscript_to_gstoraster_semantic - description: Semantic PostScript parser sweep into ASan-built gstoraster wrapper. - ppd_kind: gstoraster_coverage_options - document_kind: postscript_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstoraster - input_mime: application/postscript - expected_filters: - - gstoraster - cases: 48 - oracle: crash_or_signal - - - id: postscript_to_gstopdf_semantic - description: Semantic PostScript parser sweep into ASan-built gstopdf wrapper. - ppd_kind: gstopdf_coverage_options - document_kind: postscript_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstopdf - input_mime: application/postscript - expected_filters: - - gstopdf - cases: 48 - oracle: crash_or_signal - - - id: postscript_to_gstopxl_semantic - description: Semantic PostScript parser sweep into ASan-built gstopxl wrapper. - ppd_kind: gstopxl_coverage_options - document_kind: postscript_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/gstopxl - input_mime: application/postscript - expected_filters: - - gstopxl - cases: 48 - oracle: crash_or_signal - - - id: pwg_to_pdf_structural - description: Structural PWG Raster parser sweep into ASan-built pwgtopdf. - ppd_kind: pwgtopdf_coverage_options - document_kind: pwg_raster_structural_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopdf - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopdf - cases: 64 - oracle: crash_or_signal - - - id: pwg_to_pclm_structural - description: Structural PWG Raster parser sweep into ASan-built pwgtopclm. - ppd_kind: pwgtopclm_coverage_options - document_kind: pwg_raster_structural_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopclm - input_mime: application/vnd.cups-pwg - expected_filters: - - pwgtopclm - cases: 64 - oracle: crash_or_signal - - - id: command_to_escpx_semantic - description: Semantic CUPS command parser sweep into ASan-built commandtoescpx. - ppd_kind: commandtoescpx_coverage_options - document_kind: command_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/commandtoescpx - input_mime: application/vnd.cups-command - expected_filters: - - commandtoescpx - cases: 40 - oracle: crash_or_signal - - - id: command_to_pclx_semantic - description: Semantic CUPS command parser sweep into ASan-built commandtopclx. - ppd_kind: commandtopclx_coverage_options - document_kind: command_semantic_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/commandtopclx - input_mime: application/vnd.cups-command - expected_filters: - - commandtopclx - cases: 40 - oracle: crash_or_signal diff --git a/parser-fuzzers/configs/targets.yaml b/parser-fuzzers/configs/targets.yaml deleted file mode 100644 index e18bb1f..0000000 --- a/parser-fuzzers/configs/targets.yaml +++ /dev/null @@ -1,45 +0,0 @@ -targets: - - id: ppd_ipp_parser - name: PPD and IPP attribute parser - components: - - libppd - - libcups - - cups-filters - target_functions: - - ppdCreatePPDFromIPP2 - - ippValidateAttribute - dictionaries: - - dictionaries/ppd.dict - - dictionaries/ipp_options.dict - - id: mime_filter_chain - name: MIME and filter-chain parser - components: - - cups-filters - - cups-browsed - target_functions: - - mimeLoadTypes - - mimeLoadFilters - dictionaries: - - dictionaries/mime.dict - - id: image_options_parser - name: Image/raster header and job-option parser - components: - - cups-filters - - libcupsfilters - target_functions: - - parseOpts - - cupsRasterReadHeader2 - dictionaries: - - dictionaries/image_headers.dict - - dictionaries/ipp_options.dict - - id: template_probe_pwg - name: AFL++ instrumented template probe for generated PWG/CUPS seeds - components: - - parser-fuzzers - - afl++ - target_functions: - - afl_template_probe - dictionaries: - - dictionaries/pwg_raster.dict - - dictionaries/image_headers.dict - - dictionaries/ppd.dict diff --git a/parser-fuzzers/corpus/fuzz_cupsfilters_format_cups_raster_seed_corpus.zip b/parser-fuzzers/corpus/fuzz_cupsfilters_format_cups_raster_seed_corpus.zip new file mode 100644 index 0000000..38faab9 Binary files /dev/null and b/parser-fuzzers/corpus/fuzz_cupsfilters_format_cups_raster_seed_corpus.zip differ diff --git a/parser-fuzzers/corpus/fuzz_cupsfilters_format_image_jpeg_bounded_seed_corpus.zip b/parser-fuzzers/corpus/fuzz_cupsfilters_format_image_jpeg_bounded_seed_corpus.zip new file mode 100644 index 0000000..7f7a449 Binary files /dev/null and b/parser-fuzzers/corpus/fuzz_cupsfilters_format_image_jpeg_bounded_seed_corpus.zip differ diff --git a/parser-fuzzers/corpus/fuzz_cupsfilters_format_image_png_bounded_seed_corpus.zip b/parser-fuzzers/corpus/fuzz_cupsfilters_format_image_png_bounded_seed_corpus.zip new file mode 100644 index 0000000..1414bf2 Binary files /dev/null and b/parser-fuzzers/corpus/fuzz_cupsfilters_format_image_png_bounded_seed_corpus.zip differ diff --git a/parser-fuzzers/corpus/fuzz_cupsfilters_format_image_tiff_bounded_seed_corpus.zip b/parser-fuzzers/corpus/fuzz_cupsfilters_format_image_tiff_bounded_seed_corpus.zip new file mode 100644 index 0000000..4162c70 Binary files /dev/null and b/parser-fuzzers/corpus/fuzz_cupsfilters_format_image_tiff_bounded_seed_corpus.zip differ diff --git a/parser-fuzzers/corpus/fuzz_cupsfilters_raster_to_pclx_mode10_codec_seed_corpus.zip b/parser-fuzzers/corpus/fuzz_cupsfilters_raster_to_pclx_mode10_codec_seed_corpus.zip new file mode 100644 index 0000000..0f6a4fe Binary files /dev/null and b/parser-fuzzers/corpus/fuzz_cupsfilters_raster_to_pclx_mode10_codec_seed_corpus.zip differ diff --git a/parser-fuzzers/corpus/fuzz_cupsfilters_raster_to_pclx_mode3_codec_seed_corpus.zip b/parser-fuzzers/corpus/fuzz_cupsfilters_raster_to_pclx_mode3_codec_seed_corpus.zip new file mode 100644 index 0000000..0f6a4fe Binary files /dev/null and b/parser-fuzzers/corpus/fuzz_cupsfilters_raster_to_pclx_mode3_codec_seed_corpus.zip differ diff --git a/parser-fuzzers/corpus/fuzz_cupsfilters_state_pwg_to_raster_scale_down_seed_corpus.zip b/parser-fuzzers/corpus/fuzz_cupsfilters_state_pwg_to_raster_scale_down_seed_corpus.zip new file mode 100644 index 0000000..492d858 Binary files /dev/null and b/parser-fuzzers/corpus/fuzz_cupsfilters_state_pwg_to_raster_scale_down_seed_corpus.zip differ diff --git a/parser-fuzzers/corpus/fuzz_cupsfilters_state_pwg_to_raster_scale_up_seed_corpus.zip b/parser-fuzzers/corpus/fuzz_cupsfilters_state_pwg_to_raster_scale_up_seed_corpus.zip new file mode 100644 index 0000000..492d858 Binary files /dev/null and b/parser-fuzzers/corpus/fuzz_cupsfilters_state_pwg_to_raster_scale_up_seed_corpus.zip differ diff --git a/parser-fuzzers/corpus/fuzz_cupsfilters_state_raster_to_apple_seed_corpus.zip b/parser-fuzzers/corpus/fuzz_cupsfilters_state_raster_to_apple_seed_corpus.zip new file mode 100644 index 0000000..bdee23b Binary files /dev/null and b/parser-fuzzers/corpus/fuzz_cupsfilters_state_raster_to_apple_seed_corpus.zip differ diff --git a/parser-fuzzers/corpus/fuzz_cupsfilters_state_raster_to_pwg_seed_corpus.zip b/parser-fuzzers/corpus/fuzz_cupsfilters_state_raster_to_pwg_seed_corpus.zip new file mode 100644 index 0000000..24eda75 Binary files /dev/null and b/parser-fuzzers/corpus/fuzz_cupsfilters_state_raster_to_pwg_seed_corpus.zip differ diff --git a/parser-fuzzers/corpus/fuzz_cupsfilters_state_text_to_text_layout_seed_corpus.zip b/parser-fuzzers/corpus/fuzz_cupsfilters_state_text_to_text_layout_seed_corpus.zip new file mode 100644 index 0000000..77d6c2a Binary files /dev/null and b/parser-fuzzers/corpus/fuzz_cupsfilters_state_text_to_text_layout_seed_corpus.zip differ diff --git a/parser-fuzzers/corpus/fuzz_cupsfilters_text_to_text_selection_oracle_seed_corpus.zip b/parser-fuzzers/corpus/fuzz_cupsfilters_text_to_text_selection_oracle_seed_corpus.zip new file mode 100644 index 0000000..08b6ff0 Binary files /dev/null and b/parser-fuzzers/corpus/fuzz_cupsfilters_text_to_text_selection_oracle_seed_corpus.zip differ diff --git a/parser-fuzzers/dictionaries/fuzz_cupsfilters_format_cups_raster.dict b/parser-fuzzers/dictionaries/fuzz_cupsfilters_format_cups_raster.dict new file mode 100644 index 0000000..244f261 --- /dev/null +++ b/parser-fuzzers/dictionaries/fuzz_cupsfilters_format_cups_raster.dict @@ -0,0 +1,4 @@ +token_0="RaS2" +token_1="2SaR" +token_2="RaS3" +token_3="3SaR" diff --git a/parser-fuzzers/dictionaries/fuzz_cupsfilters_format_image_jpeg_bounded.dict b/parser-fuzzers/dictionaries/fuzz_cupsfilters_format_image_jpeg_bounded.dict new file mode 100644 index 0000000..4dfa307 --- /dev/null +++ b/parser-fuzzers/dictionaries/fuzz_cupsfilters_format_image_jpeg_bounded.dict @@ -0,0 +1,10 @@ +token_0="\xff\xd8" +token_1="\xff\xd9" +token_2="\xff\xc0" +token_3="\xff\xc2" +token_4="\xff\xc4" +token_5="\xff\xda" +token_6="\xff\xdb" +token_7="\xff\xdd" +token_8="JFIF\x00" +token_9="Exif\x00\x00" diff --git a/parser-fuzzers/dictionaries/fuzz_cupsfilters_format_image_png_bounded.dict b/parser-fuzzers/dictionaries/fuzz_cupsfilters_format_image_png_bounded.dict new file mode 100644 index 0000000..d2049c3 --- /dev/null +++ b/parser-fuzzers/dictionaries/fuzz_cupsfilters_format_image_png_bounded.dict @@ -0,0 +1,6 @@ +token_0="\x89PNG\x0d\x0a\x1a\x0a" +token_1="IHDR" +token_2="IDAT" +token_3="IEND" +token_4="tEXt" +token_5="iCCP" diff --git a/parser-fuzzers/dictionaries/fuzz_cupsfilters_format_image_tiff_bounded.dict b/parser-fuzzers/dictionaries/fuzz_cupsfilters_format_image_tiff_bounded.dict new file mode 100644 index 0000000..29c1cc3 --- /dev/null +++ b/parser-fuzzers/dictionaries/fuzz_cupsfilters_format_image_tiff_bounded.dict @@ -0,0 +1,13 @@ +token_0="II*\x00" +token_1="MM\x00*" +token_2="\x00\x01" +token_3="\x01\x00" +token_4="\x01\x01" +token_5="\x02\x01" +token_6="\x03\x01" +token_7="\x06\x01" +token_8="\x11\x01" +token_9="\x15\x01" +token_10="\x16\x01" +token_11="\x17\x01" +token_12="\x1c\x01" diff --git a/parser-fuzzers/dictionaries/fuzz_cupsfilters_raster_to_pclx_mode10_codec.dict b/parser-fuzzers/dictionaries/fuzz_cupsfilters_raster_to_pclx_mode10_codec.dict new file mode 100644 index 0000000..efbf10b --- /dev/null +++ b/parser-fuzzers/dictionaries/fuzz_cupsfilters_raster_to_pclx_mode10_codec.dict @@ -0,0 +1,14 @@ +token_0="PCLXCMP1" +token_1="\x00" +token_2="\xff" +token_3="U" +token_4="\xaa" +token_5="\x00\xff" +token_6="\xff\x00" +token_7="\x00\x00\xff\xff" +token_8="\x1f" +token_9=" " +token_10="\x07" +token_11="\x08" +token_12="\xfe" +token_13="\xff" diff --git a/parser-fuzzers/dictionaries/fuzz_cupsfilters_raster_to_pclx_mode3_codec.dict b/parser-fuzzers/dictionaries/fuzz_cupsfilters_raster_to_pclx_mode3_codec.dict new file mode 100644 index 0000000..efbf10b --- /dev/null +++ b/parser-fuzzers/dictionaries/fuzz_cupsfilters_raster_to_pclx_mode3_codec.dict @@ -0,0 +1,14 @@ +token_0="PCLXCMP1" +token_1="\x00" +token_2="\xff" +token_3="U" +token_4="\xaa" +token_5="\x00\xff" +token_6="\xff\x00" +token_7="\x00\x00\xff\xff" +token_8="\x1f" +token_9=" " +token_10="\x07" +token_11="\x08" +token_12="\xfe" +token_13="\xff" diff --git a/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_pwg_to_raster_scale_down.dict b/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_pwg_to_raster_scale_down.dict new file mode 100644 index 0000000..aadb05e --- /dev/null +++ b/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_pwg_to_raster_scale_down.dict @@ -0,0 +1,8 @@ +token_0="PWGSCL1" +token_1="\x00" +token_2="\x01" +token_3="\x02" +token_4="\x03" +token_5="U" +token_6="\xaa" +token_7="\xff" diff --git a/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_pwg_to_raster_scale_up.dict b/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_pwg_to_raster_scale_up.dict new file mode 100644 index 0000000..aadb05e --- /dev/null +++ b/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_pwg_to_raster_scale_up.dict @@ -0,0 +1,8 @@ +token_0="PWGSCL1" +token_1="\x00" +token_2="\x01" +token_3="\x02" +token_4="\x03" +token_5="U" +token_6="\xaa" +token_7="\xff" diff --git a/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_raster_to_apple.dict b/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_raster_to_apple.dict new file mode 100644 index 0000000..3bca207 --- /dev/null +++ b/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_raster_to_apple.dict @@ -0,0 +1,5 @@ +token_0="ROSTATE1" +token_1="\x00" +token_2="\xff" +token_3="U" +token_4="\xaa" diff --git a/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_raster_to_pwg.dict b/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_raster_to_pwg.dict new file mode 100644 index 0000000..3bca207 --- /dev/null +++ b/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_raster_to_pwg.dict @@ -0,0 +1,5 @@ +token_0="ROSTATE1" +token_1="\x00" +token_2="\xff" +token_3="U" +token_4="\xaa" diff --git a/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_text_to_text_layout.dict b/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_text_to_text_layout.dict new file mode 100644 index 0000000..3b3d206 --- /dev/null +++ b/parser-fuzzers/dictionaries/fuzz_cupsfilters_state_text_to_text_layout.dict @@ -0,0 +1,12 @@ +token_0="TXT2TXT1" +token_1="\x00" +token_2="\x01" +token_3="\x02" +token_4="\x03" +token_5="\x0a" +token_6="\x0d" +token_7="\x0d\x0a" +token_8="\x09" +token_9="\x0c" +token_10="\xe2\x82\xac" +token_11="\xf0\x9f\x98\x80" diff --git a/parser-fuzzers/dictionaries/fuzz_cupsfilters_text_to_text_selection_oracle.dict b/parser-fuzzers/dictionaries/fuzz_cupsfilters_text_to_text_selection_oracle.dict new file mode 100644 index 0000000..cfebd0f --- /dev/null +++ b/parser-fuzzers/dictionaries/fuzz_cupsfilters_text_to_text_selection_oracle.dict @@ -0,0 +1,21 @@ +token_0="TXTSEL01" +token_1="TXTPAGE1" +token_2="TXTORD01" +token_3="1-1" +token_4="1-2" +token_5="2-3" +token_6="1-4" +token_7="2-4" +token_8="1,3" +token_9="1-99" +token_10="-2" +token_11="3-" +token_12="page-ranges=" +token_13="all" +token_14="odd" +token_15="even" +token_16="normal" +token_17="reverse" +token_18="Collate" +token_19="\x00" +token_20="\x01" diff --git a/parser-fuzzers/dictionaries/image_headers.dict b/parser-fuzzers/dictionaries/image_headers.dict deleted file mode 100644 index a4709f3..0000000 --- a/parser-fuzzers/dictionaries/image_headers.dict +++ /dev/null @@ -1,11 +0,0 @@ -"RaS2" -"PwgRaster" -"PCLm" -"JFIF" -"PNG" -"IHDR" -"cupsWidth" -"cupsHeight" -"cupsBitsPerColor" -"cupsColorSpace" -"cupsBytesPerLine" diff --git a/parser-fuzzers/dictionaries/ipp_options.dict b/parser-fuzzers/dictionaries/ipp_options.dict deleted file mode 100644 index 18a0fb0..0000000 --- a/parser-fuzzers/dictionaries/ipp_options.dict +++ /dev/null @@ -1,13 +0,0 @@ -"printer-uri" -"job-name" -"document-format" -"media" -"sides" -"copies" -"printer-make-and-model" -"requested-attributes" -"attributes-charset" -"attributes-natural-language" -"application/pdf" -"image/pwg-raster" -"image/urf" diff --git a/parser-fuzzers/dictionaries/mime.dict b/parser-fuzzers/dictionaries/mime.dict deleted file mode 100644 index 1878633..0000000 --- a/parser-fuzzers/dictionaries/mime.dict +++ /dev/null @@ -1,11 +0,0 @@ -"application/pdf" -"application/postscript" -"image/pwg-raster" -"image/urf" -"image/jpeg" -"image/png" -"application/vnd.cups-raster" -"filter" -"cost" -"cupsfilters.types" -"cupsfilters.convs" diff --git a/parser-fuzzers/dictionaries/ppd.dict b/parser-fuzzers/dictionaries/ppd.dict deleted file mode 100644 index 2a3dd54..0000000 --- a/parser-fuzzers/dictionaries/ppd.dict +++ /dev/null @@ -1,15 +0,0 @@ -"*PPD-Adobe:" -"*OpenUI" -"*CloseUI" -"*DefaultPageSize:" -"*PageSize" -"*cupsFilter:" -"*cupsFilter2:" -"*FoomaticRIPCommandLine:" -"*NickName:" -"*ModelName:" -"*Manufacturer:" -"*Product:" -"Letter" -"A4" -"Custom." diff --git a/parser-fuzzers/dictionaries/pwg_bundle.dict b/parser-fuzzers/dictionaries/pwg_bundle.dict deleted file mode 100644 index 4f23e27..0000000 --- a/parser-fuzzers/dictionaries/pwg_bundle.dict +++ /dev/null @@ -1,24 +0,0 @@ -"SMT_PWG_BUNDLE_V1" -"--SMT-PPD--" -"--SMT-OPTIONS--" -"--SMT-DOCUMENT--" -"*PPD-Adobe:" -"*cupsFilter2:" -"*OpenUI" -"*CloseUI" -"PageSize=" -"PageRegion=" -"ColorModel=" -"PrintQuality=" -"MediaType=" -"Duplex=" -"Resolution=" -"Gray" -"Black" -"CMYK" -"RGB" -"Letter" -"A4" -"Small" -"Wide" -"2SaR" diff --git a/parser-fuzzers/dictionaries/pwg_raster.dict b/parser-fuzzers/dictionaries/pwg_raster.dict deleted file mode 100644 index e655242..0000000 --- a/parser-fuzzers/dictionaries/pwg_raster.dict +++ /dev/null @@ -1,34 +0,0 @@ -"2SaR" -"3SaR" -"PwgRaster" -"cupsWidth" -"cupsHeight" -"cupsBitsPerColor" -"cupsBitsPerPixel" -"cupsBytesPerLine" -"cupsColorOrder" -"cupsColorSpace" -"cupsCompression" -"cupsRowCount" -"cupsNumColors" -"cupsPageSize" -"cupsImagingBBox" -"cupsInteger" -"cupsReal" -"Gray" -"RGB" -"CMYK" -"Black" -"DeviceGray" -"DeviceRGB" -"DeviceCMYK" -"PageSize" -"PageRegion" -"ColorModel" -"PrintQuality" -"MediaType" -"Resolution" -"Duplex" -"PDF" -"%%Pages:" -"showpage" diff --git a/parser-fuzzers/docs/afl-plus-plus.md b/parser-fuzzers/docs/afl-plus-plus.md deleted file mode 100644 index dc9bfff..0000000 --- a/parser-fuzzers/docs/afl-plus-plus.md +++ /dev/null @@ -1,45 +0,0 @@ -# AFL++ Workflow - -This project treats AFL++ as the main driver for CLI/forkserver fuzzing. - -## Build Environment - -Print the recommended compiler environment: - -```bash -scripts/afl_build_env.sh -``` - -For an ASan/UBSan AFL++ target, build with: - -```bash -export CC=afl-clang-fast -export CXX=afl-clang-fast++ -export CFLAGS="-g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer" -export CXXFLAGS="-g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer" -export LDFLAGS="-fsanitize=address,undefined" -``` - -## Run Planning - -Generate a dry-run command: - -```bash -scripts/run_afl.sh ppd_ipp_parser A1 harnesses/bin/ppd_ipp_parser -``` - -Launch only when the instrumented binary exists and AFL++ is installed: - -```bash -scripts/run_afl.sh ppd_ipp_parser A1 harnesses/bin/ppd_ipp_parser --execute -``` - -## Config Mapping - -- `A0`: weak seeds only. -- `A1`: weak seeds plus merged target dictionary. -- `A2`: `A1` plus `-c .cmplog`. -- `A3`: `A1` plus `AFL_CUSTOM_MUTATOR_LIBRARY` when configured. -- `A4`: `A3` plus SMT candidates imported from `work/corpus/smt`. - -Private reproducers are never imported into default AFL++ seed directories. diff --git a/parser-fuzzers/docs/architecture.md b/parser-fuzzers/docs/architecture.md deleted file mode 100644 index 88463f2..0000000 --- a/parser-fuzzers/docs/architecture.md +++ /dev/null @@ -1,208 +0,0 @@ -# Layered Architecture - -This project is organized around a three-system loop: - -```text -Generator/SMT system - -> structured seeds and repaired inputs - -> runner or AFL++ - -AFL++ / fuzzer system - -> queue, crashes, hangs, fuzzer_stats - -> feedback import - -Feedback / metrics system - -> shape, crash, corpus, coverage, and metrics summaries - -> next generator round -``` - -SMT is intentionally kept outside AFL++. AFL++ only needs inputs, -dictionaries, a target command, and optionally CmpLog/custom-mutator support. - -## Layer Map - -```text -Layer 0: Core schemas and config - core/models.py - core/validation.py - core/hashing.py - core/experiment.py - core/format_specs.py - -Layer 1: Generator and SMT - generator/solver.py - generator/z3_guard.py - generator/patcher.py - generator/template_synth.py - generator/ppd_templates.py - generator/image_templates.py - generator/structured_templates.py - generator/structure_mutator.py - generator/dimension_expander.py - generator/auto_expand.py - generator/constraint_repair.py - generator/arithmetic_explorer.py - -Layer 2: Execution runner - runner/cli.py - runner/document_harness.py - runner/multitarget_runner.py - runner/cupsfilter.py - runner/ppd_pipeline.py - -Layer 3: AFL++ boundary - afl_integration/afl.py - afl_integration/afl_feedback.py - afl_integration/seed_export.py - scripts/run_afl.sh - scripts/build_afl_template_probe.sh - scripts/run_template_afl_loop.sh - scripts/run_afl_pwg_frontier.sh - scripts/afl_direct_filter_target.sh - scripts/prepare_afl_frontier_corpus.py - scripts/import_afl_frontier_feedback.py - -Layer 4: Feedback, triage, and metrics - feedback/template_feedback.py - feedback/output_feedback.py - feedback/semantic_shapes.py - feedback/crash_avoidance.py - feedback/crash_dedup.py - metrics/run_metrics.py - metrics/loop_metrics.py - metrics/run_set_metrics.py - metrics/run_recovery.py - metrics/baseline_compare.py -``` - -## Data Flow - -### Template-Only Exploration - -```text -configs/parser_targets_*.yaml - -> multitarget-monitor - -> document_harness - -> structured_templates + template_synth - -> target filter process - -> timeline.jsonl + summary.json - -> crash_dedup + run_metrics -``` - -Use this when there is no good AFL++ harness yet, or when the goal is to -measure structured template reachability directly. - -### Standard AFL++ Exploration - -```text -template export / public seeds / SMT corpus - -> AFL++ seed directory - -> afl-fuzz -i ... -o ... [-x dict] [-c cmplog] -- target @@ - -> AFL++ queue/crashes/hangs - -> summarize-run-metrics --afl-output-dir ... -``` - -The standard command builder is `src/parser_fuzzers/afl.py`. It produces regular -AFL++ invocations with `-i`, `-o`, `-m`, `-t`, optional `-x`, optional `-c`, -`-V` when requested, and `@@`. `export-template-seeds` is the bridge from a -template run's retained corpus to AFL++ `-i`. - -The clone-only AFL++ probe harness is intentionally small: - -```text -scripts/build_afl_template_probe.sh - -> afl-clang-fast harnesses/afl_template_probe.c - -> work/afl/bin/template_probe -``` - -It exists to verify that the project uses normal AFL++ instrumentation and -fuzzer_stats. Real CUPS/cups-filters harnesses should use the same AFL++ -boundary but point `--binary` at an instrumented target. - -### AFL++ Frontier Feedback Loop - -```text -template seeds - -> AFL++ run - -> import queue/crashes - -> build template feedback profile - -> generate next template round - -> AFL++ run -``` - -The current implementation is file-backed rather than in-memory. This keeps -each round reproducible and makes it easy to inspect AFL++ artifacts. -`scripts/run_template_afl_loop.sh` runs one template -> AFL++ -> feedback -template cycle and writes `loop_manifest.json` plus `loop_standard_metrics.json`. - -## Layer Boundaries - -Generator/SMT layer: - -- Creates structured inputs. -- Solves or repairs typed fields. -- Should not own long-running process scheduling. -- Should not interpret AFL++ `queue/` or `crashes/` directly. - -Runner layer: - -- Owns target execution. -- Writes per-case artifacts. -- Applies runtime skip and corpus-retention policy. -- Does not need to know how AFL++ mutates bytes. - -AFL++ boundary: - -- Prepares seed directories and dictionaries. -- Starts AFL++ or prints a dry-run command. -- Imports AFL++ output back into runner-compatible feedback cases. -- Does not need to understand SMT internals. - -Feedback/metrics layer: - -- Reads completed runs. -- Extracts semantic shapes, crash signatures, corpus density, and coverage. -- Writes summaries and profiles for the next round. -- Should be side-effect-light except for output reports/profiles. - -## Current AFL++ Status - -There are two AFL++ modes in the repository: - -1. Standard command generation - - `scripts/run_afl.sh` and `src/parser_fuzzers/afl.py` generate normal AFL++ - commands. This is the preferred path for real instrumented harnesses. - -2. Direct filter bridge - - `scripts/run_afl_pwg_frontier.sh` and - `scripts/afl_direct_filter_target.sh` can drive existing filters through a - wrapper. When `AFL_DIRECT_INSTRUMENTED=1` is not set, this uses AFL++ dumb - mode (`-n`). That is useful for fast black-box probing, but it is not the - same as coverage-guided AFL++. - -## Compatibility Imports - -The implementation now lives in layer subpackages: - -```text -src/parser_fuzzers/core/ -src/parser_fuzzers/generator/ -src/parser_fuzzers/runner/ -src/parser_fuzzers/afl_integration/ -src/parser_fuzzers/feedback/ -src/parser_fuzzers/metrics/ -``` - -The historical flat imports remain available through compatibility wrappers. -For example, both imports work: - -```python -from parser_fuzzers.solver import solve_event -from parser_fuzzers.generator.solver import solve_event -``` - -New code should prefer the layer path when it is already clear which layer it -belongs to. Existing scripts and tests can continue to use the flat imports -until they are mechanically updated. diff --git a/parser-fuzzers/docs/branch-events.md b/parser-fuzzers/docs/branch-events.md deleted file mode 100644 index b94ed56..0000000 --- a/parser-fuzzers/docs/branch-events.md +++ /dev/null @@ -1,56 +0,0 @@ -# Branch Event and Solver Result Format - -The SMT patcher consumes one local branch event at a time. The event describes -which input field should be patched so the branch condition becomes true. - -## Branch Event JSON - -Required fields: - -- `target_id`: target or harness identifier. -- `input_path`: path to the concrete input used for tracing. -- `input_sha256`: SHA-256 of the concrete input. -- `offset`: byte offset of the symbolic field. -- `width`: field width in bytes; first version supports `1`, `2`, `4`, `8`. -- `endianness`: `little` or `big`. -- `signed`: boolean annotation for the source comparison. -- `op`: one of `eq`, `ne`, `ult`, `ule`, `ugt`, `uge`, `slt`, `sle`, `sgt`, `sge`. -- `rhs`: integer right-hand side, decimal or hex. -- `description`: human-readable branch description. - -Example: - -```json -{ - "target_id": "synthetic_eq_u8", - "input_path": "work/smoke/input.bin", - "input_sha256": "64 hex characters", - "offset": 0, - "width": 1, - "endianness": "little", - "signed": false, - "op": "eq", - "rhs": 65, - "description": "first byte must become ASCII A" -} -``` - -## Solver Result JSON - -Required fields: - -- `status`: `sat`, `unsat`, or `already_satisfied`. -- `solver_ms`: wall-clock solver time in milliseconds. -- `patches`: list of byte patches. -- `reason`: backend or failure summary. -- `event`: original branch event. - -Each patch contains: - -- `offset` -- `old_hex` -- `new_hex` -- `width` - -The patcher verifies `input_sha256` and `old_hex` before writing a generated -candidate into `work/corpus/smt`. diff --git a/parser-fuzzers/docs/bug-suite.md b/parser-fuzzers/docs/bug-suite.md deleted file mode 100644 index efaf74f..0000000 --- a/parser-fuzzers/docs/bug-suite.md +++ /dev/null @@ -1,37 +0,0 @@ -# Local Bug Metadata - -The public tree does not include concrete vulnerability metadata, private -reproducers, local issue reports, or minimized crash inputs. - -For private evaluation, the optional `bugs//meta.yaml` interface can be -used to track local ground truth. Those files should stay outside public pull -requests unless the issue is already safe to disclose. - -Expected metadata fields are: - -- `id` -- `title` -- `component` -- `bug_type` -- `target_component` -- `oracle.reached` -- `oracle.triggered` -- `oracle.detected` -- `poc_path` -- `known_poc_allowed_in_seed` -- `timeout_sec` -- `memory_mb` -- `report_path` - -`known_poc_allowed_in_seed` should remain `false` for normal evaluation so -seed corpora measure discovery rather than replay. - -## Oracle Levels - -- `Reached`: execution reaches the target-relevant function, block, or state. -- `Triggered`: the issue precondition is satisfied. -- `Detected`: sanitizer, assertion, custom oracle, crash signal, or other - detector reports the candidate. - -This distinction avoids counting raw crashes as bugs and separates exploration -gains from oracle strength. diff --git a/parser-fuzzers/docs/coverage-discovery.md b/parser-fuzzers/docs/coverage-discovery.md deleted file mode 100644 index d6b3d4e..0000000 --- a/parser-fuzzers/docs/coverage-discovery.md +++ /dev/null @@ -1,101 +0,0 @@ -# Coverage Discovery - -This mode is for exploring beyond already triaged shallow crashes. - -## Entrypoints - -Avoid-only coverage discovery on the general target set: - -```bash -scripts/run_coverage_discovery_campaign.sh 300 4 5 -``` - -Deep coverage discovery with expanded structural mutation, runtime skip, and -novelty scheduling: - -```bash -scripts/run_deep_coverage_campaign.sh 300 4 5 -``` - -## Mechanics - -- Known shallow crash predicates are skipped before execution. -- Each executed case extracts feature tokens from target id, PPD kind, document - kind, document header fields, oracle, return code, and selected stderr state - lines. -- Coverage-oriented PPD, CUPS Raster, PWG Raster, and image templates are filled - by SMT slot synthesis. The solver chooses typed fields and cross-field - relationships; the format builders still write the actual PPD/document bytes. -- Cases that introduce at least one new feature are retained under - `corpus/interesting/`. -- Crash signatures are normalized and quarantined. The first case per unique - signature is copied under `quarantine/unique/`; repeats are recorded in - `quarantine/repeats.jsonl`. -- In deep mode, a crash also suppresses later cases with the same target, PPD - template slot, and document template slot. This avoids cycling on a confirmed - shallow shape while leaving other structures in the same parser enabled. -- In deep mode, target selection uses a novelty-weighted fair scheduler: - targets that keep producing new feature tokens receive more executions, while - targets dominated by repeated crashes, timeouts, or runtime-suppressed shapes - are gradually deprioritized. -- Active crash avoidance can softly deweight targets with seeded historical - suppression pressure while still reserving periodic probe slots. The default - exploratory command line can set `SMT_FUZZER_AVOIDANCE_PROBE_INTERVAL=32`, - `SMT_FUZZER_AVOIDANCE_SKIP_PROBE_RATE=0.06`, and - `SMT_FUZZER_AVOIDANCE_SCHEDULER_PENALTY_CAP=2.0`. -- In deep mode, ordinary per-case directories are pruned after their timeline - record is written. Crash, timeout, and new-feature cases are kept. -- Large binary stdout is discarded by default; every case keeps - `candidate.ppd`, `document.bin`, `command.txt`, `stderr.txt`, and `meta.json` - only when the case is retained or needs triage. -- `summary.concise.json` reports scheduler settings and per-target stats; - `discovery_state.json` records suppressed shapes and target scheduler state. - -## Current Avoid Predicates - -- `known-rastertoescpx-dotrowstep-zero-fpe` -- `known-libppd-65536dpi-fpe` - -These are deliberately narrow. They remove already triaged shallow failures -without hiding new sanitizer signatures from the same target. - -## Optional LLVM Coverage - -The Python feature-retention path works with the current ASan binaries. For -real LLVM source-based coverage, rebuild the target with: - -```bash -source scripts/llvm_coverage_env.sh -``` - -Then run a campaign with: - -```bash -SMT_FUZZER_LLVM_PROFILE_DIR=work/llvm-profraw scripts/run_deep_coverage_campaign.sh 300 4 5 -``` - -Merge profiles: - -```bash -scripts/merge_llvm_coverage.sh work/llvm-profraw /path/to/filter work/llvm-coverage -``` - -Run-level metrics can be summarized with: - -```bash -python3 -m parser_fuzzers.cli summarize-run-metrics \ - --run-dir work/auto-hybrid/ \ - --llvm-coverage-json work/llvm-coverage/coverage.json \ - --output work/metrics/.json -``` - -## Optional AFL++ Coverage - -For AFL++ edge coverage, rebuild the target with: - -```bash -source scripts/afl_coverage_env.sh -``` - -Then use the existing AFL++ launch scripts or the generated -`corpus/interesting/` inputs as AFL seed material. diff --git a/parser-fuzzers/docs/evaluation.md b/parser-fuzzers/docs/evaluation.md deleted file mode 100644 index a4eb0ad..0000000 --- a/parser-fuzzers/docs/evaluation.md +++ /dev/null @@ -1,59 +0,0 @@ -# Evaluation Plan - -The project tracks four goals: - -- `G1`: general source and edge coverage improvement. -- `G2`: target parser/filter-option coverage improvement. -- `G3`: known bug reached/triggered/detected improvement. -- `G4`: measurable SMT contribution. - -## Experiment Matrix - -- `A0`: vanilla. -- `A1`: dictionary. -- `A2`: dictionary + CmpLog. -- `A3`: dictionary + grammar/custom mutator. -- `A4`: dictionary + grammar/custom mutator + SMT patcher. - -Use the same seeds, resource limits, target binaries, and coverage replay -binary across all configurations. - -## AFL++ Integration - -`configs/afl.yaml` controls AFL++ defaults: - -- fuzzer and compiler wrapper names. -- weak seed directory. -- SMT corpus import directory. -- AFL++ work/output directories. -- timeout and memory limit. -- optional custom mutator library. - -Use `python3 -m parser_fuzzers.cli afl-prepare` to materialize an AFL++ run -without launching it. This command merges the per-target dictionaries into one -AFL++ dictionary, prepares `work/afl/input//`, imports -`work/corpus/smt` for `A4`, and prints the exact `afl-fuzz` command. - -`python3 -m parser_fuzzers.cli afl-run ... --execute` is the only command that -starts AFL++. It checks that `afl-fuzz` and the target binary exist before -launching. - -## Metrics - -- line/function/branch coverage from a neutral coverage binary. -- target function and target branch coverage. -- reached/triggered/detected time for every known bug. -- solver attempts, sat/unsat/timeout, solver time, patched fields. -- SMT coverage yield: solver inputs with new coverage / solver inputs. -- SMT bug yield: solver inputs triggering bugs / solver inputs. -- cost per new edge: total solver time / new edges from solver inputs. - -Use `python3 -m parser_fuzzers.cli summarize-run-metrics --run-dir ` -to generate the common per-run JSON metrics payload. It can also attach AFL++ -`fuzzer_stats` and LLVM `coverage.json` data. - -The first smoke run can be `5 repetitions x 6h`. A stronger internal run should -use `10 repetitions x 24h`. - -See `docs/next-stage-evaluation.md` for coverage, historical-reproducer, and -target-expansion policy. diff --git a/parser-fuzzers/docs/expanded-parser-support.md b/parser-fuzzers/docs/expanded-parser-support.md deleted file mode 100644 index 138d73b..0000000 --- a/parser-fuzzers/docs/expanded-parser-support.md +++ /dev/null @@ -1,86 +0,0 @@ -# Expanded Parser Support - -Date: 2026-06-01 - -The coverage parser campaign now includes 20 direct-filter targets across these -input families: - -- CUPS Raster: `rastertopclx`, `rastertoescpx`, `rastertops` -- PWG Raster: `pwgtoraster`, `pwgtopdf`, `pwgtopclm` -- PDF: `pdftopdf`, `pdftops`, `pdftoraster`, `mupdftopwg` -- Image: `imagetoraster`, `imagetopdf`, `imagetops` -- Text: `texttopdf`, `texttotext` -- PostScript/Ghostscript wrappers: `gstoraster`, `gstopdf`, `gstopxl` -- CUPS command streams: `commandtoescpx`, `commandtopclx` - -## Inputs - -`document_harness.py` now generates extension-aware structured documents: - -- `.ras` CUPS Raster -- `.pwg` PWG Raster -- `.pdf` minimal PDF with valid xref -- `.ps` minimal PostScript -- `.png` plus PNM variants for image parsers -- `.txt` text parser inputs -- `.cmd` CUPS command inputs - -The runner writes the correct `document.` name per case so filters that -infer type from the filename are exercised properly. Replay helpers search the -same extension set. - -## Current Run Command - -```bash -scripts/run_deep_coverage_campaign.sh 300 4 5 -``` - -Equivalent CLI: - -```bash -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config configs/parser_targets_coverage.yaml \ - --work-root work/deep-coverage \ - --workers 4 \ - --timeout-sec 5 \ - --duration-sec 300 \ - --discard-stdout \ - --discovery-mode coverage -``` - -## Smoke Result - -Command: - -```bash -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config configs/parser_targets_coverage.yaml \ - --work-root work/expanded-parser-smoke \ - --workers 4 \ - --cases-per-target 1 \ - --timeout-sec 5 \ - --discard-stdout \ - --discovery-mode coverage -``` - -Latest smoke directory: - -```text -work/expanded-parser-smoke/20260601-130736 -``` - -Result: - -- Targets: 20 -- Cases: 20 -- Reached expected filter: 20 -- Valid PPDs: 20 -- Crashes: 0 -- Timeouts: 0 -- Retained feature cases: 20 -- Coverage features: 90 - -One expected limitation remains: `pwgtopclm` currently reaches the wrapper but -returns an error because the synthetic PPD/input does not yet provide the -printer IPP attributes needed for PCLm output. It is retained as a parser entry -point, but deeper PCLm exploration needs an IPP-attribute template. diff --git a/parser-fuzzers/docs/local-cups-filters.md b/parser-fuzzers/docs/local-cups-filters.md deleted file mode 100644 index fffe9ef..0000000 --- a/parser-fuzzers/docs/local-cups-filters.md +++ /dev/null @@ -1,98 +0,0 @@ -# Local CUPS/cups-filters Testing - -The Python smoke path does not need CUPS. Parser campaigns need either system -filters or a local OpenPrinting build. Crash triage should use a local ASan -build; system filters are useful for reachability smoke only. - -## Path 1: System Filters Smoke - -Use this path when your distribution already provides CUPS and cups-filters. -This does not guarantee sanitizer diagnostics: - -```bash -scripts/install_ubuntu_deps.sh --system-filters -y -scripts/check_cups_filters_targets.sh -scripts/run_multitarget_ppd_fuzz.sh 1 4 -``` - -The default filter root is `/usr/lib/cups/filter`. Override it when your system -uses another location: - -```bash -export SMT_FUZZER_FILTER_ROOT=/usr/libexec/cups/filter -scripts/check_cups_filters_targets.sh "$SMT_FUZZER_FILTER_ROOT" -scripts/run_local_cups_filters_campaign.sh "$SMT_FUZZER_FILTER_ROOT" 60 4 5 -``` - -## Path 2: Isolated Local ASan Build - -Recommended for crash discovery and issue reports. This keeps source, -intermediate objects, and the install prefix under `work/openprinting-asan/`. -It does not install into `/usr`, `/usr/local`, or any system path. - -Print a build plan for `libcupsfilters`, `libppd`, and `cups-filters`: - -```bash -scripts/install_ubuntu_deps.sh --asan-build -y -scripts/print_cups_filters_build_plan.sh -``` - -The script prints commands instead of running them. This keeps dependency -installation and network fetches explicit. The generated plan uses: - -```bash -work/openprinting-asan/src -work/openprinting-asan/prefix -``` - -After building, run: - -```bash -scripts/run_asan_cups_filters_campaign.sh work/openprinting-asan 60 4 5 -``` - -For the expanded 20-target profile: - -```bash -scripts/run_asan_cups_filters_campaign.sh work/openprinting-asan 300 4 5 configs/parser_targets_coverage.yaml -``` - -The ASan runner exports these variables for you: - -```bash -SMT_FUZZER_FILTER_ROOT=work/openprinting-asan/src/cups-filters -SMT_FUZZER_LD_LIBRARY_PATH=work/openprinting-asan/src/libcupsfilters/.libs:work/openprinting-asan/src/libppd/.libs:work/openprinting-asan/prefix/lib:work/openprinting-asan/prefix/lib64 -SMT_FUZZER_ASSUME_ASAN=1 -ASAN_OPTIONS=abort_on_error=0:detect_leaks=0:symbolize=1:exitcode=86 -``` - -## Outputs - -Each run creates a timestamped directory under `work/` with: - -- `run_manifest.json`: config, target list, and guidance policy. -- `commands.txt`: exact command line for every executed case. -- `timeline.jsonl`: one compact JSON record per case. -- `summary.json` and `summary.concise.json`: aggregate counters. -- per-case `candidate.ppd`, `document.*`, `stderr.txt`, `stdout.bin`, and - `meta.json`. -- `crash_dedup.json` and `crash_dedup.md` when unique crash signatures are - found. - -## Recent Local Smoke Result - -In the development environment, a short discovery-only run used: - -```bash -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config configs/parser_targets.yaml \ - --work-root work/discovery-only-smoke \ - --workers 2 \ - --cases-per-target 1 \ - --timeout-sec 5 \ - --discard-stdout -``` - -It reached all four configured targets, produced four valid PPDs, and observed -one `signal 11` from the generic direct `rastertopclx` CUPS Raster path. That -case did not use a string-format payload or a known reproduction input. diff --git a/parser-fuzzers/docs/next-stage-evaluation.md b/parser-fuzzers/docs/next-stage-evaluation.md deleted file mode 100644 index 6698e16..0000000 --- a/parser-fuzzers/docs/next-stage-evaluation.md +++ /dev/null @@ -1,110 +0,0 @@ -# Next-Stage Evaluation - -This note records the next practical extensions for the SMT-assisted fuzzing -pipeline. - -## 1. Coverage Metrics - -Use `summary.concise.json` for fast campaign-level metrics: - -```bash -python3 -m parser_fuzzers.cli summarize-run-metrics \ - --run-dir work/auto-iterative-active-avoid-15m/20260605-030610 \ - --output work/metrics/active-avoid-15m.json -``` - -The summary reports: - -- executed cases, retained cases, skipped cases, crashes, and timeouts -- retained density and crash density -- retained/features per minute -- deduplicated crash clusters -- `z3-structure-avoid` timeline records -- per-target retained/crash densities - -For AFL++ runs, add the AFL output directory: - -```bash -python3 -m parser_fuzzers.cli summarize-run-metrics \ - --run-dir work/smt-run \ - --afl-output-dir work/afl/out/target/config \ - --output work/metrics/smt-afl.json -``` - -For LLVM source coverage, build the targets with the LLVM coverage environment, -run a campaign, merge profiles, then attach `coverage.json`: - -```bash -source scripts/llvm_coverage_env.sh -SMT_FUZZER_ENABLE_LLVM_PROFILES=1 PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config configs/parser_targets_auto_hybrid.yaml \ - --work-root work/llvm-auto-hybrid \ - --workers 4 \ - --timeout-sec 5 \ - --duration-sec 900 \ - --max-run-gb 10 \ - --discard-stdout \ - --discovery-mode coverage \ - --scheduler novelty \ - --runtime-skip \ - --summary-mode concise \ - --prune-uninteresting -scripts/merge_llvm_coverage.sh work/llvm-profraw /path/to/filter work/llvm-coverage -python3 -m parser_fuzzers.cli summarize-run-metrics \ - --run-dir work/llvm-auto-hybrid/ \ - --llvm-coverage-json work/llvm-coverage/coverage.json \ - --output work/metrics/llvm.json -``` - -Fuzz Introspector can be added as a later reporting backend. The immediate -contract should stay the same: one JSON metrics file per run, plus a comparison -table across A0-A4. - -## 2. Private Reproducer Corpora - -Private reproducers should not silently enter the default fuzz seed corpus. -They are too strong for a fair discovery experiment and can make a run look -better without improving exploration. - -Use three separate buckets: - -- `seeds/public/`: weak generic seeds only. -- `work/repro-corpus/`: private reproducers for oracle validation. -- `work/frontier-seeds/`: retained, non-crashing, coverage-interesting cases - generated by prior campaigns. - -Recommended experiment labels: - -- `A0-A4`: no private reproducers in seeds. -- `A4-frontier`: imports retained non-crashing frontier cases. -- `A4-repro-seeded`: explicitly includes private reproducers, reported - separately as a regression/oracle experiment, not as discovery. - -This keeps the original policy intact: `known_poc_allowed_in_seed: false` -remains the default for discovery runs. - -## 3. Parser And Filter Target Expansion - -The current strong paths are image/PDF/PostScript/text/command direct filters -with PPD/job-option variation. The next targets should be selected by marginal -coverage gain, not only by crash count. - -Prioritize: - -- Underexplored converters: `mupdftopwg`, `pdftopdf`, `pdftops`, - `gstopdf`, `gstopxl`, `texttopdf`, `texttotext`. -- Raster families: CUPS Raster and PWG Raster into `rastertopclx`, - `rastertoescpx`, `rastertopwg`, `pwgtopdf`, `pwgtopclm`, `pwgtoraster`. -- Command streams: `commandtoescpx` and `commandtopclx`. - -For each target family, add: - -- a valid minimal document template -- one boundary template -- one structure-mutation template -- target-specific dictionaries for AFL++ -- at least one neutral coverage replay binary - -The main acceptance metric is not "did it crash quickly". A target is useful if -it increases retained density, source/edge coverage, or reaches previously -unseen parser states without relying on known reproducers. diff --git a/parser-fuzzers/docs/oss-fuzz-comparison.md b/parser-fuzzers/docs/oss-fuzz-comparison.md deleted file mode 100644 index 295a103..0000000 --- a/parser-fuzzers/docs/oss-fuzz-comparison.md +++ /dev/null @@ -1,80 +0,0 @@ -# OSS-Fuzz Baseline Comparison - -This project can compare the semantic SMT-assisted pipeline against an -OSS-Fuzz-style local baseline. - -## Docker Boundary - -The official OSS-Fuzz workflow is container based. On a machine with Docker, -the usual cups-filters commands are: - -```bash -cd /data/pre-gsoc/oss-fuzz -python3 infra/helper.py build_image cups-filters -python3 infra/helper.py build_fuzzers --sanitizer address --engine libfuzzer cups-filters -python3 infra/helper.py run_fuzzer cups-filters -python3 infra/helper.py coverage cups-filters -``` - -If Docker is unavailable, those commands cannot run locally. The fallback is a -local fair baseline: - -- same cups-filters build and same target config as the optimized run -- same time budget, worker count, timeout, and generated input families -- baseline: coverage discovery, round-robin scheduling, no runtime crash-shape - suppression -- optimized: coverage discovery, novelty scheduling, runtime crash-shape - suppression, semantic suppression, and deterministic skip probes - -This is not an official OSS-Fuzz execution. It is a reproducible local control -group that uses the same metrics contract. - -## Run - -```bash -scripts/run_baseline_comparison.sh 60 4 5 -``` - -The script uses `work/parser_targets_cold_semantic_llvm.yaml` by default. Set -`SMT_FUZZER_COMPARE_CONFIG` to use another target config. - -To keep novelty scheduling but remove crash-reduction guidance: - -```bash -SMT_FUZZER_COMPARE_POLICY=no-crash-avoidance \ - scripts/run_baseline_comparison.sh 60 4 5 -``` - -This disables runtime crash-shape suppression, semantic suppression, -generalized skip, deterministic skip probes, short-PNG abort skipping, and the -novelty scheduler's crash/repeat-crash penalties. - -Outputs are written under: - -```text -work/baseline-comparison// - baseline// - optimized// - coverage/baseline/ - coverage/optimized/ - metrics/baseline.json - metrics/optimized.json - comparison.json - comparison.md -``` - -## Metrics - -The comparison table reports: - -- executed cases -- retained coverage-interesting cases -- semantic coverage feature count -- semantic features per minute -- retained density -- crash density -- unique crash signatures -- LLVM function, line, and branch coverage when coverage profiles are available - -Crash count alone is not the main metric. Runtime suppression can reduce repeat -crashes while improving retained density or coverage depth. diff --git a/parser-fuzzers/fonts.conf b/parser-fuzzers/fonts.conf new file mode 100644 index 0000000..bff2ae6 --- /dev/null +++ b/parser-fuzzers/fonts.conf @@ -0,0 +1,7 @@ + + + + fonts + /tmp/fontconfig-cache + + diff --git a/parser-fuzzers/harnesses/README.md b/parser-fuzzers/harnesses/README.md deleted file mode 100644 index d88892d..0000000 --- a/parser-fuzzers/harnesses/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Harness Notes - -This directory holds interface templates, not production CUPS harnesses. - -The first implementation target is the Python-side SMT data flow: - -1. a harness or tracer emits a branch-event JSON file; -2. `solve-event` emits a solver-result JSON file; -3. `patch-input` creates a candidate input in `work/corpus/smt`; -4. AFL++/libFuzzer imports or replays that candidate. - -Future CUPS harnesses should keep private reproducers out of the initial corpus -and use `bugs/*/meta.yaml` only for local oracle validation. diff --git a/parser-fuzzers/harnesses/afl_pwg_bundle_harness.c b/parser-fuzzers/harnesses/afl_pwg_bundle_harness.c deleted file mode 100644 index 640cae8..0000000 --- a/parser-fuzzers/harnesses/afl_pwg_bundle_harness.c +++ /dev/null @@ -1,310 +0,0 @@ -#define _GNU_SOURCE - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#define BUNDLE_MAGIC "SMT_PWG_BUNDLE_V1\n" -#define PPD_MARK "--SMT-PPD--\n" -#define OPTIONS_MARK "--SMT-OPTIONS--\n" -#define DOCUMENT_MARK "--SMT-DOCUMENT--\n" -#define MAX_PPD_BYTES (512 * 1024) -#define MAX_OPTIONS_BYTES 4096 -#define MAX_DOCUMENT_BYTES (2 * 1024 * 1024) - -static int JobCanceled = 0; - -static void cancel_job(int sig) { - (void)sig; - JobCanceled = 1; -} - -static const unsigned char *find_bytes(const unsigned char *haystack, - size_t haystack_len, - const char *needle) { - size_t needle_len = strlen(needle); - if (needle_len == 0 || haystack_len < needle_len) { - return NULL; - } - for (size_t i = 0; i <= haystack_len - needle_len; i++) { - if (memcmp(haystack + i, needle, needle_len) == 0) { - return haystack + i; - } - } - return NULL; -} - -static unsigned char *read_file(const char *path, size_t *out_len) { - FILE *fp = fopen(path, "rb"); - unsigned char *buf = NULL; - long size; - - *out_len = 0; - if (!fp) { - return NULL; - } - if (fseek(fp, 0, SEEK_END) != 0) { - fclose(fp); - return NULL; - } - size = ftell(fp); - if (size < 0) { - fclose(fp); - return NULL; - } - if (fseek(fp, 0, SEEK_SET) != 0) { - fclose(fp); - return NULL; - } - buf = (unsigned char *)malloc((size_t)size + 1); - if (!buf) { - fclose(fp); - return NULL; - } - if (size > 0 && fread(buf, 1, (size_t)size, fp) != (size_t)size) { - free(buf); - fclose(fp); - return NULL; - } - fclose(fp); - buf[size] = 0; - *out_len = (size_t)size; - return buf; -} - -static int write_file(const char *path, const unsigned char *data, size_t len) { - int fd = open(path, O_CREAT | O_TRUNC | O_WRONLY, 0600); - ssize_t written; - size_t offset = 0; - - if (fd < 0) { - return -1; - } - while (offset < len) { - written = write(fd, data + offset, len - offset); - if (written <= 0) { - close(fd); - return -1; - } - offset += (size_t)written; - } - close(fd); - return 0; -} - -static int copy_file(const char *src, const char *dst) { - size_t len = 0; - unsigned char *data = read_file(src, &len); - int ok; - - if (!data) { - return -1; - } - ok = write_file(dst, data, len); - free(data); - return ok; -} - -static char *sanitize_options(const unsigned char *data, size_t len) { - size_t out_len = len < MAX_OPTIONS_BYTES ? len : MAX_OPTIONS_BYTES; - char *out = (char *)malloc(out_len + 1); - if (!out) { - return NULL; - } - for (size_t i = 0; i < out_len; i++) { - unsigned char c = data[i]; - if (c == 0 || c == '\n' || c == '\r' || c == '\t') { - out[i] = ' '; - } else if (c < 32 || c > 126) { - out[i] = '_'; - } else { - out[i] = (char)c; - } - } - out[out_len] = 0; - return out; -} - -static int parse_bundle(const unsigned char *data, size_t len, - const unsigned char **ppd, size_t *ppd_len, - const unsigned char **options, size_t *options_len, - const unsigned char **document, - size_t *document_len) { - const unsigned char *ppd_mark; - const unsigned char *options_mark; - const unsigned char *document_mark; - const unsigned char *ppd_start; - const unsigned char *options_start; - const unsigned char *document_start; - - *ppd = NULL; - *ppd_len = 0; - *options = NULL; - *options_len = 0; - *document = data; - *document_len = len; - - if (len < strlen(BUNDLE_MAGIC) || - memcmp(data, BUNDLE_MAGIC, strlen(BUNDLE_MAGIC)) != 0) { - return 0; - } - - ppd_mark = find_bytes(data, len, PPD_MARK); - if (!ppd_mark) { - return 0; - } - options_mark = find_bytes(ppd_mark, len - (size_t)(ppd_mark - data), - OPTIONS_MARK); - if (!options_mark) { - return 0; - } - document_mark = find_bytes(options_mark, - len - (size_t)(options_mark - data), - DOCUMENT_MARK); - if (!document_mark) { - return 0; - } - - ppd_start = ppd_mark + strlen(PPD_MARK); - options_start = options_mark + strlen(OPTIONS_MARK); - document_start = document_mark + strlen(DOCUMENT_MARK); - - *ppd = ppd_start; - *ppd_len = (size_t)(options_mark - ppd_start); - *options = options_start; - *options_len = (size_t)(document_mark - options_start); - *document = document_start; - *document_len = len - (size_t)(document_start - data); - - if (*ppd_len > MAX_PPD_BYTES) { - *ppd_len = MAX_PPD_BYTES; - } - if (*document_len > MAX_DOCUMENT_BYTES) { - *document_len = MAX_DOCUMENT_BYTES; - } - return 1; -} - -static void remove_temp_files(const char *dir) { - char path[4096]; - snprintf(path, sizeof(path), "%s/candidate.ppd", dir); - unlink(path); - snprintf(path, sizeof(path), "%s/document.pwg", dir); - unlink(path); - rmdir(dir); -} - -int main(int argc, char **argv) { - unsigned char *input = NULL; - size_t input_len = 0; - const unsigned char *ppd = NULL; - const unsigned char *options = NULL; - const unsigned char *document = NULL; - size_t ppd_len = 0; - size_t options_len = 0; - size_t document_len = 0; - char tmp_template[4096]; - char ppd_path[4096]; - char document_path[4096]; - char *tmpdir = NULL; - char *job_options = NULL; - const char *fallback_ppd; - int devnull; - int ret; - char *filter_argv[8]; - cf_filter_out_format_t outformat = CF_FILTER_OUT_FORMAT_PDF; - - if (argc != 2) { - return 0; - } - - signal(SIGTERM, cancel_job); - input = read_file(argv[1], &input_len); - if (!input) { - return 0; - } - - parse_bundle(input, input_len, &ppd, &ppd_len, &options, &options_len, - &document, &document_len); - - snprintf(tmp_template, sizeof(tmp_template), "%s/smt-pwg-bundle.XXXXXX", - getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); - tmpdir = mkdtemp(tmp_template); - if (!tmpdir) { - free(input); - return 0; - } - snprintf(ppd_path, sizeof(ppd_path), "%s/candidate.ppd", tmpdir); - snprintf(document_path, sizeof(document_path), "%s/document.pwg", tmpdir); - - fallback_ppd = getenv("SMT_AFL_BUNDLE_FALLBACK_PPD"); - if (ppd && ppd_len > 0) { - if (write_file(ppd_path, ppd, ppd_len) != 0) { - remove_temp_files(tmpdir); - free(input); - return 0; - } - } else if (fallback_ppd && copy_file(fallback_ppd, ppd_path) == 0) { - /* fallback PPD copied */ - } else { - remove_temp_files(tmpdir); - free(input); - return 0; - } - - if (!document || document_len == 0 || - write_file(document_path, document, document_len) != 0) { - remove_temp_files(tmpdir); - free(input); - return 0; - } - - if (options && options_len > 0) { - job_options = sanitize_options(options, options_len); - } else { - job_options = strdup("PageSize=Letter ColorModel=Gray PrintQuality=Normal MediaType=Plain"); - } - if (!job_options) { - remove_temp_files(tmpdir); - free(input); - return 0; - } - - setenv("PPD", ppd_path, 1); - setenv("CONTENT_TYPE", "application/vnd.cups-pwg", 0); - setenv("FINAL_CONTENT_TYPE", "application/pdf", 0); - setenv("PRINTER", "parser-fuzzers", 0); - setenv("DEVICE_URI", "file:/dev/null", 0); - - devnull = open("/dev/null", O_WRONLY); - if (devnull >= 0) { - dup2(devnull, STDOUT_FILENO); - close(devnull); - } - - filter_argv[0] = (char *)"pwgtopdf-bundle"; - filter_argv[1] = (char *)"1"; - filter_argv[2] = (char *)"afl"; - filter_argv[3] = (char *)"afl"; - filter_argv[4] = (char *)"1"; - filter_argv[5] = job_options; - filter_argv[6] = document_path; - filter_argv[7] = NULL; - - ret = ppdFilterCUPSWrapper(7, filter_argv, cfFilterPWGToPDF, &outformat, - &JobCanceled); - (void)ret; - - free(job_options); - remove_temp_files(tmpdir); - free(input); - return 0; -} diff --git a/parser-fuzzers/harnesses/afl_template_probe.c b/parser-fuzzers/harnesses/afl_template_probe.c deleted file mode 100644 index 02f1c80..0000000 --- a/parser-fuzzers/harnesses/afl_template_probe.c +++ /dev/null @@ -1,106 +0,0 @@ -#include -#include -#include -#include - -static volatile unsigned sink; - -static int has(const uint8_t *data, size_t size, const char *needle) { - size_t n = strlen(needle); - if (n == 0 || size < n) { - return 0; - } - for (size_t i = 0; i + n <= size; ++i) { - if (memcmp(data + i, needle, n) == 0) { - return 1; - } - } - return 0; -} - -static uint32_t le32(const uint8_t *p) { - return ((uint32_t)p[0]) | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); -} - -static void probe_pwg_or_cups(const uint8_t *data, size_t size) { - if (size < 64) { - return; - } - if (memcmp(data, "2SaR", 4) == 0) { - sink += 1; - } else if (memcmp(data, "3SaR", 4) == 0) { - sink += 2; - } else { - return; - } - - for (size_t off = 0; off + 16 <= size && off < 4096; off += 4) { - uint32_t value = le32(data + off); - if (value == 0) sink += 3; - if (value == 1) sink += 5; - if (value == 5) sink += 7; - if (value == 72 || value == 300 || value == 600 || value == 720 || value == 1200) sink += 11; - if (value == 0x7fffffffU || value == 0xffffffffU) sink += 13; - if (value > 0 && value < 65536 && (value % 3) == 0) sink += 17; - } - - if (has(data, size, "PageSize")) sink += 19; - if (has(data, size, "ColorModel")) sink += 23; - if (has(data, size, "DeviceRGB")) sink += 29; -} - -static void probe_text_formats(const uint8_t *data, size_t size) { - if (has(data, size, "*PPD-Adobe:")) sink += 31; - if (has(data, size, "*cupsFilter:")) sink += 37; - if (has(data, size, "*cupsFilter2:")) sink += 41; - if (has(data, size, "*OpenUI")) sink += 43; - if (has(data, size, "rastertopclx")) sink += 47; - if (has(data, size, "pwgtopdf")) sink += 53; - if (has(data, size, "imagetops")) sink += 59; - if (has(data, size, "%PDF")) sink += 61; - if (has(data, size, "%%Pages:")) sink += 67; - if (has(data, size, "showpage")) sink += 71; - if (has(data, size, "P1\n") || has(data, size, "P2\n") || has(data, size, "P3\n")) sink += 73; - if (has(data, size, "P4\n") || has(data, size, "P5\n") || has(data, size, "P6\n")) sink += 79; -} - -int main(int argc, char **argv) { - if (argc != 2) { - return 2; - } - FILE *fp = fopen(argv[1], "rb"); - if (!fp) { - return 2; - } - if (fseek(fp, 0, SEEK_END) != 0) { - fclose(fp); - return 2; - } - long length = ftell(fp); - if (length < 0) { - fclose(fp); - return 2; - } - rewind(fp); - size_t size = (size_t)length; - uint8_t *data = (uint8_t *)malloc(size ? size : 1); - if (!data) { - fclose(fp); - return 2; - } - size_t read_bytes = fread(data, 1, size, fp); - fclose(fp); - if (read_bytes != size) { - free(data); - return 2; - } - - probe_pwg_or_cups(data, size); - probe_text_formats(data, size); - - if (size >= 4 && memcmp(data, "CRSH", 4) == 0) { - abort(); - } - free(data); - return 0; -} diff --git a/parser-fuzzers/harnesses/branch_event_template.cc b/parser-fuzzers/harnesses/branch_event_template.cc deleted file mode 100644 index 13603ed..0000000 --- a/parser-fuzzers/harnesses/branch_event_template.cc +++ /dev/null @@ -1,43 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -static std::string HexByte(uint8_t value) { - std::ostringstream stream; - stream << std::hex << std::setfill('0') << std::setw(2) - << static_cast(value); - return stream.str(); -} - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - if (size < 1) { - return 0; - } - - const char *event_path = std::getenv("SMT_FUZZER_EVENT_OUT"); - if (event_path != nullptr && data[0] != 0x41) { - std::ofstream event(event_path); - event << "{\n" - << " \"target_id\": \"branch_event_template\",\n" - << " \"input_path\": \"REPLACE_WITH_INPUT_PATH\",\n" - << " \"input_sha256\": \"REPLACE_WITH_INPUT_SHA256\",\n" - << " \"offset\": 0,\n" - << " \"width\": 1,\n" - << " \"endianness\": \"little\",\n" - << " \"signed\": false,\n" - << " \"op\": \"eq\",\n" - << " \"rhs\": 65,\n" - << " \"description\": \"first byte must equal ASCII A; observed 0x" - << HexByte(data[0]) << "\"\n" - << "}\n"; - } - - if (data[0] == 0x41) { - std::fprintf(stderr, "template branch reached\n"); - } - return 0; -} diff --git a/parser-fuzzers/harnesses/compact_state_mutator.c b/parser-fuzzers/harnesses/compact_state_mutator.c new file mode 100644 index 0000000..accc956 --- /dev/null +++ b/parser-fuzzers/harnesses/compact_state_mutator.c @@ -0,0 +1,54 @@ +#include +#include + +extern size_t LLVMFuzzerMutate(uint8_t *data, size_t size, size_t max_size); + +#ifndef CF_FUZZ_STATE_PREFIX_SIZE +#define CF_FUZZ_STATE_PREFIX_SIZE 0U +#endif + +#ifndef CF_FUZZ_STATE_SELECTOR_SIZE +#error "CF_FUZZ_STATE_SELECTOR_SIZE must name the compact state selector bytes" +#endif + +#ifndef CF_FUZZ_STATE_MIN_PAYLOAD +#define CF_FUZZ_STATE_MIN_PAYLOAD 0U +#endif + +size_t +LLVMFuzzerCustomMutator(uint8_t *data, size_t size, size_t max_size, + unsigned int seed) +{ + const size_t state_size = + CF_FUZZ_STATE_PREFIX_SIZE + CF_FUZZ_STATE_SELECTOR_SIZE; + size_t payload_size; + size_t mutated_size; + + if (!data || size < state_size + CF_FUZZ_STATE_MIN_PAYLOAD || + max_size < state_size + CF_FUZZ_STATE_MIN_PAYLOAD) + return LLVMFuzzerMutate(data, size, max_size); + + /* Selector bytes are total functions: every value maps to a finite state. */ + if ((seed & 3U) != 0U) + { + const size_t slot = CF_FUZZ_STATE_PREFIX_SIZE + + ((seed >> 2U) % CF_FUZZ_STATE_SELECTOR_SIZE); + const uint8_t delta = (uint8_t)(1U + ((seed >> 10U) & 0xffU)); + + if (seed & (1U << 18U)) + data[slot] ^= delta; + else + data[slot] += delta; + return size; + } + + payload_size = size - state_size; + mutated_size = LLVMFuzzerMutate(data + state_size, payload_size, + max_size - state_size); + if (mutated_size < CF_FUZZ_STATE_MIN_PAYLOAD) + { + data[state_size] = (uint8_t)seed; + mutated_size = CF_FUZZ_STATE_MIN_PAYLOAD; + } + return state_size + mutated_size; +} diff --git a/parser-fuzzers/harnesses/control.h b/parser-fuzzers/harnesses/control.h new file mode 100644 index 0000000..a292734 --- /dev/null +++ b/parser-fuzzers/harnesses/control.h @@ -0,0 +1,93 @@ +#ifndef CUPSFILTERS_FUZZ_CONTROL_H +#define CUPSFILTERS_FUZZ_CONTROL_H + +#include +#include + +#define CF_FUZZ_CONTROL_SIZE 16U + +typedef struct cf_fuzz_control_s { + uint8_t ppd_profile; + uint8_t page_size; + uint8_t color_model; + uint8_t resolution; + uint8_t sides; + uint8_t orientation; + uint8_t scaling; + uint8_t copies; + uint8_t number_up; + uint8_t position; + uint8_t quality; + uint8_t output_order; + uint8_t media_type; + uint8_t mirror; + uint8_t route_mode; + uint8_t reserved; +} cf_fuzz_control_t; + +/* Compile-time policies bind a target to one meaningful configuration while + * leaving the remaining lightweight dimensions under fuzzer control. */ +static inline void cf_fuzz_apply_control_policy(cf_fuzz_control_t *control) { +#ifdef CF_FUZZ_FORCE_PPD_PROFILE + control->ppd_profile = (uint8_t)CF_FUZZ_FORCE_PPD_PROFILE; +#endif +#ifdef CF_FUZZ_FORCE_PAGE_SIZE + control->page_size = (uint8_t)CF_FUZZ_FORCE_PAGE_SIZE; +#endif +#ifdef CF_FUZZ_FORCE_COLOR_MODEL + control->color_model = (uint8_t)CF_FUZZ_FORCE_COLOR_MODEL; +#endif +#ifdef CF_FUZZ_FORCE_RESOLUTION + control->resolution = (uint8_t)CF_FUZZ_FORCE_RESOLUTION; +#endif +#ifdef CF_FUZZ_FORCE_SIDES + control->sides = (uint8_t)CF_FUZZ_FORCE_SIDES; +#endif +#ifdef CF_FUZZ_FORCE_ORIENTATION + control->orientation = (uint8_t)CF_FUZZ_FORCE_ORIENTATION; +#endif +#ifdef CF_FUZZ_FORCE_NUMBER_UP + control->number_up = (uint8_t)CF_FUZZ_FORCE_NUMBER_UP; +#endif +#ifdef CF_FUZZ_FORCE_OUTPUT_ORDER + control->output_order = (uint8_t)CF_FUZZ_FORCE_OUTPUT_ORDER; +#endif +} + +static inline int cf_fuzz_split_input(const uint8_t *data, size_t size, + size_t max_document, + const uint8_t **document, + size_t *document_size, + cf_fuzz_control_t *control) { + const uint8_t *tail; + + if (!data || !document || !document_size || !control || + size <= CF_FUZZ_CONTROL_SIZE || + size - CF_FUZZ_CONTROL_SIZE > max_document) { + return 0; + } + + *document = data; + *document_size = size - CF_FUZZ_CONTROL_SIZE; + tail = data + *document_size; + control->ppd_profile = tail[0]; + control->page_size = tail[1]; + control->color_model = tail[2]; + control->resolution = tail[3]; + control->sides = tail[4]; + control->orientation = tail[5]; + control->scaling = tail[6]; + control->copies = tail[7]; + control->number_up = tail[8]; + control->position = tail[9]; + control->quality = tail[10]; + control->output_order = tail[11]; + control->media_type = tail[12]; + control->mirror = tail[13]; + control->route_mode = tail[14]; + control->reserved = tail[15]; + cf_fuzz_apply_control_policy(control); + return 1; +} + +#endif diff --git a/parser-fuzzers/harnesses/direct_route.h b/parser-fuzzers/harnesses/direct_route.h new file mode 100644 index 0000000..06c7868 --- /dev/null +++ b/parser-fuzzers/harnesses/direct_route.h @@ -0,0 +1,482 @@ +#ifndef CUPSFILTERS_FUZZ_DIRECT_ROUTE_H +#define CUPSFILTERS_FUZZ_DIRECT_ROUTE_H + +#include "control.h" +#include "job.h" +#include "profiles.h" +#include "runtime.h" +#include "validity.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CF_FUZZ_STDIO_STREAM_CONTINUATION +static void cf_fuzz_release_fdopen_streams(void); +#endif + +#ifndef CF_FUZZ_FILTER_FUNCTION +#error "CF_FUZZ_FILTER_FUNCTION must name one cfFilter or ppdFilter entry" +#endif +#ifndef CF_FUZZ_TARGET_NAME +#error "CF_FUZZ_TARGET_NAME must be a string literal" +#endif +#ifndef CF_FUZZ_INPUT_MIME +#error "CF_FUZZ_INPUT_MIME must be a string literal" +#endif +#ifndef CF_FUZZ_OUTPUT_MIME +#error "CF_FUZZ_OUTPUT_MIME must be a string literal" +#endif + +#define CF_FUZZ_CAPTURE_LIMIT (8U * 1024U * 1024U) + +typedef struct cf_fuzz_run_result_s { + uint8_t *output; + size_t output_size; + int status; + int captured; +#ifdef CF_FUZZ_CAPTURE_PAGE_LOGS + unsigned page_log_count; + unsigned page_log_overflow; + unsigned page_log_page[16]; + unsigned page_log_copies[16]; +#endif +#ifdef CF_FUZZ_CAPTURE_TEXT_TAIL_LOGS + unsigned text_incomplete_log_count; + unsigned text_iconv_log_count; + unsigned text_illegal_log_count; +#endif +} cf_fuzz_run_result_t; + +static void cf_fuzz_log(void *data, cf_loglevel_t level, + const char *message, ...) { +#if defined(CF_FUZZ_CAPTURE_PAGE_LOGS) || \ + defined(CF_FUZZ_CAPTURE_TEXT_TAIL_LOGS) + cf_fuzz_run_result_t *result = (cf_fuzz_run_result_t *)data; + char formatted[128]; +#ifdef CF_FUZZ_CAPTURE_PAGE_LOGS + unsigned page; + unsigned copies; +#endif + va_list arguments; + + if (!result || !message) { + return; + } + va_start(arguments, message); + vsnprintf(formatted, sizeof(formatted), message, arguments); + va_end(arguments); +#ifdef CF_FUZZ_CAPTURE_PAGE_LOGS + if (level == CF_LOGLEVEL_CONTROL && + sscanf(formatted, "PAGE: %u %u", &page, &copies) == 2) { + if (result->page_log_count < 16U) { + const unsigned index = result->page_log_count++; + result->page_log_page[index] = page; + result->page_log_copies[index] = copies; + } else { + result->page_log_overflow = 1U; + } + } +#endif +#ifdef CF_FUZZ_CAPTURE_TEXT_TAIL_LOGS + if (strstr(formatted, "ends with incomplete UTF-8 character sequence")) { + result->text_incomplete_log_count++; + } + if (strstr(formatted, "iconv() message:")) { + result->text_iconv_log_count++; + } + if (strstr(formatted, "Illegal UTF-8 sequence found")) { + result->text_illegal_log_count++; + } +#endif +#else + (void)data; + (void)level; + (void)message; +#endif +} + +static int cf_fuzz_not_canceled(void *data) { + (void)data; + return 0; +} + +#ifdef CF_FUZZ_DIRECT_FAULT_INJECTION +enum { + CF_FUZZ_DIRECT_FAULT_NONE = 0, + CF_FUZZ_DIRECT_FAULT_EMPTY_INPUT = 1, + CF_FUZZ_DIRECT_FAULT_INVALID_INPUT = 2, + CF_FUZZ_DIRECT_FAULT_INVALID_OUTPUT = 3, + CF_FUZZ_DIRECT_FAULT_OUTPUT_FULL = 4, + CF_FUZZ_DIRECT_FAULT_CANCELED = 5, + CF_FUZZ_DIRECT_FAULT_COUNT = 6 +}; + +static int cf_fuzz_canceled(void *data) { + (void)data; + return 1; +} +#endif + +static int cf_fuzz_capture_file(FILE *file, cf_fuzz_run_result_t *result) { + long length; + + if (fflush(file) != 0 || fseek(file, 0, SEEK_END) != 0) { + return -1; + } + length = ftell(file); + if (length < 0 || (unsigned long)length > CF_FUZZ_CAPTURE_LIMIT || + fseek(file, 0, SEEK_SET) != 0) { + return -1; + } + if (length > 0) { + result->output = (uint8_t *)malloc((size_t)length); + if (!result->output || + fread(result->output, 1U, (size_t)length, file) != (size_t)length) { + free(result->output); + result->output = NULL; + return -1; + } + } + result->output_size = (size_t)length; + result->captured = 1; + return 0; +} + +static void cf_fuzz_free_run_result(cf_fuzz_run_result_t *result) { + if (!result) { + return; + } + free(result->output); + memset(result, 0, sizeof(*result)); +} + +static int cf_fuzz_execute_direct_internal(const uint8_t *document, + size_t document_size, + const cf_fuzz_control_t *control, + const cf_fuzz_job_input_t *job, + int capture_output, + cf_fuzz_run_result_t *result) { + char input_path[] = "/tmp/cupsfilters-fuzz-input.XXXXXX"; + char ppd_path[] = "/tmp/cupsfilters-fuzz-ppd.XXXXXX"; + char options_text[1024]; + cf_filter_data_t filter_data; + char *job_options = NULL; + char *job_title = NULL; + cups_option_t *options = NULL; + ipp_t *printer_attrs = NULL; + FILE *capture = NULL; + int input_fd = -1; + int output_fd = -1; + int filter_input_fd = -1; + int filter_output_fd = -1; + int ppd_fd = -1; + int ppd_loaded = 0; + int status = 0; + void *parameters = NULL; +#ifdef CF_FUZZ_DIRECT_FAULT_INJECTION + unsigned fault_mode = control->reserved % CF_FUZZ_DIRECT_FAULT_COUNT; +#endif +#ifdef CF_FUZZ_OUTPUT_FORMAT + cf_filter_out_format_t output_format = CF_FUZZ_OUTPUT_FORMAT; +#endif +#ifdef CF_FUZZ_TEXTTOPDF_PARAMETERS + cf_filter_texttopdf_parameter_t text_parameters; +#endif +#ifdef CF_FUZZ_BANNER_PARAMETERS + char banner_directory[PATH_MAX]; +#endif + + memset(result, 0, sizeof(*result)); + memset(&filter_data, 0, sizeof(filter_data)); + cf_fuzz_init_runtime(); + +#ifdef CF_FUZZ_VALIDATE_PNG + if (!cf_fuzz_validate_png(document, document_size)) { + return 0; + } +#endif +#ifdef CF_FUZZ_VALIDATE_SIMPLE_RASTER + if (!cf_fuzz_validate_simple_raster(document, document_size)) { + return 0; + } +#endif +#ifdef CF_FUZZ_IMAGE_ASCII85_CONTINUATION + if (!cf_fuzz_png_width_aligned(document, document_size, 4U)) { + return 0; + } +#endif +#ifdef CF_FUZZ_VALIDATE_PDF_DEPTH + if (!cf_fuzz_validate_pdf_policy(document, document_size, + CF_FUZZ_PDF_REJECT_INTERACTIVE)) { + return 0; + } +#endif +#ifdef CF_FUZZ_VALIDATE_PDF_INTERACTIVE + if (!cf_fuzz_validate_pdf_policy(document, document_size, + CF_FUZZ_PDF_REQUIRE_INTERACTIVE)) { + return 0; + } +#endif + if (job) { + job_options = (char *)malloc(job->options_size + 1U); + job_title = (char *)malloc(job->title_size + 1U); + if (!job_options || !job_title) { + goto cleanup; + } + memcpy(job_options, job->options, job->options_size); + job_options[job->options_size] = '\0'; + memcpy(job_title, job->title, job->title_size); + job_title[job->title_size] = '\0'; + } else if (cf_fuzz_build_options(options_text, sizeof(options_text), control) != + 0) { + return 0; + } + + input_fd = mkstemp(input_path); + if (input_fd < 0) { + return 0; + } + unlink(input_path); + if ( +#ifdef CF_FUZZ_DIRECT_FAULT_INJECTION + (fault_mode != CF_FUZZ_DIRECT_FAULT_EMPTY_INPUT && + cf_fuzz_write_all(input_fd, document, document_size) != 0) || +#else + cf_fuzz_write_all(input_fd, document, document_size) != 0 || +#endif + lseek(input_fd, 0, SEEK_SET) < 0) { + goto cleanup; + } +#ifdef CF_FUZZ_VALIDATE_RASTER + { + unsigned raster_flags = 0; +#ifdef CF_FUZZ_RASTER_PDF_COLORSPACE + raster_flags |= CF_FUZZ_RASTER_REQUIRE_PDF_COLORSPACE; +#endif +#ifdef CF_FUZZ_RASTER_ESCPX_SAFE_WEAVE + raster_flags |= CF_FUZZ_RASTER_REQUIRE_ESCPX_WEAVE; +#endif +#ifdef CF_FUZZ_RASTER_POLICY_COMPRESSION_1 + raster_flags |= CF_FUZZ_RASTER_REQUIRE_COMPRESSION_1; +#endif +#ifdef CF_FUZZ_RASTER_POLICY_COMPRESSION_2 + raster_flags |= CF_FUZZ_RASTER_REQUIRE_COMPRESSION_2; +#endif +#ifdef CF_FUZZ_RASTER_POLICY_COMPRESSION_3 + raster_flags |= CF_FUZZ_RASTER_REQUIRE_COMPRESSION_3; +#endif +#ifdef CF_FUZZ_RASTER_POLICY_COMPRESSION_10 + raster_flags |= CF_FUZZ_RASTER_REQUIRE_COMPRESSION_10; +#endif +#ifdef CF_FUZZ_RASTER_POLICY_REJECT_COMPRESSION_3 + raster_flags |= CF_FUZZ_RASTER_REJECT_COMPRESSION_3; +#endif +#ifdef CF_FUZZ_RASTER_POLICY_MULTIROW + raster_flags |= CF_FUZZ_RASTER_REQUIRE_MULTIROW; +#endif +#ifdef CF_FUZZ_RASTER_POLICY_MODE10_RGB + raster_flags |= CF_FUZZ_RASTER_REQUIRE_MODE10_RGB; +#endif + if (!cf_fuzz_validate_raster_fd(input_fd, raster_flags)) { + goto cleanup; + } + } +#endif + + if (capture_output) { + capture = tmpfile(); + if (!capture || (output_fd = dup(fileno(capture))) < 0) { + goto cleanup; + } + } else { + output_fd = open( +#ifdef CF_FUZZ_DIRECT_FAULT_INJECTION + fault_mode == CF_FUZZ_DIRECT_FAULT_OUTPUT_FULL ? "/dev/full" : +#endif + "/dev/null", O_WRONLY); + if (output_fd < 0) { + goto cleanup; + } + } + + if (!job || job->ppd_size) { + ppd_fd = mkstemp(ppd_path); + if (ppd_fd < 0) { + goto cleanup; + } + if (job) { + if (cf_fuzz_write_all(ppd_fd, job->ppd, job->ppd_size) != 0 || + close(ppd_fd) != 0) { + goto cleanup; + } + ppd_fd = -1; + } else { + FILE *ppd_file = fdopen(ppd_fd, "w"); + if (!ppd_file) { + goto cleanup; + } + ppd_fd = -1; + if (cf_fuzz_write_ppd(ppd_file, control, CF_FUZZ_TARGET_NAME) != 0) { + (void)fclose(ppd_file); + goto cleanup; + } + if (fclose(ppd_file) != 0) { + goto cleanup; + } + } + } + + filter_data.printer = (char *)"oss-fuzz"; + filter_data.job_id = 1; + filter_data.job_user = (char *)"fuzzer"; + filter_data.job_title = + job && job->title_size ? job_title : (char *)CF_FUZZ_TARGET_NAME; + filter_data.copies = 1 + control->copies % 4U; + filter_data.content_type = (char *)CF_FUZZ_INPUT_MIME; + filter_data.final_content_type = (char *)CF_FUZZ_OUTPUT_MIME; + filter_data.back_pipe[0] = filter_data.back_pipe[1] = -1; + filter_data.side_pipe[0] = filter_data.side_pipe[1] = -1; + filter_data.logfunc = cf_fuzz_log; +#if defined(CF_FUZZ_CAPTURE_PAGE_LOGS) || \ + defined(CF_FUZZ_CAPTURE_TEXT_TAIL_LOGS) + filter_data.logdata = result; +#endif +#ifdef CF_FUZZ_DIRECT_FAULT_INJECTION + filter_data.iscanceledfunc = + fault_mode == CF_FUZZ_DIRECT_FAULT_CANCELED ? cf_fuzz_canceled + : cf_fuzz_not_canceled; +#else + filter_data.iscanceledfunc = cf_fuzz_not_canceled; +#endif + filter_data.num_options = cupsParseOptions(job ? job_options : options_text, + 0, &options); + filter_data.options = options; + filter_input_fd = input_fd; + filter_output_fd = output_fd; +#ifdef CF_FUZZ_DIRECT_FAULT_INJECTION + if (fault_mode == CF_FUZZ_DIRECT_FAULT_INVALID_INPUT) { + filter_input_fd = -1; + } else if (fault_mode == CF_FUZZ_DIRECT_FAULT_INVALID_OUTPUT) { + filter_output_fd = -1; + } +#endif + +#ifdef CF_FUZZ_NEEDS_PCLM_ATTRS + printer_attrs = ippNew(); + if (printer_attrs && !job) { + ippAddInteger(printer_attrs, IPP_TAG_PRINTER, IPP_TAG_INTEGER, + "pclm-strip-height-preferred", 16); + ippAddResolution(printer_attrs, IPP_TAG_PRINTER, + "pclm-source-resolution-supported", IPP_RES_PER_INCH, + 300, 300); + ippAddResolution(printer_attrs, IPP_TAG_PRINTER, + "pclm-source-resolution-default", IPP_RES_PER_INCH, + 300, 300); + ippAddString(printer_attrs, IPP_TAG_PRINTER, IPP_TAG_KEYWORD, + "pclm-compression-method-preferred", NULL, "flate"); + } + filter_data.printer_attrs = printer_attrs; +#endif + +#ifdef CF_FUZZ_OUTPUT_FORMAT + parameters = &output_format; +#elif defined(CF_FUZZ_TEXTTOPDF_PARAMETERS) + memset(&text_parameters, 0, sizeof(text_parameters)); + text_parameters.data_dir = (char *)cf_fuzz_data_dir(); +#ifdef CF_FUZZ_CHARSET + text_parameters.char_set = (char *)CF_FUZZ_CHARSET; +#else + text_parameters.char_set = (char *)"utf-8"; +#endif + text_parameters.content_type = (char *)CF_FUZZ_INPUT_MIME; + parameters = &text_parameters; +#elif defined(CF_FUZZ_BANNER_PARAMETERS) + snprintf(banner_directory, sizeof(banner_directory), "%s/data", + cf_fuzz_data_dir()); + parameters = banner_directory; +#endif + + if (job && !job->ppd_size) { + result->status = CF_FUZZ_FILTER_FUNCTION(filter_input_fd, filter_output_fd, 1, + &filter_data, parameters); + status = 1; + } else if (ppdFilterLoadPPDFile(&filter_data, ppd_path) == 0) { + ppd_loaded = 1; +#ifdef CF_FUZZ_POST_PPD_LOAD_HOOK + if (CF_FUZZ_POST_PPD_LOAD_HOOK(&filter_data) != 0) { + goto cleanup; + } +#endif + result->status = CF_FUZZ_FILTER_FUNCTION(filter_input_fd, filter_output_fd, 1, + &filter_data, parameters); + status = 1; + } + +cleanup: +#ifdef CF_FUZZ_STDIO_STREAM_CONTINUATION + /* Release filter-owned stdio objects before their descriptors can be reused. */ + cf_fuzz_release_fdopen_streams(); +#endif + if (filter_data.options) { + cupsFreeOptions(filter_data.num_options, filter_data.options); + filter_data.options = NULL; + } + if (ppd_loaded) { + ppdFilterFreePPDFile(&filter_data); + } + if (printer_attrs) { + ippDelete(printer_attrs); + } + unlink(ppd_path); + if (output_fd >= 0) { + (void)close(output_fd); + } + if (input_fd >= 0) { + (void)close(input_fd); + } + if (ppd_fd >= 0) { + (void)close(ppd_fd); + } + if (capture) { + if (status && cf_fuzz_capture_file(capture, result) != 0) { + status = 0; + } + fclose(capture); + } + free(job_options); + free(job_title); + return status; +} + +static int cf_fuzz_execute_direct(const uint8_t *document, + size_t document_size, + const cf_fuzz_control_t *control, + int capture_output, + cf_fuzz_run_result_t *result) { + return cf_fuzz_execute_direct_internal(document, document_size, control, NULL, + capture_output, result); +} + +static int cf_fuzz_execute_direct_job(const cf_fuzz_job_input_t *job, + int capture_output, + cf_fuzz_run_result_t *result) { + if (!job) { + return 0; + } + return cf_fuzz_execute_direct_internal( + job->document, job->document_size, &job->control, job, capture_output, + result); +} + +#endif diff --git a/parser-fuzzers/harnesses/dynamic_compare_trace.c b/parser-fuzzers/harnesses/dynamic_compare_trace.c deleted file mode 100644 index 8f614fa..0000000 --- a/parser-fuzzers/harnesses/dynamic_compare_trace.c +++ /dev/null @@ -1,252 +0,0 @@ -#define _GNU_SOURCE - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define TRACE_BYTES 32 -#define TRACE_ASCII 48 -#define DEFAULT_TRACE_LIMIT 256 - -static int trace_fd = -2; -static long trace_count = 0; -static long trace_limit = DEFAULT_TRACE_LIMIT; -static __thread int in_hook = 0; - -static int (*real_memcmp_fn)(const void *, const void *, size_t) = NULL; -static int (*real_strcmp_fn)(const char *, const char *) = NULL; -static int (*real_strncmp_fn)(const char *, const char *, size_t) = NULL; -static int (*real_strcasecmp_fn)(const char *, const char *) = NULL; -static int (*real_strncasecmp_fn)(const char *, const char *, size_t) = NULL; - -static int simple_memcmp(const void *a, const void *b, size_t n) { - const unsigned char *pa = (const unsigned char *)a; - const unsigned char *pb = (const unsigned char *)b; - for (size_t i = 0; i < n; i++) { - if (pa[i] != pb[i]) { - return (int)pa[i] - (int)pb[i]; - } - } - return 0; -} - -static int simple_strcmp(const char *a, const char *b) { - while (*a && *a == *b) { - a++; - b++; - } - return (unsigned char)*a - (unsigned char)*b; -} - -static int simple_strncmp(const char *a, const char *b, size_t n) { - for (size_t i = 0; i < n; i++) { - unsigned char ca = (unsigned char)a[i]; - unsigned char cb = (unsigned char)b[i]; - if (ca != cb || ca == 0 || cb == 0) { - return (int)ca - (int)cb; - } - } - return 0; -} - -static int simple_strcasecmp(const char *a, const char *b) { - while (*a && tolower((unsigned char)*a) == tolower((unsigned char)*b)) { - a++; - b++; - } - return tolower((unsigned char)*a) - tolower((unsigned char)*b); -} - -static int simple_strncasecmp(const char *a, const char *b, size_t n) { - for (size_t i = 0; i < n; i++) { - unsigned char ca = (unsigned char)tolower((unsigned char)a[i]); - unsigned char cb = (unsigned char)tolower((unsigned char)b[i]); - if (ca != cb || ca == 0 || cb == 0) { - return (int)ca - (int)cb; - } - } - return 0; -} - -static size_t bounded_strlen(const char *s, size_t limit) { - size_t n = 0; - if (!s) { - return 0; - } - while (n < limit && s[n] != 0) { - n++; - } - return n; -} - -static void ensure_real_functions(void) { - if (real_memcmp_fn && real_strcmp_fn && real_strncmp_fn && - real_strcasecmp_fn && real_strncasecmp_fn) { - return; - } - if (in_hook) { - return; - } - in_hook = 1; - real_memcmp_fn = (int (*)(const void *, const void *, size_t))dlsym(RTLD_NEXT, "memcmp"); - real_strcmp_fn = (int (*)(const char *, const char *))dlsym(RTLD_NEXT, "strcmp"); - real_strncmp_fn = (int (*)(const char *, const char *, size_t))dlsym(RTLD_NEXT, "strncmp"); - real_strcasecmp_fn = (int (*)(const char *, const char *))dlsym(RTLD_NEXT, "strcasecmp"); - real_strncasecmp_fn = (int (*)(const char *, const char *, size_t))dlsym(RTLD_NEXT, "strncasecmp"); - in_hook = 0; -} - -static void ensure_trace_fd(void) { - const char *path; - const char *limit; - if (trace_fd != -2 || in_hook) { - return; - } - in_hook = 1; - path = getenv("SMT_FUZZER_COMPARE_TRACE"); - limit = getenv("SMT_FUZZER_COMPARE_TRACE_LIMIT"); - if (limit && *limit) { - char *end = NULL; - long parsed = strtol(limit, &end, 10); - if (end && *end == 0 && parsed >= 0) { - trace_limit = parsed; - } - } - if (!path || !*path || trace_limit == 0) { - trace_fd = -1; - in_hook = 0; - return; - } - trace_fd = open(path, O_CREAT | O_WRONLY | O_APPEND, 0600); - if (trace_fd >= 0) { - const char *header = "pid\tpc\top\tret\tlen\ta_hex\tb_hex\ta_ascii\tb_ascii\n"; - ssize_t ignored = write(trace_fd, header, strlen(header)); - (void)ignored; - } - in_hook = 0; -} - -static void encode_hex(char *out, size_t out_len, const unsigned char *data, size_t len) { - static const char hexdigits[] = "0123456789abcdef"; - size_t n = len < TRACE_BYTES ? len : TRACE_BYTES; - size_t offset = 0; - if (out_len == 0) { - return; - } - for (size_t i = 0; i < n && offset + 2 < out_len; i++) { - out[offset++] = hexdigits[data[i] >> 4]; - out[offset++] = hexdigits[data[i] & 15]; - } - out[offset] = 0; -} - -static void encode_ascii(char *out, size_t out_len, const unsigned char *data, size_t len) { - size_t n = len < TRACE_ASCII ? len : TRACE_ASCII; - size_t offset = 0; - if (out_len == 0) { - return; - } - for (size_t i = 0; i < n && offset + 1 < out_len; i++) { - unsigned char c = data[i]; - if (c == '\t' || c == '\n' || c == '\r') { - out[offset++] = ' '; - } else if (c >= 32 && c <= 126) { - out[offset++] = (char)c; - } else { - out[offset++] = '.'; - } - } - out[offset] = 0; -} - -static void trace_compare(const char *op, int ret, size_t len, const void *a, - const void *b, void *pc) { - char a_hex[(TRACE_BYTES * 2) + 1]; - char b_hex[(TRACE_BYTES * 2) + 1]; - char a_ascii[TRACE_ASCII + 1]; - char b_ascii[TRACE_ASCII + 1]; - char line[512]; - int written; - - if (in_hook) { - return; - } - ensure_trace_fd(); - if (trace_fd < 0 || trace_count >= trace_limit) { - return; - } - - in_hook = 1; - encode_hex(a_hex, sizeof(a_hex), (const unsigned char *)a, len); - encode_hex(b_hex, sizeof(b_hex), (const unsigned char *)b, len); - encode_ascii(a_ascii, sizeof(a_ascii), (const unsigned char *)a, len); - encode_ascii(b_ascii, sizeof(b_ascii), (const unsigned char *)b, len); - written = snprintf(line, sizeof(line), "%ld\t%p\t%s\t%d\t%zu\t%s\t%s\t%s\t%s\n", - (long)getpid(), pc, op, ret, len, a_hex, b_hex, a_ascii, b_ascii); - if (written > 0) { - if ((size_t)written > sizeof(line)) { - written = (int)sizeof(line); - } - ssize_t ignored = write(trace_fd, line, (size_t)written); - (void)ignored; - trace_count++; - } - in_hook = 0; -} - -int memcmp(const void *a, const void *b, size_t n) { - int ret; - ensure_real_functions(); - ret = real_memcmp_fn ? real_memcmp_fn(a, b, n) : simple_memcmp(a, b, n); - trace_compare("memcmp", ret, n, a, b, __builtin_return_address(0)); - return ret; -} - -int strcmp(const char *a, const char *b) { - size_t n; - int ret; - ensure_real_functions(); - ret = real_strcmp_fn ? real_strcmp_fn(a, b) : simple_strcmp(a, b); - n = bounded_strlen(a, TRACE_ASCII); - if (bounded_strlen(b, TRACE_ASCII) > n) { - n = bounded_strlen(b, TRACE_ASCII); - } - trace_compare("strcmp", ret, n + 1, a, b, __builtin_return_address(0)); - return ret; -} - -int strncmp(const char *a, const char *b, size_t n) { - int ret; - ensure_real_functions(); - ret = real_strncmp_fn ? real_strncmp_fn(a, b, n) : simple_strncmp(a, b, n); - trace_compare("strncmp", ret, n, a, b, __builtin_return_address(0)); - return ret; -} - -int strcasecmp(const char *a, const char *b) { - size_t n; - int ret; - ensure_real_functions(); - ret = real_strcasecmp_fn ? real_strcasecmp_fn(a, b) : simple_strcasecmp(a, b); - n = bounded_strlen(a, TRACE_ASCII); - if (bounded_strlen(b, TRACE_ASCII) > n) { - n = bounded_strlen(b, TRACE_ASCII); - } - trace_compare("strcasecmp", ret, n + 1, a, b, __builtin_return_address(0)); - return ret; -} - -int strncasecmp(const char *a, const char *b, size_t n) { - int ret; - ensure_real_functions(); - ret = real_strncasecmp_fn ? real_strncasecmp_fn(a, b, n) : simple_strncasecmp(a, b, n); - trace_compare("strncasecmp", ret, n, a, b, __builtin_return_address(0)); - return ret; -} diff --git a/parser-fuzzers/harnesses/fuzz_cups_raster_reader.c b/parser-fuzzers/harnesses/fuzz_cups_raster_reader.c new file mode 100644 index 0000000..9b06b03 --- /dev/null +++ b/parser-fuzzers/harnesses/fuzz_cups_raster_reader.c @@ -0,0 +1,179 @@ +#define _GNU_SOURCE + +#include + +#include +#include +#include +#include + +#ifndef CUPS_RASTER_READER_MAX_INPUT +#define CUPS_RASTER_READER_MAX_INPUT (4U * 1024U * 1024U) +#endif + +#ifndef CUPS_RASTER_READER_MAX_PAGES +#define CUPS_RASTER_READER_MAX_PAGES 64U +#endif + +#ifndef CUPS_RASTER_READER_MAX_LINE +#define CUPS_RASTER_READER_MAX_LINE (1U * 1024U * 1024U) +#endif + +#ifndef CUPS_RASTER_READER_MAX_DECODED +#define CUPS_RASTER_READER_MAX_DECODED (32U * 1024U * 1024U) +#endif + +static int write_all(int fd, const uint8_t *data, size_t size) { + size_t offset = 0; + + while (offset < size) { + ssize_t written = write(fd, data + offset, size - offset); + + if (written < 0 && errno == EINTR) { + continue; + } + if (written <= 0) { + return -1; + } + offset += (size_t)written; + } + + return 0; +} + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + char input_name[] = "/tmp/cups-raster-reader.XXXXXX"; + cups_page_header2_t header; + cups_raster_t *raster = NULL; + unsigned char *line = NULL; + uint64_t decoded = 0; + uint64_t pages = 0; + unsigned schedule; + int inputfd = -1; + + if (!data || size == 0 || + (uint64_t)size > (uint64_t)CUPS_RASTER_READER_MAX_INPUT) { + return 0; + } +#ifdef CUPS_RASTER_READER_OVERREAD_ORACLE + schedule = 3; +#else + schedule = data[size - 1] % 3; +#endif + + inputfd = mkstemp(input_name); + if (inputfd < 0) { + return 0; + } + unlink(input_name); + + if (write_all(inputfd, data, size) < 0 || + lseek(inputfd, 0, SEEK_SET) < 0) { + goto done; + } + + raster = cupsRasterOpen(inputfd, CUPS_RASTER_READ); + if (!raster) { + goto done; + } + + while (pages < (uint64_t)CUPS_RASTER_READER_MAX_PAGES && + cupsRasterReadHeader2(raster, &header)) { + uint64_t line_bytes = (uint64_t)header.cupsBytesPerLine; + uint64_t rows = (uint64_t)header.cupsHeight; + uint64_t page_bytes; + uint64_t row; + + pages++; + + if (header.cupsColorOrder == CUPS_ORDER_PLANAR) { + if (header.cupsNumColors != 0 && + rows > UINT64_MAX / (uint64_t)header.cupsNumColors) { + break; + } + rows *= (uint64_t)header.cupsNumColors; + } + + if (line_bytes == 0 || + line_bytes > (uint64_t)CUPS_RASTER_READER_MAX_LINE || + (rows != 0 && line_bytes > UINT64_MAX / rows)) { + break; + } + + page_bytes = line_bytes * rows; + if (page_bytes > (uint64_t)CUPS_RASTER_READER_MAX_DECODED - decoded) { + break; + } + + { + uint64_t allocation = line_bytes; + + if ((schedule == 2 && rows > 1) || schedule == 3) { + if (line_bytes > SIZE_MAX / 2) { + break; + } + allocation *= 2; + } + line = (unsigned char *)malloc((size_t)allocation); + } + if (!line) { + break; + } + + for (row = 0; row < rows;) { + uint64_t batch = schedule == 2 && rows - row > 1 ? 2 : 1; + uint64_t requested = line_bytes * batch; + unsigned read_bytes; + + if (schedule == 3 && row + 1 == rows) { + requested = line_bytes * 2; + read_bytes = + cupsRasterReadPixels(raster, line, (unsigned)requested); + if ((uint64_t)read_bytes > line_bytes) { + __builtin_trap(); + } + if ((uint64_t)read_bytes != line_bytes) { + break; + } + requested = line_bytes; + } else if (schedule == 1 && line_bytes > 1) { + unsigned first = (header.cupsBytesPerLine + 1) / 2; + unsigned second = header.cupsBytesPerLine - first; + + read_bytes = cupsRasterReadPixels(raster, line, first); + if (read_bytes != first) { + break; + } + read_bytes = cupsRasterReadPixels(raster, line + first, second); + if (read_bytes != second) { + break; + } + } else { + read_bytes = + cupsRasterReadPixels(raster, line, (unsigned)requested); + if ((uint64_t)read_bytes != requested) { + break; + } + } + decoded += requested; + row += batch; + } + + free(line); + line = NULL; + + if (row != rows) { + break; + } + } + +done: + free(line); + if (raster) { + cupsRasterClose(raster); + } + if (inputfd >= 0) { + close(inputfd); + } + return 0; +} diff --git a/parser-fuzzers/harnesses/fuzz_cupsfilters_image_codec.c b/parser-fuzzers/harnesses/fuzz_cupsfilters_image_codec.c new file mode 100644 index 0000000..9a84e43 --- /dev/null +++ b/parser-fuzzers/harnesses/fuzz_cupsfilters_image_codec.c @@ -0,0 +1,764 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(CUPSFILTERS_IMAGE_CODEC_JPEG) +# include +# include +#elif defined(CUPSFILTERS_IMAGE_CODEC_TIFF) +# include +# include +#endif + +#if !defined(CUPSFILTERS_IMAGE_CODEC_JPEG) && \ + !defined(CUPSFILTERS_IMAGE_CODEC_TIFF) && \ + !defined(CUPSFILTERS_IMAGE_CODEC_PNG) +# define CUPSFILTERS_IMAGE_CODEC_PNG +#endif + +#ifndef CUPSFILTERS_IMAGE_CODEC_MAX_INPUT +# define CUPSFILTERS_IMAGE_CODEC_MAX_INPUT (2U * 1024U * 1024U) +#endif + +#define IMAGE_CODEC_MAX_DIMENSION 4096U +#define IMAGE_CODEC_MAX_DECODED (16U * 1024U * 1024U) +#define IMAGE_CODEC_MAX_SCANLINE (64U * 1024U) + +#if defined(CUPSFILTERS_IMAGE_CODEC_TIFF) +typedef struct tiff_read_state_s +{ + int active; + uint8_t sentinel; + unsigned failures; + uint32_t first_failed_row; +} tiff_read_state_t; + +static _Thread_local tiff_read_state_t tiff_read_state; + +int +LLVMFuzzerInitialize(int *argc, char ***argv) +{ + (void)argc; + (void)argv; + TIFFSetErrorHandler(NULL); + TIFFSetErrorHandlerExt(NULL); + TIFFSetWarningHandler(NULL); + TIFFSetWarningHandlerExt(NULL); + return (0); +} + +extern TIFF *__real_TIFFFdOpen(int fd, const char *name, const char *mode); +extern int __real_TIFFReadScanline(TIFF *tif, void *buffer, uint32_t row, + uint16_t sample); + +TIFF * +__wrap_TIFFFdOpen(int fd, const char *name, const char *mode) +{ + int duplicate = dup(fd); + TIFF *tif; + + if (duplicate < 0) + return (NULL); + tif = __real_TIFFFdOpen(duplicate, name, mode); + if (!tif) + close(duplicate); + return (tif); +} + +int +__wrap_TIFFReadScanline(TIFF *tif, void *buffer, uint32_t row, + uint16_t sample) +{ + tmsize_t size; + int result; + +#if !defined(CUPSFILTERS_IMAGE_CODEC_TIFF_GRAY_MULTISAMPLE_DISCOVERY) && \ + !defined(CUPSFILTERS_IMAGE_CODEC_TIFF_GRAY_MULTISAMPLE_CONTINUATION) + if (tiff_read_state.active && buffer) + { + size = TIFFScanlineSize(tif); + if (size > 0 && size <= IMAGE_CODEC_MAX_SCANLINE) + memset(buffer, tiff_read_state.sentinel, (size_t)size); + } +#else + (void)size; +#endif + result = __real_TIFFReadScanline(tif, buffer, row, sample); + if (tiff_read_state.active && result < 0) + { + if (!tiff_read_state.failures) + tiff_read_state.first_failed_row = row; + tiff_read_state.failures ++; + } + return (result); +} +#endif + +static void +close_image(cf_image_t *image) +{ + cf_ic_t *cached[IMAGE_CODEC_MAX_DECODED / + (CF_TILE_SIZE * CF_TILE_SIZE)] = {0}; + unsigned cached_count = 0; + unsigned x_tiles; + unsigned y_tiles; + + if (!image) + return; + + if (image->cachefile >= 0) + { + close(image->cachefile); + unlink(image->cachename); + } + + x_tiles = (image->xsize + CF_TILE_SIZE - 1) / CF_TILE_SIZE; + y_tiles = (image->ysize + CF_TILE_SIZE - 1) / CF_TILE_SIZE; + if (image->tiles && x_tiles <= IMAGE_CODEC_MAX_DIMENSION / CF_TILE_SIZE && + y_tiles <= IMAGE_CODEC_MAX_DIMENSION / CF_TILE_SIZE) + { + for (unsigned y = 0; y < y_tiles; y ++) + { + if (!image->tiles[y]) + continue; + for (unsigned x = 0; x < x_tiles; x ++) + { + cf_ic_t *entry = image->tiles[y][x].ic; + unsigned seen = 0; + + if (!entry) + continue; + while (seen < cached_count && cached[seen] != entry) + seen ++; + if (seen == cached_count && + cached_count < sizeof(cached) / sizeof(cached[0])) + cached[cached_count ++] = entry; + } + } + } + + for (unsigned i = 0; i < cached_count; i ++) + free(cached[i]); + if (image->tiles) + { + free(image->tiles[0]); + free(image->tiles); + } + free(image); +} + +static cf_image_t * +open_codec(FILE *fp, cf_icspace_t primary, cf_icspace_t secondary, + int saturation, int hue, const cf_ib_t *lut) +{ +#if defined(CUPSFILTERS_IMAGE_CODEC_TIFF_GRAY_MULTISAMPLE_DISCOVERY) || \ + defined(CUPSFILTERS_IMAGE_CODEC_TIFF_GRAY_MULTISAMPLE_CONTINUATION) + cf_image_t *image = cfImageOpenFP(fp, primary, secondary, saturation, hue, + lut); + + /* TIFFFdOpen owns a duplicate fd; a successful decode leaves fp open. */ + if (image) + (void)fclose(fp); + return (image); +#else + cf_image_t *image = (cf_image_t *)calloc(1, sizeof(cf_image_t)); + int status; + + if (!image) + { + fclose(fp); + return (NULL); + } + + image->cachefile = -1; + image->max_ics = CF_TILE_MINIMUM; + image->xppi = 200; + image->yppi = 200; +#if defined(CUPSFILTERS_IMAGE_CODEC_JPEG) + status = _cfImageReadJPEG(image, fp, primary, secondary, saturation, hue, + lut); +#elif defined(CUPSFILTERS_IMAGE_CODEC_TIFF) + status = _cfImageReadTIFF(image, fp, primary, secondary, saturation, hue, + lut); + /* TIFFFdOpen receives a duplicate fd from the linker wrapper above. */ + if (!status) + (void)fclose(fp); +#else + status = _cfImageReadPNG(image, fp, primary, secondary, saturation, hue, + lut); +#endif + if (status) + { + close_image(image); + return (NULL); + } + + return (image); +#endif +} + +static uint32_t +read_be32(const uint8_t *data) +{ + return ((uint32_t)data[0] << 24) | ((uint32_t)data[1] << 16) | + ((uint32_t)data[2] << 8) | data[3]; +} + +static uint32_t +hash_bytes(const uint8_t *data, size_t size) +{ + uint32_t hash = 2166136261U; + + for (size_t i = 0; i < size; i ++) + { + hash ^= data[i]; + hash *= 16777619U; + } + return (hash); +} + +#if defined(CUPSFILTERS_IMAGE_CODEC_JPEG) +typedef struct jpeg_guard_s +{ + struct jpeg_error_mgr error; + jmp_buf jump; + JSAMPLE *row; + int created; +} jpeg_guard_t; + +extern struct jpeg_error_mgr *__real_jpeg_std_error( + struct jpeg_error_mgr *error); + +static void +jpeg_quiet_output(j_common_ptr cinfo) +{ + (void)cinfo; +} + +static void +jpeg_quiet_emit(j_common_ptr cinfo, int message_level) +{ + if (message_level < 0) + cinfo->err->num_warnings ++; +} + +struct jpeg_error_mgr * +__wrap_jpeg_std_error(struct jpeg_error_mgr *error) +{ + struct jpeg_error_mgr *result = __real_jpeg_std_error(error); + + result->emit_message = jpeg_quiet_emit; + result->output_message = jpeg_quiet_output; + return (result); +} + +static void +jpeg_guard_error_exit(j_common_ptr cinfo) +{ + jpeg_guard_t *guard = (jpeg_guard_t *)cinfo->err; + + longjmp(guard->jump, 1); +} + +static int +valid_jpeg_budget(const uint8_t *data, size_t size) +{ + struct jpeg_decompress_struct *cinfo; + jpeg_guard_t *guard; + uint64_t decoded; + size_t row_size; + int valid = 0; + + if (size < 4 || data[0] != 0xff || data[1] != 0xd8 || + size > ULONG_MAX) + return (0); + + cinfo = (struct jpeg_decompress_struct *)calloc(1, sizeof(*cinfo)); + guard = (jpeg_guard_t *)calloc(1, sizeof(*guard)); + if (!cinfo || !guard) + goto cleanup; + + cinfo->err = jpeg_std_error(&guard->error); + guard->error.error_exit = jpeg_guard_error_exit; + if (setjmp(guard->jump)) + goto cleanup; + + jpeg_create_decompress(cinfo); + guard->created = 1; + jpeg_mem_src(cinfo, data, (unsigned long)size); + if (jpeg_read_header(cinfo, TRUE) != JPEG_HEADER_OK || + (cinfo->num_components != 1 && cinfo->num_components != 3 && + cinfo->num_components != 4)) + goto cleanup; + + if (cinfo->num_components == 1) + cinfo->out_color_space = JCS_GRAYSCALE; + else if (cinfo->num_components == 4) + cinfo->out_color_space = JCS_CMYK; + else + cinfo->out_color_space = JCS_RGB; + jpeg_calc_output_dimensions(cinfo); + decoded = (uint64_t)cinfo->output_width * cinfo->output_height * + cinfo->num_components; + if (!cinfo->output_width || !cinfo->output_height || + cinfo->output_width > IMAGE_CODEC_MAX_DIMENSION || + cinfo->output_height > IMAGE_CODEC_MAX_DIMENSION || + decoded > IMAGE_CODEC_MAX_DECODED) + goto cleanup; + + if (!jpeg_start_decompress(cinfo) || !cinfo->output_components || + cinfo->output_width > SIZE_MAX / cinfo->output_components) + goto cleanup; + row_size = (size_t)cinfo->output_width * cinfo->output_components; + guard->row = (JSAMPLE *)malloc(row_size); + if (!guard->row) + goto cleanup; + + while (cinfo->output_scanline < cinfo->output_height) + { + JSAMPROW row = guard->row; + + if (jpeg_read_scanlines(cinfo, &row, 1) != 1) + goto cleanup; + } + if (!jpeg_finish_decompress(cinfo)) + goto cleanup; + valid = 1; + +cleanup: + if (guard) + { + free(guard->row); + if (guard->created) + jpeg_destroy_decompress(cinfo); + } + free(guard); + free(cinfo); + return (valid); +} +#elif defined(CUPSFILTERS_IMAGE_CODEC_TIFF) +static uint16_t +read_u16(const uint8_t *data, int little_endian) +{ + if (little_endian) + return ((uint16_t)data[1] << 8) | data[0]; + return ((uint16_t)data[0] << 8) | data[1]; +} + +static uint32_t +read_u32(const uint8_t *data, int little_endian) +{ + if (little_endian) + return ((uint32_t)data[3] << 24) | ((uint32_t)data[2] << 16) | + ((uint32_t)data[1] << 8) | data[0]; + return read_be32(data); +} + +static int +tiff_scalar(const uint8_t *data, size_t size, const uint8_t *entry, + int little_endian, uint32_t *value) +{ + uint16_t type = read_u16(entry + 2, little_endian); + uint32_t count = read_u32(entry + 4, little_endian); + uint32_t offset; + size_t element_size; + + if (!count) + return (0); + if (type == 3) + element_size = 2; + else if (type == 4) + element_size = 4; + else + return (0); + + if (count <= 4 / element_size) + offset = (uint32_t)(entry + 8 - data); + else + offset = read_u32(entry + 8, little_endian); + if (offset > size || element_size > size - offset) + return (0); + + *value = element_size == 2 ? read_u16(data + offset, little_endian) + : read_u32(data + offset, little_endian); + return (1); +} + +static int +valid_tiff_budget(const uint8_t *data, size_t size) +{ + uint32_t width = 0, height = 0, bits = 1, samples = 1; + uint32_t photometric = UINT32_MAX, compression = UINT32_MAX; + uint32_t planar = 1, orientation = 1; + uint32_t seen_tags = 0; + uint32_t ifd_offset; + uint16_t count; + int little_endian; + + if (size < 10) + return (0); + if (!memcmp(data, "II\x2a\x00", 4)) + little_endian = 1; + else if (!memcmp(data, "MM\x00\x2a", 4)) + little_endian = 0; + else + return (0); + + ifd_offset = read_u32(data + 4, little_endian); + if (ifd_offset > size - 2) + return (0); + count = read_u16(data + ifd_offset, little_endian); + if (!count || count > 256 || + (size_t)count > (size - ifd_offset - 2) / 12) + return (0); + + for (unsigned i = 0; i < count; i ++) + { + const uint8_t *entry = data + ifd_offset + 2 + i * 12; + uint16_t tag = read_u16(entry, little_endian); + uint32_t value; + + if (!tiff_scalar(data, size, entry, little_endian, &value)) + continue; + switch (tag) + { + case 256 : + case 257 : + case 258 : + case 259 : + case 262 : + case 274 : + case 277 : + case 284 : + { + unsigned bit = tag == 256 ? 0 : tag == 257 ? 1 : tag == 258 ? 2 : + tag == 259 ? 3 : tag == 262 ? 4 : tag == 274 ? 5 : + tag == 277 ? 6 : 7; + + if (seen_tags & (1U << bit)) + return (0); + seen_tags |= 1U << bit; + if (tag == 256) + width = value; + else if (tag == 257) + height = value; + else if (tag == 258) + bits = value; + else if (tag == 259) + compression = value; + else if (tag == 262) + photometric = value; + else if (tag == 274) + orientation = value; + else if (tag == 277) + samples = value; + else + planar = value; + break; + } + default : break; + } + } + + if (!width || !height || width > IMAGE_CODEC_MAX_DIMENSION || + height > IMAGE_CODEC_MAX_DIMENSION || + (bits != 1 && bits != 2 && bits != 4 && bits != 8) || + !samples || samples > 4 || (bits == 1 && samples > 1) || + planar == 2 || orientation < 1 || orientation > 4 || + (compression != 1 && compression != 5 && compression != 7 && + compression != 8 && compression != 32773 && compression != 32946)) + return (0); + if (photometric != 0 && photometric != 1 && photometric != 2 && + photometric != 3 && photometric != 5) + return (0); +#if defined(CUPSFILTERS_IMAGE_CODEC_TIFF_GRAY_MULTISAMPLE_DISCOVERY) || \ + defined(CUPSFILTERS_IMAGE_CODEC_TIFF_GRAY_MULTISAMPLE_CONTINUATION) + if ((photometric == 0 || photometric == 1) && samples > 4) +#else + if ((photometric == 0 || photometric == 1) && + samples != 1 && samples != 2) +#endif + return (0); + if (photometric == 3 && samples != 1) + return (0); + if (photometric == 2 && samples != 3 && samples != 4) + return (0); +#if !defined(CUPSFILTERS_IMAGE_CODEC_TIFF_PACKED_REGRESSION) + /* + * RGB2 is kept as an archived regression input. It otherwise terminates + * every continuation worker at the known image-tiff.c packed-row OOB read. + */ + if (photometric == 2 && bits == 2) + return (0); +#endif + if (photometric == 5 && samples != 4) + return (0); +#if defined(CUPSFILTERS_IMAGE_CODEC_TIFF_GRAY_MULTISAMPLE_CONTINUATION) + /* + * Continue past the confirmed Gray8/SPP4 scanline overwrite while retaining + * the adjacent grayscale layouts that the normal TIFF lane rejects. + */ + if ((photometric != 0 && photometric != 1) || samples < 3 || + (bits == 8 && samples == 4)) + return (0); +#endif + + if (((uint64_t)width * bits * samples + 7U) / 8U > + IMAGE_CODEC_MAX_SCANLINE) + return (0); + return ((uint64_t)width * height * 4U <= IMAGE_CODEC_MAX_DECODED); +} +#else +static int +valid_png_budget(const uint8_t *data, size_t size) +{ + static const uint8_t signature[8] = + {0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}; + uint32_t width; + uint32_t height; + uint64_t row_bits; + uint64_t decoded; + unsigned bit_depth; + unsigned color_type; + unsigned channels; + + if (size < 33 || memcmp(data, signature, sizeof(signature)) || + read_be32(data + 8) != 13 || memcmp(data + 12, "IHDR", 4)) + return (0); + + width = read_be32(data + 16); + height = read_be32(data + 20); + bit_depth = data[24]; + color_type = data[25]; + if (!width || !height || width > IMAGE_CODEC_MAX_DIMENSION || + height > IMAGE_CODEC_MAX_DIMENSION || data[26] != 0 || + data[27] != 0 || data[28] > 1) + return (0); + + switch (color_type) + { + case 0 : + channels = 1; + if (bit_depth != 1 && bit_depth != 2 && bit_depth != 4 && + bit_depth != 8 && bit_depth != 16) + return (0); + break; + case 2 : + channels = 3; + if (bit_depth != 8 && bit_depth != 16) + return (0); + break; + case 3 : + channels = 1; + if (bit_depth != 1 && bit_depth != 2 && bit_depth != 4 && + bit_depth != 8) + return (0); + break; + case 4 : + channels = 2; + if (bit_depth != 8 && bit_depth != 16) + return (0); + break; + case 6 : + channels = 4; + if (bit_depth != 8 && bit_depth != 16) + return (0); + break; + default : + return (0); + } + + row_bits = (uint64_t)width * channels * bit_depth; + decoded = ((row_bits + 7U) / 8U + 1U) * height; + return (decoded <= IMAGE_CODEC_MAX_DECODED); +} +#endif + +static int +valid_codec_budget(const uint8_t *data, size_t size) +{ +#if defined(CUPSFILTERS_IMAGE_CODEC_JPEG) + return (valid_jpeg_budget(data, size)); +#elif defined(CUPSFILTERS_IMAGE_CODEC_TIFF) + return (valid_tiff_budget(data, size)); +#else + return (valid_png_budget(data, size)); +#endif +} + +static FILE * +open_input(const uint8_t *data, size_t size) +{ +#if defined(CUPSFILTERS_IMAGE_CODEC_TIFF) + FILE *fp = tmpfile(); + + if (!fp) + return (NULL); + if (fwrite(data, 1, size, fp) != size || fflush(fp) || fseek(fp, 0, SEEK_SET)) + { + fclose(fp); + return (NULL); + } + return (fp); +#else + return (fmemopen((void *)data, size, "rb")); +#endif +} + +typedef struct codec_run_s +{ + uint32_t image_hash; + unsigned scanline_failures; + int decoded; +} codec_run_t; + +static void +hash_pixels(uint32_t *hash, const cf_ib_t *pixels, size_t size) +{ + for (size_t i = 0; i < size; i ++) + { + *hash ^= pixels[i]; + *hash *= 16777619U; + } +} + +static codec_run_t +run_codec(const uint8_t *data, size_t size, uint32_t input_hash, + uint8_t sentinel) +{ + static const cf_icspace_t primary_modes[] = { + CF_IMAGE_RGB, CF_IMAGE_WHITE, CF_IMAGE_RGB_CMYK, + CF_IMAGE_CMYK, CF_IMAGE_CMY}; + static const cf_icspace_t secondary_modes[] = { + CF_IMAGE_WHITE, CF_IMAGE_RGB, CF_IMAGE_RGB, + CF_IMAGE_RGB, CF_IMAGE_RGB}; + cf_image_t *image; + cf_ib_t lut[256]; + cf_ib_t *row = NULL; + cf_ib_t *column = NULL; + FILE *fp; + codec_run_t result = {2166136261U, 0, 0}; + unsigned width; + unsigned height; + unsigned depth; + unsigned mode; + unsigned rows[5]; + unsigned row_count = 3; + size_t row_size; + size_t column_size; + + mode = input_hash % (sizeof(primary_modes) / sizeof(primary_modes[0])); + for (unsigned i = 0; i < sizeof(lut); i ++) + lut[i] = (input_hash & 0x20U) ? (cf_ib_t)(255U - i) : (cf_ib_t)i; + + fp = open_input(data, size); + if (!fp) + return (result); + +#if defined(CUPSFILTERS_IMAGE_CODEC_TIFF) + memset(&tiff_read_state, 0, sizeof(tiff_read_state)); + tiff_read_state.active = 1; + tiff_read_state.sentinel = sentinel; + tiff_read_state.first_failed_row = UINT32_MAX; +#else + (void)sentinel; +#endif + image = open_codec(fp, primary_modes[mode], secondary_modes[mode], + 50 + (int)(input_hash % 151U), + (int)((input_hash >> 8) % 361U) - 180, + (input_hash & 0x40U) ? lut : NULL); +#if defined(CUPSFILTERS_IMAGE_CODEC_TIFF) + tiff_read_state.active = 0; + result.scanline_failures = tiff_read_state.failures; +#endif + /* The selected codec owns and closes fp. */ + if (!image) + return (result); + result.decoded = 1; + + width = cfImageGetWidth(image); + height = cfImageGetHeight(image); + depth = (unsigned)cfImageGetDepth(image); + if (!width || !height || !depth || width > IMAGE_CODEC_MAX_DIMENSION || + height > IMAGE_CODEC_MAX_DIMENSION || + (uint64_t)width * depth > IMAGE_CODEC_MAX_DECODED || + (uint64_t)height * depth > IMAGE_CODEC_MAX_DECODED) + goto cleanup; + + row_size = (size_t)width * depth; + column_size = (size_t)height * depth; + row = (cf_ib_t *)malloc(row_size); + column = (cf_ib_t *)malloc(column_size); + if (!row || !column) + goto cleanup; + + cfImageSetMaxTiles(image, 10 + (int)((input_hash >> 16) % 3U) * 8); + rows[0] = 0; + rows[1] = height / 2; + rows[2] = height - 1; +#if defined(CUPSFILTERS_IMAGE_CODEC_TIFF) + if (result.scanline_failures && + tiff_read_state.first_failed_row < height) + { + rows[row_count ++] = tiff_read_state.first_failed_row; + rows[row_count ++] = height - 1 - tiff_read_state.first_failed_row; + } +#endif + hash_pixels(&result.image_hash, (const cf_ib_t *)&width, sizeof(width)); + hash_pixels(&result.image_hash, (const cf_ib_t *)&height, sizeof(height)); + hash_pixels(&result.image_hash, (const cf_ib_t *)&depth, sizeof(depth)); + for (unsigned i = 0; i < row_count; i ++) + { + memset(row, 0x3c, row_size); + if (!cfImageGetRow(image, 0, (int)rows[i], (int)width, row)) + hash_pixels(&result.image_hash, row, row_size); + } + for (unsigned x_index = 0; x_index < 3; x_index ++) + { + unsigned x = x_index == 0 ? 0 : (x_index == 1 ? width / 2 : width - 1); + + memset(column, 0xc3, column_size); + if (!cfImageGetCol(image, (int)x, 0, (int)height, column)) + hash_pixels(&result.image_hash, column, column_size); + } + +cleanup: + free(column); + free(row); + close_image(image); + return (result); +} + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + codec_run_t first; + uint32_t input_hash; + + if (!data || size > CUPSFILTERS_IMAGE_CODEC_MAX_INPUT || + !valid_codec_budget(data, size)) + return (0); + + input_hash = hash_bytes(data, size); + first = run_codec(data, size, input_hash, 0xa5); +#if defined(CUPSFILTERS_IMAGE_CODEC_TIFF) && \ + defined(CUPSFILTERS_IMAGE_CODEC_TIFF_SCANLINE_ORACLE) + if (first.scanline_failures) + { + codec_run_t second = run_codec(data, size, input_hash, 0x5a); + + if (first.decoded != second.decoded || + (first.decoded && second.decoded && + first.image_hash != second.image_hash)) + __builtin_trap(); + } +#endif + return (0); +} diff --git a/parser-fuzzers/harnesses/fuzz_png_bounded_format.c b/parser-fuzzers/harnesses/fuzz_png_bounded_format.c new file mode 100644 index 0000000..a905148 --- /dev/null +++ b/parser-fuzzers/harnesses/fuzz_png_bounded_format.c @@ -0,0 +1,44 @@ +#include +#include +#include + +#define LLVMFuzzerTestOneInput cf_fuzz_png_unbounded_test_one_input +#include "fuzz_cupsfilters_image_codec.c" +#undef LLVMFuzzerTestOneInput + +static uint32_t cf_fuzz_png_chunk_length(const uint8_t *data) { + return ((uint32_t)data[0] << 24) | ((uint32_t)data[1] << 16) | + ((uint32_t)data[2] << 8) | data[3]; +} + +static int cf_fuzz_png_chunks_fit_input(const uint8_t *data, size_t size) { + static const uint8_t signature[8] = {0x89, 'P', 'N', 'G', '\r', '\n', + 0x1a, '\n'}; + size_t offset = sizeof(signature); + + if (!data || size < sizeof(signature) || + memcmp(data, signature, sizeof(signature)) != 0) { + return 0; + } + + while (offset <= size && size - offset >= 12U) { + uint32_t length = cf_fuzz_png_chunk_length(data + offset); + const uint8_t *type = data + offset + 4U; + + if ((size_t)length > size - offset - 12U) { + return 0; + } + offset += (size_t)length + 12U; + if (memcmp(type, "IEND", 4U) == 0) { + return 1; + } + } + return 0; +} + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + if (!cf_fuzz_png_chunks_fit_input(data, size)) { + return 0; + } + return cf_fuzz_png_unbounded_test_one_input(data, size); +} diff --git a/parser-fuzzers/harnesses/fuzz_pwg_scale_state.c b/parser-fuzzers/harnesses/fuzz_pwg_scale_state.c new file mode 100644 index 0000000..1c19dca --- /dev/null +++ b/parser-fuzzers/harnesses/fuzz_pwg_scale_state.c @@ -0,0 +1,426 @@ +#define _GNU_SOURCE + +#include "runtime.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#define CF_FUZZ_PWG_SCALE_MAGIC "PWGSCL1" +#define CF_FUZZ_PWG_SCALE_SELECTORS 8U +#define CF_FUZZ_PWG_SCALE_MAX_MATERIAL 4096U + +#if !defined(CF_FUZZ_PWG_SCALE_UP) && !defined(CF_FUZZ_PWG_SCALE_DOWN) +#error "Select one PWG scale direction" +#endif + +#if defined(CF_FUZZ_PWG_SCALE_UP) && defined(CF_FUZZ_PWG_SCALE_DOWN) +#error "Select only one PWG scale direction" +#endif + +typedef struct cf_fuzz_pwg_scale_format_s { + cups_cspace_t color_space; + unsigned bits_per_color; + unsigned bits_per_pixel; + unsigned num_colors; +} cf_fuzz_pwg_scale_format_t; + +typedef struct cf_fuzz_pwg_scale_trace_s { + unsigned input_pages; + unsigned output_pages; + unsigned raise_events; + unsigned reduce_events; +} cf_fuzz_pwg_scale_trace_t; + +static const unsigned cf_fuzz_pwg_scale_dimensions[] = { + 1U, 2U, 3U, 4U, 7U, 8U, 15U, 16U, 31U, 32U, +}; + +static const unsigned cf_fuzz_pwg_scale_factors[] = {2U, 3U, 4U}; + +static const cf_fuzz_pwg_scale_format_t cf_fuzz_pwg_scale_input_formats[] = { + {CUPS_CSPACE_K, 1U, 1U, 1U}, + {CUPS_CSPACE_W, 8U, 8U, 1U}, + {CUPS_CSPACE_SRGB, 8U, 24U, 3U}, +}; + +static const cf_fuzz_pwg_scale_format_t cf_fuzz_pwg_scale_output_formats[] = { + {CUPS_CSPACE_K, 1U, 1U, 1U}, + {CUPS_CSPACE_W, 8U, 8U, 1U}, + {CUPS_CSPACE_RGB, 8U, 24U, 3U}, + {CUPS_CSPACE_CMYK, 8U, 32U, 4U}, +}; + +static void cf_fuzz_pwg_scale_log(void *data, cf_loglevel_t level, + const char *message, ...) { + cf_fuzz_pwg_scale_trace_t *trace = (cf_fuzz_pwg_scale_trace_t *)data; + + (void)level; + if (!trace || !message) { + return; + } + if (strstr(message, "Input page %d")) { + trace->input_pages++; + } else if (strstr(message, "Output page %d")) { + trace->output_pages++; + } else if (strstr(message, "Raising by factor %d")) { + trace->raise_events++; + } else if (strstr(message, "Reducing by factor %d")) { + trace->reduce_events++; + } +} + +static int cf_fuzz_pwg_scale_not_canceled(void *data) { + (void)data; + return 0; +} + +static void cf_fuzz_pwg_scale_close_fd(int *fd) { + if (*fd >= 0) { + if (fcntl(*fd, F_GETFD) >= 0 || errno != EBADF) { + (void)close(*fd); + } + *fd = -1; + } +} + +static uint8_t cf_fuzz_pwg_scale_material(const uint8_t *material, + size_t material_size, + unsigned pattern, size_t offset) { + uint8_t value = material_size + ? material[(offset + pattern * 257U) % material_size] + : (uint8_t)(offset * 131U + pattern * 67U); + + switch (pattern % 6U) { + case 0: + return value; + case 1: + return 0x00U; + case 2: + return 0xffU; + case 3: + return offset & 1U ? 0xaaU : 0x55U; + case 4: + return (uint8_t)(offset & 0xffU); + default: + return (uint8_t)(value ^ (uint8_t)(offset * 17U)); + } +} + +static int cf_fuzz_pwg_scale_write_input( + int fd, const uint8_t selector[CF_FUZZ_PWG_SCALE_SELECTORS], + const uint8_t *material, size_t material_size, unsigned width, + unsigned height, unsigned x_dpi, unsigned y_dpi, + const cf_fuzz_pwg_scale_format_t *format, unsigned pages) { + cups_page_header2_t header; + cups_raster_t *raster = NULL; + uint8_t *line = NULL; + unsigned bytes_per_line = (width * format->bits_per_pixel + 7U) / 8U; + int ok = 0; + + if (!bytes_per_line || bytes_per_line > 4096U) { + return 0; + } + raster = cupsRasterOpen(fd, CUPS_RASTER_WRITE_PWG); + line = (uint8_t *)malloc(bytes_per_line); + if (!raster || !line) { + goto done; + } + + for (unsigned page = 0; page < pages; page++) { + memset(&header, 0, sizeof(header)); + memcpy(header.MediaClass, "PwgRaster", sizeof("PwgRaster")); + memcpy(header.MediaType, "Plain", sizeof("Plain")); + memcpy(header.cupsPageSizeName, "Tiny", sizeof("Tiny")); + header.HWResolution[0] = x_dpi; + header.HWResolution[1] = y_dpi; + header.PageSize[0] = (unsigned)((uint64_t)width * 72U / x_dpi); + header.PageSize[1] = (unsigned)((uint64_t)height * 72U / y_dpi); + header.ImagingBoundingBox[2] = width; + header.ImagingBoundingBox[3] = height; + header.cupsPageSize[0] = (float)width * 72.0f / (float)x_dpi; + header.cupsPageSize[1] = (float)height * 72.0f / (float)y_dpi; + header.cupsImagingBBox[2] = header.cupsPageSize[0]; + header.cupsImagingBBox[3] = header.cupsPageSize[1]; + header.cupsWidth = width; + header.cupsHeight = height; + header.cupsBitsPerColor = format->bits_per_color; + header.cupsBitsPerPixel = format->bits_per_pixel; + header.cupsBytesPerLine = bytes_per_line; + header.cupsColorOrder = CUPS_ORDER_CHUNKED; + header.cupsColorSpace = format->color_space; + header.cupsCompression = 0U; + header.cupsRowCount = 1U; + header.cupsRowFeed = 1U; + header.cupsRowStep = 1U; + header.cupsNumColors = format->num_colors; + header.NumCopies = 1U; + header.cupsInteger[CUPS_RASTER_PWG_CrossFeedTransform] = 1U; + header.cupsInteger[CUPS_RASTER_PWG_FeedTransform] = 1U; + header.cupsInteger[CUPS_RASTER_PWG_ImageBoxRight] = width; + header.cupsInteger[CUPS_RASTER_PWG_ImageBoxBottom] = height; + + if (!cupsRasterWriteHeader2(raster, &header)) { + goto done; + } + for (unsigned row = 0; row < height; row++) { + for (unsigned column = 0; column < bytes_per_line; column++) { + size_t offset = ((size_t)page * height + row) * bytes_per_line + column; + line[column] = cf_fuzz_pwg_scale_material( + material, material_size, selector[6] + selector[7], offset); + } + if (cupsRasterWritePixels(raster, line, bytes_per_line) != + bytes_per_line) { + goto done; + } + } + } + ok = 1; + +done: + free(line); + if (raster) { + cupsRasterClose(raster); + } + return ok; +} + +static int cf_fuzz_pwg_scale_write_ppd( + FILE *stream, unsigned page_width, unsigned page_height, + unsigned output_x_dpi, unsigned output_y_dpi, + const cf_fuzz_pwg_scale_format_t *format, cups_order_t order) { + return fprintf( + stream, + "*PPD-Adobe: \"4.3\"\n" + "*FormatVersion: \"4.3\"\n" + "*FileVersion: \"1.0\"\n" + "*LanguageVersion: English\n" + "*LanguageEncoding: ISOLatin1\n" + "*Manufacturer: \"OpenPrinting\"\n" + "*ModelName: \"PWG scale state\"\n" + "*ShortNickName: \"PWG scale state\"\n" + "*NickName: \"PWG scale state\"\n" + "*PCFileName: \"PWGSCALE.PPD\"\n" + "*Product: \"(PWG scale state)\"\n" + "*PSVersion: \"(3010) 0\"\n" + "*cupsVersion: 2.4\n" + "*cupsFilter: \"image/pwg-raster 0 pwgtoraster\"\n" + "*OpenUI *PageSize/Page Size: PickOne\n" + "*DefaultPageSize: Tiny\n" + "*PageSize Tiny/Tiny: \"<>setpagedevice\"\n" + "*CloseUI: *PageSize\n" + "*OpenUI *PageRegion/Page Region: PickOne\n" + "*DefaultPageRegion: Tiny\n" + "*PageRegion Tiny/Tiny: \"<>setpagedevice\"\n" + "*CloseUI: *PageRegion\n" + "*DefaultImageableArea: Tiny\n" + "*ImageableArea Tiny/Tiny: \"0 0 %u %u\"\n" + "*DefaultPaperDimension: Tiny\n" + "*PaperDimension Tiny/Tiny: \"%u %u\"\n" + "*OpenUI *ColorModel/Color: PickOne\n" + "*DefaultColorModel: Test\n" + "*ColorModel Test/Test: \"<>setpagedevice\"\n" + "*CloseUI: *ColorModel\n" + "*OpenUI *Resolution/Resolution: PickOne\n" + "*DefaultResolution: Testdpi\n" + "*Resolution Testdpi/Test dpi: \"<>setpagedevice\"\n" + "*CloseUI: *Resolution\n", + page_width, page_height, page_width, page_height, page_width, + page_height, page_width, page_height, + (unsigned)format->color_space, (unsigned)order, + format->bits_per_color, format->bits_per_pixel, output_x_dpi, + output_y_dpi) >= 0; +} + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + static const size_t magic_size = sizeof(CF_FUZZ_PWG_SCALE_MAGIC) - 1U; + const uint8_t *selector; + const uint8_t *material; + const cf_fuzz_pwg_scale_format_t *input_format; + const cf_fuzz_pwg_scale_format_t *output_format; + char input_name[] = "/tmp/cupsfilters-fuzz-pwg-scale-input.XXXXXX"; + char ppd_name[] = "/tmp/cupsfilters-fuzz-pwg-scale-ppd.XXXXXX"; + char options_text[256]; + cf_filter_data_t filter_data; + cf_fuzz_pwg_scale_trace_t trace; + cups_option_t *options = NULL; + unsigned base_width; + unsigned base_height; + unsigned x_factor; + unsigned y_factor; + unsigned input_width; + unsigned input_height; + unsigned input_x_dpi; + unsigned input_y_dpi; + unsigned output_x_dpi; + unsigned output_y_dpi; + unsigned output_width; + unsigned output_height; + unsigned pages; + cups_order_t output_order; + size_t material_size; + int input_fd = -1; + int output_fd = -1; + int ppd_fd = -1; + int ppd_loaded = 0; + int status = 1; + + if (!data || size < magic_size + CF_FUZZ_PWG_SCALE_SELECTORS || + size > magic_size + CF_FUZZ_PWG_SCALE_SELECTORS + + CF_FUZZ_PWG_SCALE_MAX_MATERIAL || + memcmp(data, CF_FUZZ_PWG_SCALE_MAGIC, magic_size) != 0) { + return 0; + } + + selector = data + magic_size; + material = selector + CF_FUZZ_PWG_SCALE_SELECTORS; + material_size = size - magic_size - CF_FUZZ_PWG_SCALE_SELECTORS; + base_width = cf_fuzz_pwg_scale_dimensions[ + selector[0] % (sizeof(cf_fuzz_pwg_scale_dimensions) / + sizeof(cf_fuzz_pwg_scale_dimensions[0]))]; + base_height = cf_fuzz_pwg_scale_dimensions[ + selector[1] % (sizeof(cf_fuzz_pwg_scale_dimensions) / + sizeof(cf_fuzz_pwg_scale_dimensions[0]))]; + x_factor = cf_fuzz_pwg_scale_factors[ + selector[2] % (sizeof(cf_fuzz_pwg_scale_factors) / + sizeof(cf_fuzz_pwg_scale_factors[0]))]; + y_factor = cf_fuzz_pwg_scale_factors[ + selector[3] % (sizeof(cf_fuzz_pwg_scale_factors) / + sizeof(cf_fuzz_pwg_scale_factors[0]))]; + input_format = &cf_fuzz_pwg_scale_input_formats[ + selector[4] % (sizeof(cf_fuzz_pwg_scale_input_formats) / + sizeof(cf_fuzz_pwg_scale_input_formats[0]))]; + output_format = &cf_fuzz_pwg_scale_output_formats[ + (selector[4] >> 4U) % (sizeof(cf_fuzz_pwg_scale_output_formats) / + sizeof(cf_fuzz_pwg_scale_output_formats[0]))]; + output_order = (cups_order_t)(selector[5] % 3U); + pages = 1U + ((selector[5] >> 4U) & 1U); + +#ifdef CF_FUZZ_PWG_SCALE_UP + input_width = base_width; + input_height = base_height; + input_x_dpi = 72U; + input_y_dpi = 72U; + output_x_dpi = 72U * x_factor; + output_y_dpi = 72U * y_factor; + output_width = base_width * x_factor; + output_height = base_height * y_factor; +#else + input_width = base_width * x_factor; + input_height = base_height * y_factor; + input_x_dpi = 72U * x_factor; + input_y_dpi = 72U * y_factor; + output_x_dpi = 72U; + output_y_dpi = 72U; + output_width = base_width; + output_height = base_height; +#endif + + cf_fuzz_init_runtime(); + memset(&filter_data, 0, sizeof(filter_data)); + memset(&trace, 0, sizeof(trace)); + input_fd = mkstemp(input_name); + if (input_fd < 0) { + goto cleanup; + } + unlink(input_name); + if (!cf_fuzz_pwg_scale_write_input( + input_fd, selector, material, material_size, input_width, + input_height, input_x_dpi, input_y_dpi, input_format, pages) || + lseek(input_fd, 0, SEEK_SET) < 0) { + goto cleanup; + } + + output_fd = open("/dev/null", O_WRONLY); + if (output_fd < 0) { + goto cleanup; + } + ppd_fd = mkstemp(ppd_name); + if (ppd_fd < 0) { + goto cleanup; + } + { + FILE *ppd_stream = fdopen(ppd_fd, "w"); + int ppd_ok; + + if (!ppd_stream) { + goto cleanup; + } + ppd_fd = -1; + ppd_ok = cf_fuzz_pwg_scale_write_ppd( + ppd_stream, base_width, base_height, output_x_dpi, output_y_dpi, + output_format, output_order); + if (fclose(ppd_stream) != 0 || !ppd_ok) { + goto cleanup; + } + } + + if (snprintf(options_text, sizeof(options_text), + "PageSize=Tiny PageRegion=Tiny ColorModel=Test " + "Resolution=Testdpi cm-calibration=true emit-jcl=false") < 0) { + goto cleanup; + } + filter_data.printer = (char *)"oss-fuzz"; + filter_data.job_id = 1; + filter_data.job_user = (char *)"fuzzer"; +#ifdef CF_FUZZ_PWG_SCALE_UP + filter_data.job_title = (char *)"pwg-scale-up-state"; +#else + filter_data.job_title = (char *)"pwg-scale-down-state"; +#endif + filter_data.copies = 1; + filter_data.content_type = (char *)"image/pwg-raster"; + filter_data.final_content_type = + (char *)"application/vnd.cups-raster"; + filter_data.num_options = cupsParseOptions(options_text, 0, &options); + filter_data.options = options; + filter_data.back_pipe[0] = filter_data.back_pipe[1] = -1; + filter_data.side_pipe[0] = filter_data.side_pipe[1] = -1; + filter_data.logfunc = cf_fuzz_pwg_scale_log; + filter_data.logdata = &trace; + filter_data.iscanceledfunc = cf_fuzz_pwg_scale_not_canceled; + + if (ppdFilterLoadPPDFile(&filter_data, ppd_name) != 0) { + goto cleanup; + } + ppd_loaded = 1; + status = cfFilterPWGToRaster(input_fd, output_fd, 1, &filter_data, NULL); + input_fd = -1; + output_fd = -1; + + if (getenv("CF_FUZZ_TRACE_STATE")) { + fprintf(stderr, + "pwg-scale status=%d input=%ux%u@%ux%u output=%ux%u@%ux%u " + "factor=%ux%u input-mode=%u output-mode=%u order=%u pages=%u " + "input-pages=%u output-pages=%u raise-events=%u " + "reduce-events=%u\n", + status, input_width, input_height, input_x_dpi, input_y_dpi, + output_width, output_height, output_x_dpi, output_y_dpi, x_factor, + y_factor, selector[4] % 3U, (selector[4] >> 4U) % 4U, + (unsigned)output_order, pages, trace.input_pages, + trace.output_pages, trace.raise_events, trace.reduce_events); + } + +cleanup: + cf_fuzz_pwg_scale_close_fd(&ppd_fd); + cf_fuzz_pwg_scale_close_fd(&output_fd); + cf_fuzz_pwg_scale_close_fd(&input_fd); + if (filter_data.options) { + cupsFreeOptions(filter_data.num_options, filter_data.options); + filter_data.options = NULL; + } + if (ppd_loaded) { + ppdFilterFreePPDFile(&filter_data); + } + unlink(ppd_name); + return 0; +} diff --git a/parser-fuzzers/harnesses/fuzz_raster_output_state.c b/parser-fuzzers/harnesses/fuzz_raster_output_state.c new file mode 100644 index 0000000..70e54ce --- /dev/null +++ b/parser-fuzzers/harnesses/fuzz_raster_output_state.c @@ -0,0 +1,361 @@ +#include "control.h" + +#ifdef CF_FUZZ_RASTER_OUTPUT_PCLM_CHAIN +#include +static int cf_fuzz_filter_raster_to_pclm(int inputfd, int outputfd, + int inputseekable, + cf_filter_data_t *data, + void *parameters); +#endif + +#include "direct_route.h" + +#include +#include +#include +#include +#include + +#ifdef CF_FUZZ_RASTER_OUTPUT_PS_WRITE_ERROR_BOUNDARY +#define CF_FUZZ_RASTER_OUTPUT_MAGIC "PSWRITE1" +#elif defined(CF_FUZZ_RASTER_OUTPUT_PS_LIFECYCLE) +#define CF_FUZZ_RASTER_OUTPUT_MAGIC "PSLIFE01" +#else +#define CF_FUZZ_RASTER_OUTPUT_MAGIC "ROSTATE1" +#endif +#define CF_FUZZ_RASTER_OUTPUT_SELECTORS 8U +#define CF_FUZZ_RASTER_OUTPUT_MAX_MATERIAL 4096U + +typedef struct cf_fuzz_raster_output_format_s { + cups_cspace_t color_space; + unsigned colors; + unsigned bits_per_color; +} cf_fuzz_raster_output_format_t; + +static const unsigned cf_fuzz_raster_output_widths[] = { + 1U, 2U, 7U, 8U, 15U, 31U, 63U, 127U, 128U, 255U, 256U, +}; +static const unsigned cf_fuzz_raster_output_heights[] = { + 1U, 2U, 3U, 4U, 8U, 16U, +}; +static const unsigned cf_fuzz_raster_output_margins[] = { + 0U, 1U, 2U, 4U, 8U, +}; +#ifdef CF_FUZZ_RASTER_OUTPUT_PS +/* rastertops has distinct 1-bpc expansion, 16-bpc DataSource, SW decode, + * process-color, and fallback-device paths. Keep their finite products + * explicit so every branch is reachable without malformed Raster headers. */ +static const cf_fuzz_raster_output_format_t cf_fuzz_raster_output_formats[] = { + {CUPS_CSPACE_K, 1U, 1U}, + {CUPS_CSPACE_K, 1U, 8U}, + {CUPS_CSPACE_K, 1U, 16U}, + {CUPS_CSPACE_W, 1U, 1U}, + {CUPS_CSPACE_W, 1U, 8U}, + {CUPS_CSPACE_W, 1U, 16U}, + {CUPS_CSPACE_SW, 1U, 1U}, + {CUPS_CSPACE_SW, 1U, 8U}, + {CUPS_CSPACE_SW, 1U, 16U}, + {CUPS_CSPACE_RGB, 3U, 1U}, + {CUPS_CSPACE_RGB, 3U, 8U}, + {CUPS_CSPACE_RGB, 3U, 16U}, + {CUPS_CSPACE_SRGB, 3U, 1U}, + {CUPS_CSPACE_SRGB, 3U, 8U}, + {CUPS_CSPACE_SRGB, 3U, 16U}, + {CUPS_CSPACE_ADOBERGB, 3U, 1U}, + {CUPS_CSPACE_ADOBERGB, 3U, 8U}, + {CUPS_CSPACE_ADOBERGB, 3U, 16U}, + {CUPS_CSPACE_CMY, 3U, 1U}, + {CUPS_CSPACE_CMY, 3U, 8U}, + {CUPS_CSPACE_CMY, 3U, 16U}, + {CUPS_CSPACE_CMYK, 4U, 1U}, + {CUPS_CSPACE_CMYK, 4U, 8U}, + {CUPS_CSPACE_CMYK, 4U, 16U}, + {CUPS_CSPACE_DEVICE1, 1U, 8U}, + {CUPS_CSPACE_DEVICE4, 4U, 8U}, +}; +#else +static const cf_fuzz_raster_output_format_t cf_fuzz_raster_output_formats[] = { + {CUPS_CSPACE_K, 1U, 1U}, + {CUPS_CSPACE_K, 1U, 8U}, + {CUPS_CSPACE_K, 1U, 16U}, + {CUPS_CSPACE_W, 1U, 1U}, + {CUPS_CSPACE_W, 1U, 8U}, + {CUPS_CSPACE_W, 1U, 16U}, + {CUPS_CSPACE_RGB, 3U, 8U}, + {CUPS_CSPACE_RGB, 3U, 16U}, + {CUPS_CSPACE_SRGB, 3U, 8U}, + {CUPS_CSPACE_ADOBERGB, 3U, 8U}, + {CUPS_CSPACE_CMYK, 4U, 8U}, + {CUPS_CSPACE_CMYK, 4U, 16U}, + {CUPS_CSPACE_DEVICE1, 1U, 8U}, + {CUPS_CSPACE_DEVICE4, 4U, 8U}, +}; +#endif + +static void cf_fuzz_raster_output_page_state( + const uint8_t selector[CF_FUZZ_RASTER_OUTPUT_SELECTORS], unsigned page, + const cf_fuzz_raster_output_format_t **format, unsigned *width, + unsigned *height) { + size_t format_index = selector[1]; + size_t width_index = selector[0] & 0x0fU; + size_t height_index = selector[0] >> 4U; + +#ifdef CF_FUZZ_RASTER_OUTPUT_PS + if (page) { + format_index += (size_t)page * (1U + selector[4] % 13U); + width_index += (size_t)page * (1U + (selector[5] & 0x0fU)); + height_index += (size_t)page * (1U + (selector[5] >> 4U)); + } +#else + (void)page; +#endif + *format = &cf_fuzz_raster_output_formats[ + format_index % (sizeof(cf_fuzz_raster_output_formats) / + sizeof(cf_fuzz_raster_output_formats[0]))]; + *width = cf_fuzz_raster_output_widths[ + width_index % (sizeof(cf_fuzz_raster_output_widths) / + sizeof(cf_fuzz_raster_output_widths[0]))]; + *height = cf_fuzz_raster_output_heights[ + height_index % (sizeof(cf_fuzz_raster_output_heights) / + sizeof(cf_fuzz_raster_output_heights[0]))]; +} + +#ifdef CF_FUZZ_RASTER_OUTPUT_PCLM_CHAIN +static int cf_fuzz_filter_raster_to_pclm(int inputfd, int outputfd, + int inputseekable, + cf_filter_data_t *data, + void *parameters) { + cf_filter_out_format_t format = CF_FILTER_OUT_FORMAT_PCLM; + FILE *intermediate; + int first_input = -1; + int first_output = -1; + int second_input = -1; + int second_output = -1; + int status = 1; + + (void)inputseekable; + (void)parameters; + intermediate = tmpfile(); + if (!intermediate || + (first_input = dup(inputfd)) < 0 || + (first_output = dup(fileno(intermediate))) < 0) { + goto done; + } + status = cfFilterRasterToPWG(first_input, first_output, 1, data, NULL); + first_input = -1; + first_output = -1; + if (status != 0 || fflush(intermediate) != 0 || + fseek(intermediate, 0, SEEK_SET) != 0 || + (second_input = dup(fileno(intermediate))) < 0 || + (second_output = dup(outputfd)) < 0) { + goto done; + } + status = cfFilterPWGToPDF(second_input, second_output, 1, data, &format); + second_input = -1; + second_output = -1; + +done: + if (first_input >= 0) close(first_input); + if (first_output >= 0) close(first_output); + if (second_input >= 0) close(second_input); + if (second_output >= 0) close(second_output); + if (intermediate) fclose(intermediate); + return status; +} +#endif + +static uint8_t cf_fuzz_raster_output_material(const uint8_t *material, + size_t material_size, + unsigned pattern, + unsigned page, size_t offset) { + uint8_t value = material_size + ? material[(offset + (size_t)page * 257U) % material_size] + : (uint8_t)(1U + (offset * 131U + page * 67U) % 254U); + +#ifdef CF_FUZZ_RASTER_OUTPUT_PS_WRITE_ERROR_BOUNDARY + { + uint32_t mixed = (uint32_t)offset ^ ((uint32_t)page * 0x9e3779b9U) ^ + ((uint32_t)value << 24U); + + /* Keep the /dev/full case incompressible enough for fwrite to flush its + * real stdio buffer and report ENOSPC inside rastertops. */ + mixed ^= mixed >> 16U; + mixed *= 0x7feb352dU; + mixed ^= mixed >> 15U; + mixed *= 0x846ca68bU; + mixed ^= mixed >> 16U; + return (uint8_t)(mixed >> 24U); + } +#endif + + switch (pattern % 6U) { + case 0: + return value; + case 1: + return 0x00U; + case 2: + return 0xffU; + case 3: + return offset & 1U ? 0xaaU : 0x55U; + case 4: + return (uint8_t)(value ^ (uint8_t)(page * 0x31U)); + default: + return (uint8_t)(offset & 0xffU); + } +} + +static uint8_t *cf_fuzz_raster_output_document(const uint8_t selector[8], + const uint8_t *material, + size_t material_size, + size_t *document_size) { + unsigned pages = 1U + selector[7] % 3U; + unsigned left = cf_fuzz_raster_output_margins[ + selector[2] % (sizeof(cf_fuzz_raster_output_margins) / + sizeof(cf_fuzz_raster_output_margins[0]))]; + unsigned right = cf_fuzz_raster_output_margins[ + (selector[2] >> 4U) % (sizeof(cf_fuzz_raster_output_margins) / + sizeof(cf_fuzz_raster_output_margins[0]))]; + unsigned bottom = cf_fuzz_raster_output_margins[ + selector[3] % (sizeof(cf_fuzz_raster_output_margins) / + sizeof(cf_fuzz_raster_output_margins[0]))]; + unsigned top = cf_fuzz_raster_output_margins[ + (selector[3] >> 4U) % (sizeof(cf_fuzz_raster_output_margins) / + sizeof(cf_fuzz_raster_output_margins[0]))]; + size_t total_size = 4U; + uint8_t *document; + size_t offset = 4U; + + for (unsigned page = 0; page < pages; page++) { + const cf_fuzz_raster_output_format_t *format; + unsigned width; + unsigned height; + unsigned bits_per_pixel; + unsigned bytes_per_line; + + cf_fuzz_raster_output_page_state(selector, page, &format, &width, &height); + bits_per_pixel = format->colors * format->bits_per_color; + bytes_per_line = (width * bits_per_pixel + 7U) / 8U; + total_size += sizeof(cups_page_header2_t) + + (size_t)height * bytes_per_line; + } + document = (uint8_t *)malloc(total_size); + if (!document) { + return NULL; + } + memcpy(document, "3SaR", 4U); + for (unsigned page = 0; page < pages; page++) { + const cf_fuzz_raster_output_format_t *format; + unsigned width; + unsigned height; + unsigned bits_per_pixel; + unsigned bytes_per_line; + cups_page_header2_t header; + size_t pixels; + + cf_fuzz_raster_output_page_state(selector, page, &format, &width, &height); + bits_per_pixel = format->colors * format->bits_per_color; + bytes_per_line = (width * bits_per_pixel + 7U) / 8U; + memset(&header, 0, sizeof(header)); + memcpy(header.MediaClass, "PwgRaster", sizeof("PwgRaster")); + memcpy(header.MediaType, "PLAIN", sizeof("PLAIN")); + header.HWResolution[0] = 72U; + header.HWResolution[1] = 72U; + header.PageSize[0] = width + left + right; + header.PageSize[1] = height + bottom + top; + header.ImagingBoundingBox[0] = left; + header.ImagingBoundingBox[1] = bottom; + header.ImagingBoundingBox[2] = left + width; + header.ImagingBoundingBox[3] = bottom + height; + header.cupsPageSize[0] = (float)header.PageSize[0]; + header.cupsPageSize[1] = (float)header.PageSize[1]; + header.cupsImagingBBox[0] = (float)header.ImagingBoundingBox[0]; + header.cupsImagingBBox[1] = (float)header.ImagingBoundingBox[1]; + header.cupsImagingBBox[2] = (float)header.ImagingBoundingBox[2]; + header.cupsImagingBBox[3] = (float)header.ImagingBoundingBox[3]; + header.cupsWidth = width; + header.cupsHeight = height; + header.cupsBitsPerColor = format->bits_per_color; + header.cupsBitsPerPixel = bits_per_pixel; + header.cupsBytesPerLine = bytes_per_line; + header.cupsColorOrder = CUPS_ORDER_CHUNKED; + header.cupsColorSpace = format->color_space; + header.cupsCompression = 0U; + header.cupsRowCount = 1U; + header.cupsRowFeed = 1U; + header.cupsRowStep = 1U; + header.cupsNumColors = format->colors; + header.NumCopies = 1U; + header.Duplex = selector[4] & 1U; + header.Tumble = (selector[4] >> 1U) & 1U; + header.Orientation = (cups_orient_t)(selector[5] % 4U); + + memcpy(document + offset, &header, sizeof(header)); + offset += sizeof(header); + pixels = (size_t)height * bytes_per_line; + for (size_t index = 0; index < pixels; index++) { + document[offset + index] = cf_fuzz_raster_output_material( + material, material_size, selector[6], page, index); + } + offset += pixels; + } + *document_size = total_size; + return document; +} + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + static const size_t magic_size = sizeof(CF_FUZZ_RASTER_OUTPUT_MAGIC) - 1U; + const uint8_t *selector; + const uint8_t *material; + size_t material_size; + size_t document_size = 0; + uint8_t *document; + cf_fuzz_control_t control; + cf_fuzz_run_result_t result; + int executed; + + if (!data || size < magic_size + CF_FUZZ_RASTER_OUTPUT_SELECTORS || + size > magic_size + CF_FUZZ_RASTER_OUTPUT_SELECTORS + + CF_FUZZ_RASTER_OUTPUT_MAX_MATERIAL || + memcmp(data, CF_FUZZ_RASTER_OUTPUT_MAGIC, magic_size) != 0) { + return 0; + } + selector = data + magic_size; + material = selector + CF_FUZZ_RASTER_OUTPUT_SELECTORS; + material_size = size - magic_size - CF_FUZZ_RASTER_OUTPUT_SELECTORS; + document = cf_fuzz_raster_output_document(selector, material, material_size, + &document_size); + if (!document) { + return 0; + } + + for (size_t index = 0; index < sizeof(control); index++) { + ((uint8_t *)&control)[index] = + selector[index % CF_FUZZ_RASTER_OUTPUT_SELECTORS] ^ + (uint8_t)(index * 29U); + } + cf_fuzz_apply_control_policy(&control); +#ifdef CF_FUZZ_RASTER_OUTPUT_PS_WRITE_ERROR_BOUNDARY + control.reserved = CF_FUZZ_DIRECT_FAULT_OUTPUT_FULL; +#elif defined(CF_FUZZ_RASTER_OUTPUT_PS_LIFECYCLE) + { + static const uint8_t fault_modes[] = { + CF_FUZZ_DIRECT_FAULT_NONE, + CF_FUZZ_DIRECT_FAULT_EMPTY_INPUT, + CF_FUZZ_DIRECT_FAULT_INVALID_INPUT, + CF_FUZZ_DIRECT_FAULT_INVALID_OUTPUT, + CF_FUZZ_DIRECT_FAULT_CANCELED, + }; + control.reserved = fault_modes[ + selector[6] % (sizeof(fault_modes) / sizeof(fault_modes[0]))]; + } +#endif + executed = + cf_fuzz_execute_direct(document, document_size, &control, 0, &result); + if (getenv("CF_FUZZ_TRACE_STATE")) { + fprintf(stderr, "%s route_executed=%d status=%d document_size=%zu\n", + CF_FUZZ_TARGET_NAME, executed, result.status, document_size); + } + cf_fuzz_free_run_result(&result); + free(document); + return 0; +} diff --git a/parser-fuzzers/harnesses/fuzz_rastertopclx_compress_state.c b/parser-fuzzers/harnesses/fuzz_rastertopclx_compress_state.c new file mode 100644 index 0000000..9322287 --- /dev/null +++ b/parser-fuzzers/harnesses/fuzz_rastertopclx_compress_state.c @@ -0,0 +1,496 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include + +#ifndef CF_FUZZ_RASTERTOPCLX_SOURCE +#error "CF_FUZZ_RASTERTOPCLX_SOURCE must be a quoted source path" +#endif +#if defined(CF_FUZZ_PCLX_MODE3_CODEC) == defined(CF_FUZZ_PCLX_MODE10_CODEC) +#error "select exactly one PCLX compression codec" +#endif + +#define CF_FUZZ_MAGIC "PCLXCMP1" +#define CF_FUZZ_MAGIC_SIZE 8U +#define CF_FUZZ_SELECTOR_SIZE 12U +#define CF_FUZZ_HEADER_SIZE (CF_FUZZ_MAGIC_SIZE + CF_FUZZ_SELECTOR_SIZE) +#define CF_FUZZ_MAX_MATERIAL 4096U +#define CF_FUZZ_MAX_ROW 1024U + +static unsigned char *cf_fuzz_capture; +static size_t cf_fuzz_capture_capacity; +static size_t cf_fuzz_capture_size; + +static int +cf_fuzz_capture_printf(const char *format, ...) +{ + (void)format; + return 0; +} + +static size_t +cf_fuzz_capture_fwrite(const void *data, size_t size, size_t count, FILE *stream) +{ + size_t bytes; + + (void)stream; + if (size != 0U && count > SIZE_MAX / size) + __builtin_trap(); + bytes = size * count; + if (bytes > cf_fuzz_capture_capacity - cf_fuzz_capture_size) + __builtin_trap(); + memcpy(cf_fuzz_capture + cf_fuzz_capture_size, data, bytes); + cf_fuzz_capture_size += bytes; + return count; +} + +#define printf cf_fuzz_capture_printf +#define fwrite cf_fuzz_capture_fwrite +#define main cf_fuzz_unused_rastertopclx_main +#include CF_FUZZ_RASTERTOPCLX_SOURCE +#undef main +#undef fwrite +#undef printf + +static const size_t cf_fuzz_mode3_lengths[] = { + 1U, 2U, 7U, 8U, 9U, 30U, 31U, 32U, 33U, + 254U, 255U, 256U, 257U, 510U, 768U, 1024U +}; + +static const size_t cf_fuzz_mode10_tuples[] = { + 1U, 2U, 3U, 6U, 7U, 8U, 9U, 30U, 31U, 32U, + 84U, 85U, 86U, 254U, 255U, 256U, 257U, 341U +}; + +static unsigned char +cf_fuzz_material_byte(const uint8_t *material, size_t material_size, + size_t index, unsigned int salt) +{ + unsigned char value = material[(index + salt) % material_size]; + return (unsigned char)(value ^ (unsigned char)(salt * 29U + index * 17U)); +} + +static void +cf_fuzz_fill_initial_seed(unsigned char *seed, size_t size, + const uint8_t *material, size_t material_size, + unsigned int pattern) +{ + size_t index; + + for (index = 0; index < size; index ++) + { + switch (pattern % 6U) + { + case 0U : seed[index] = 0x00U; break; + case 1U : seed[index] = 0xffU; break; + case 2U : seed[index] = (index & 1U) ? 0xaaU : 0x55U; break; + case 3U : seed[index] = (unsigned char)index; break; + case 4U : seed[index] = cf_fuzz_material_byte(material, material_size, + index, pattern); break; + default : seed[index] = (unsigned char)(index * 37U + pattern); break; + } + } +} + +static void +cf_fuzz_make_row(unsigned char *row, const unsigned char *previous, + size_t length, const uint8_t *material, size_t material_size, + const uint8_t *selectors, unsigned int row_number, + unsigned int plane) +{ + static const size_t prefix_points[] = { + 0U, 1U, 2U, 3U, 7U, 8U, 30U, 31U, 32U, 254U, 255U, 256U + }; + static const size_t run_points[] = { + 1U, 2U, 3U, 6U, 7U, 8U, 9U, 31U, 32U, 254U, 255U + }; + size_t prefix = prefix_points[selectors[6] % + (sizeof(prefix_points) / + sizeof(prefix_points[0]))]; + size_t run = run_points[selectors[7] % + (sizeof(run_points) / sizeof(run_points[0]))]; + unsigned int pattern = (selectors[5] + row_number + plane) % 8U; + unsigned int delta = 1U + (selectors[8] % 127U); + size_t index; + + memcpy(row, previous, length); + if (prefix > length) + prefix = length; + if (run > length - prefix) + run = length - prefix; + + switch (pattern) + { + case 0U : + break; + case 1U : + if (prefix < length) + row[prefix] ^= (unsigned char)delta; + break; + case 2U : + for (index = prefix; index < prefix + run; index ++) + row[index] = (unsigned char)(previous[index] + delta); + break; + case 3U : + for (index = prefix; index < length; index ++) + row[index] = (unsigned char)(previous[index] + + ((index & 1U) ? delta : 1U)); + break; + case 4U : + for (index = 0; index < length; index ++) + row[index] = cf_fuzz_material_byte(material, material_size, index, + row_number + plane); + break; + case 5U : + memset(row + prefix, 0x00, length - prefix); + break; + case 6U : + memset(row + prefix, 0xff, length - prefix); + break; + default : + for (index = prefix; index < length; index ++) + row[index] ^= (unsigned char)(delta + index * 13U); + break; + } +} + +#ifdef CF_FUZZ_PCLX_MODE3_CODEC +static int +cf_fuzz_decode_mode3(unsigned char *decoded, size_t length, + const unsigned char *compressed, size_t compressed_size) +{ + size_t input = 0U; + size_t output = 0U; + + while (input < compressed_size) + { + unsigned int command = compressed[input ++]; + size_t count = 1U + (command >> 5U); + size_t offset = command & 31U; + + if (offset == 31U) + { + unsigned int extension; + do + { + if (input >= compressed_size) + return 0; + extension = compressed[input ++]; + if (offset > SIZE_MAX - extension) + return 0; + offset += extension; + } + while (extension == 255U); + } + if (offset > length - output) + return 0; + output += offset; + if (count > length - output || count > compressed_size - input) + return 0; + memcpy(decoded + output, compressed + input, count); + input += count; + output += count; + } + return 1; +} +#endif + +#ifdef CF_FUZZ_PCLX_MODE10_CODEC +static int +cf_fuzz_signed_five(unsigned int value) +{ + value &= 31U; + return (value & 16U) ? (int)value - 32 : (int)value; +} + +static int +cf_fuzz_mode10_pixel(unsigned char *decoded, const unsigned char *seed, + size_t pixel, int rgb, const unsigned char *compressed, + size_t compressed_size, size_t *input) +{ + const size_t index = rgb ? pixel * 3U : pixel; + int red; + int green; + int blue; + unsigned int first; + + if (*input >= compressed_size) + return 0; + first = compressed[(*input) ++]; + if (first & 0x80U) + { + unsigned int second; + int seed_red; + int seed_green; + int seed_blue; + + if (*input >= compressed_size) + return 0; + second = compressed[(*input) ++]; + seed_red = seed[index]; + seed_green = rgb ? seed[index + 1U] : seed[index]; + seed_blue = rgb ? (seed[index + 2U] & 0xfeU) : + (seed[index] & 0xfeU); + red = seed_red + cf_fuzz_signed_five(first >> 2U); + green = seed_green + + cf_fuzz_signed_five(((first & 3U) << 3U) | (second >> 5U)); + blue = seed_blue + 2 * cf_fuzz_signed_five(second); + } + else + { + unsigned int second; + unsigned int third; + + if (compressed_size - *input < 2U) + return 0; + second = compressed[(*input) ++]; + third = compressed[(*input) ++]; + red = (int)((first << 1U) | (second >> 7U)); + green = (int)(((second & 0x7fU) << 1U) | (third >> 7U)); + blue = (int)((third & 0x7fU) << 1U); + } + if (red < 0 || red > 255 || green < 0 || green > 255 || + blue < 0 || blue > 255) + return 0; + + if (rgb) + { + decoded[index] = (unsigned char)red; + decoded[index + 1U] = (unsigned char)green; + decoded[index + 2U] = (unsigned char)blue; + } + else + { + if (red != green || blue < red - 1 || blue > red + 1) + return 0; + decoded[index] = (unsigned char)red; + } + return 1; +} + +static int +cf_fuzz_decode_mode10(unsigned char *decoded, const unsigned char *seed, + size_t length, int rgb, + const unsigned char *compressed, size_t compressed_size) +{ + const size_t pixels = rgb ? length / 3U : length; + size_t input = 0U; + size_t pixel = 0U; + + memcpy(decoded, seed, length); + while (input < compressed_size) + { + unsigned int command = compressed[input ++]; + size_t offset = (command >> 3U) & 3U; + size_t count = (command & 7U) + 1U; + int extended = (command & 7U) == 7U; + + if (command & 0xe0U) + return 0; + if (offset == 3U) + { + unsigned int extension; + do + { + if (input >= compressed_size) + return 0; + extension = compressed[input ++]; + if (offset > SIZE_MAX - extension) + return 0; + offset += extension; + } + while (extension == 255U); + } + if (offset > pixels - pixel) + return 0; + pixel += offset; + + for (;;) + { + size_t index; + + if (count > pixels - pixel) + return 0; + for (index = 0U; index < count; index ++, pixel ++) + { + if (!cf_fuzz_mode10_pixel(decoded, seed, pixel, rgb, compressed, + compressed_size, &input)) + return 0; + } + if (!extended) + break; + if (input >= compressed_size) + return 0; + count = compressed[input ++]; + if (count == 0U) + break; + extended = count == 255U; + } + } + return 1; +} + +static int +cf_fuzz_mode10_matches(const unsigned char *decoded, + const unsigned char *line, size_t length, int rgb) +{ + size_t index; + + if (!rgb) + return memcmp(decoded, line, length) == 0; + for (index = 0U; index < length; index += 3U) + { + int blue_difference; + + if (decoded[index] != line[index] || + decoded[index + 1U] != line[index + 1U]) + return 0; + blue_difference = (int)decoded[index + 2U] - (int)line[index + 2U]; + if (blue_difference < -1 || blue_difference > 1) + return 0; + } + return 1; +} +#endif + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + const uint8_t *selectors; + const uint8_t *material; + size_t material_size; + size_t length; + size_t line_padding; + size_t line_capacity; + size_t seed_capacity; + size_t comp_capacity; + size_t planes; + size_t rows; + unsigned char *line = NULL; + unsigned char *seed_history = NULL; + unsigned char *previous = NULL; +#ifdef CF_FUZZ_PCLX_MODE10_CODEC + unsigned char *decoded = NULL; + unsigned char *decoder_seed = NULL; +#endif + size_t row_number; + size_t plane; + + if (!data || size < CF_FUZZ_HEADER_SIZE + 1U || + size > CF_FUZZ_HEADER_SIZE + CF_FUZZ_MAX_MATERIAL || + memcmp(data, CF_FUZZ_MAGIC, CF_FUZZ_MAGIC_SIZE) != 0) + return 0; + + selectors = data + CF_FUZZ_MAGIC_SIZE; + material = data + CF_FUZZ_HEADER_SIZE; + material_size = size - CF_FUZZ_HEADER_SIZE; + +#ifdef CF_FUZZ_PCLX_MODE3_CODEC + length = cf_fuzz_mode3_lengths[selectors[0] % + (sizeof(cf_fuzz_mode3_lengths) / sizeof(cf_fuzz_mode3_lengths[0]))]; + planes = 1U + selectors[2] % 4U; + line_padding = 1U; +#else + PrinterPlanes = (selectors[2] & 1U) ? 3 : 1; + if (PrinterPlanes == 3) + length = 3U * cf_fuzz_mode10_tuples[selectors[0] % + (sizeof(cf_fuzz_mode10_tuples) / + sizeof(cf_fuzz_mode10_tuples[0]))]; + else + length = cf_fuzz_mode3_lengths[selectors[0] % + (sizeof(cf_fuzz_mode3_lengths) / sizeof(cf_fuzz_mode3_lengths[0]))]; + planes = 1U; + line_padding = 3U; +#endif + if (length == 0U || length > CF_FUZZ_MAX_ROW) + return 0; + + rows = 1U + selectors[1] % 8U; + line_capacity = length + line_padding; + seed_capacity = planes * length + line_padding; + comp_capacity = 4U * length; + + line = (unsigned char *)malloc(line_capacity); + seed_history = (unsigned char *)malloc(seed_capacity); + previous = (unsigned char *)malloc(length); +#ifdef CF_FUZZ_PCLX_MODE10_CODEC + decoded = (unsigned char *)malloc(length); + decoder_seed = (unsigned char *)malloc(length); +#endif + CompBuffer = (unsigned char *)malloc(comp_capacity); + cf_fuzz_capture = (unsigned char *)malloc(comp_capacity); + SeedBuffer = seed_history; + if (!line || !seed_history || !previous || !CompBuffer || !cf_fuzz_capture +#ifdef CF_FUZZ_PCLX_MODE10_CODEC + || !decoded || !decoder_seed +#endif + ) + goto cleanup; + + cf_fuzz_capture_capacity = comp_capacity; + cf_fuzz_fill_initial_seed(seed_history, planes * length, material, + material_size, selectors[4]); + memset(seed_history + planes * length, 0x5a, line_padding); +#ifdef CF_FUZZ_PCLX_MODE10_CODEC + memcpy(decoder_seed, seed_history, length); +#endif + SeedInvalid = selectors[3] & 1U; + + for (row_number = 0; row_number < rows; row_number ++) + { + for (plane = 0; plane < planes; plane ++) + { + unsigned char *plane_seed = SeedBuffer + plane * length; + + memcpy(previous, plane_seed, length); + cf_fuzz_make_row(line, previous, length, material, material_size, + selectors, (unsigned int)row_number, + (unsigned int)plane); + memset(line + length, 0xa5, line_padding); + line[length] = (unsigned char)(plane_seed[length] ^ 0xffU); + cf_fuzz_capture_size = 0U; + +#ifdef CF_FUZZ_PCLX_MODE3_CODEC + CompressData(line, (int)length, (int)plane, + (plane + 1U == planes) ? 'W' : 'V', 3); + if (!cf_fuzz_decode_mode3(previous, length, cf_fuzz_capture, + cf_fuzz_capture_size) || + memcmp(previous, line, length) != 0) + __builtin_trap(); +#else + CompressData(line, (int)length, 0, 'W', 10); + if (!cf_fuzz_decode_mode10(decoded, decoder_seed, length, + PrinterPlanes == 3, cf_fuzz_capture, + cf_fuzz_capture_size) || + !cf_fuzz_mode10_matches(decoded, line, length, + PrinterPlanes == 3)) + __builtin_trap(); + memcpy(decoder_seed, decoded, length); +#endif + if (memcmp(plane_seed, line, length) != 0) + __builtin_trap(); + } + SeedInvalid = 0; + } + +cleanup: +#ifdef CF_FUZZ_PCLX_MODE10_CODEC + free(decoder_seed); + free(decoded); +#endif + free(cf_fuzz_capture); + free(CompBuffer); + free(previous); + free(seed_history); + free(line); + cf_fuzz_capture = NULL; + cf_fuzz_capture_capacity = 0U; + cf_fuzz_capture_size = 0U; + CompBuffer = NULL; + SeedBuffer = NULL; + return 0; +} diff --git a/parser-fuzzers/harnesses/fuzz_text_to_text_page_order.c b/parser-fuzzers/harnesses/fuzz_text_to_text_page_order.c new file mode 100644 index 0000000..991b14f --- /dev/null +++ b/parser-fuzzers/harnesses/fuzz_text_to_text_page_order.c @@ -0,0 +1,196 @@ +#include "direct_route.h" + +#include +#include +#include +#include + +#ifdef CF_FUZZ_TEXTTOTEXT_PAGE_ARRAY_BOUNDARY +#define CF_FUZZ_TEXT_ORDER_MAGIC "TXTPAGE1" +#else +#define CF_FUZZ_TEXT_ORDER_MAGIC "TXTSEL01" +#endif + +#define CF_FUZZ_TEXT_ORDER_MAGIC_SIZE 8U +#define CF_FUZZ_TEXT_ORDER_SELECTORS 16U +#define CF_FUZZ_TEXT_ORDER_MAX_PAYLOAD 256U +#define CF_FUZZ_TEXT_ORDER_PAGES 4U + +static int +cf_fuzz_text_order_range_selected(unsigned range, unsigned page) +{ + static const uint8_t selected[7][CF_FUZZ_TEXT_ORDER_PAGES] = { + {1, 0, 0, 0}, + {1, 1, 0, 0}, + {0, 1, 1, 0}, + {1, 1, 1, 1}, + {0, 1, 1, 1}, + {1, 0, 1, 0}, + {1, 1, 1, 1}, + }; + + return page >= 1U && page <= CF_FUZZ_TEXT_ORDER_PAGES && + selected[range % 7U][page - 1U]; +} + +static int +cf_fuzz_text_order_set_selected(unsigned page_set, unsigned page) +{ + if (page_set % 3U == 1U) + return page & 1U; + if (page_set % 3U == 2U) + return !(page & 1U); + return 1; +} + +static uint8_t * +cf_fuzz_text_order_document(const uint8_t *payload, size_t payload_size, + size_t *document_size) +{ + const size_t capacity = CF_FUZZ_TEXT_ORDER_PAGES * 16U; + uint8_t *document = (uint8_t *)malloc(capacity); + size_t used = 0; + unsigned page; + + if (!document) + return NULL; + for (page = 0; page < CF_FUZZ_TEXT_ORDER_PAGES; page ++) + { + const uint8_t material = payload[page % payload_size]; + const unsigned body_size = 1U + material % 6U; + unsigned index; + + document[used ++] = (uint8_t)('A' + page); + for (index = 0; index < body_size; index ++) + document[used ++] = + (uint8_t)('a' + (material + index * 7U + page * 11U) % 26U); + document[used ++] = '\n'; + if (page + 1U < CF_FUZZ_TEXT_ORDER_PAGES) + document[used ++] = '\f'; + } + *document_size = used; + return document; +} + +static void +cf_fuzz_text_order_control(const uint8_t *selector, cf_fuzz_control_t *control) +{ + memset(control, 0, sizeof(*control)); + control->page_size = 8U; + control->sides = 0U; + control->position = 4U; + control->number_up = selector[0] % 7U; + control->ppd_profile = selector[1] % 3U; + control->mirror = selector[2] % 3U; + control->route_mode = 3U; + +#ifdef CF_FUZZ_TEXTTOTEXT_PAGE_ARRAY_BOUNDARY + control->number_up = 6U; + control->ppd_profile = 0U; + if (selector[3] & 1U) + { + control->copies = 0U; + control->output_order = 1U; + control->reserved = 0U; + } + else + { + control->copies = 1U; + control->output_order = 0U; + control->reserved = 1U; + } +#endif +} + +static size_t +cf_fuzz_text_order_expected(const uint8_t *selector, uint8_t *markers) +{ + size_t count = 0; + unsigned page; + +#ifdef CF_FUZZ_TEXTTOTEXT_PAGE_ARRAY_BOUNDARY + if (selector[3] & 1U) + { + for (page = CF_FUZZ_TEXT_ORDER_PAGES; page >= 1U; page --) + markers[count ++] = (uint8_t)('A' + page - 1U); + } + else + { + unsigned copy; + for (copy = 0; copy < 2U; copy ++) + for (page = 1U; page <= CF_FUZZ_TEXT_ORDER_PAGES; page ++) + markers[count ++] = (uint8_t)('A' + page - 1U); + } +#else + for (page = 1U; page <= CF_FUZZ_TEXT_ORDER_PAGES; page ++) + if (cf_fuzz_text_order_range_selected(selector[0], page) && + cf_fuzz_text_order_set_selected(selector[1], page)) + markers[count ++] = (uint8_t)('A' + page - 1U); +#endif + return count; +} + +static size_t +cf_fuzz_text_order_actual(const cf_fuzz_run_result_t *result, uint8_t *markers, + size_t capacity) +{ + size_t count = 0; + size_t index; + + for (index = 0; index < result->output_size; index ++) + if (result->output[index] >= 'A' && result->output[index] <= 'D') + { + if (count == capacity) + return capacity + 1U; + markers[count ++] = result->output[index]; + } + return count; +} + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + const size_t fixed_size = + CF_FUZZ_TEXT_ORDER_MAGIC_SIZE + CF_FUZZ_TEXT_ORDER_SELECTORS; + const uint8_t *selector; + const uint8_t *payload; + size_t payload_size; + size_t document_size = 0; + uint8_t expected[CF_FUZZ_TEXT_ORDER_PAGES * 2U]; + uint8_t actual[CF_FUZZ_TEXT_ORDER_PAGES * 2U]; + size_t expected_count; + size_t actual_count; + cf_fuzz_control_t control; + cf_fuzz_run_result_t result; + uint8_t *document; + int executed; + + if (!data || size < fixed_size + 1U || + size > fixed_size + CF_FUZZ_TEXT_ORDER_MAX_PAYLOAD || + memcmp(data, CF_FUZZ_TEXT_ORDER_MAGIC, + CF_FUZZ_TEXT_ORDER_MAGIC_SIZE) != 0) + return 0; + + selector = data + CF_FUZZ_TEXT_ORDER_MAGIC_SIZE; + payload = data + fixed_size; + payload_size = size - fixed_size; + document = cf_fuzz_text_order_document(payload, payload_size, &document_size); + if (!document) + return 0; + + cf_fuzz_text_order_control(selector, &control); + executed = + cf_fuzz_execute_direct(document, document_size, &control, 1, &result); + if (executed && result.captured && result.status == 0) + { + expected_count = cf_fuzz_text_order_expected(selector, expected); + actual_count = cf_fuzz_text_order_actual(&result, actual, sizeof(actual)); + if (actual_count != expected_count || + (actual_count && memcmp(actual, expected, actual_count) != 0)) + __builtin_trap(); + } + + cf_fuzz_free_run_result(&result); + free(document); + return 0; +} diff --git a/parser-fuzzers/harnesses/fuzz_text_to_text_state.c b/parser-fuzzers/harnesses/fuzz_text_to_text_state.c new file mode 100644 index 0000000..95e1397 --- /dev/null +++ b/parser-fuzzers/harnesses/fuzz_text_to_text_state.c @@ -0,0 +1,230 @@ +#include "direct_route.h" + +#include +#include +#include +#include + +#define CF_FUZZ_TEXT_TO_TEXT_MAGIC "TXT2TXT1" +#define CF_FUZZ_TEXT_TO_TEXT_MAGIC_SIZE 8U +#define CF_FUZZ_TEXT_TO_TEXT_SELECTORS 16U +#define CF_FUZZ_TEXT_TO_TEXT_MAX_PAYLOAD 4096U +#define CF_FUZZ_TEXT_TO_TEXT_CHUNK_BOUNDARY 2047U + +static int +cf_fuzz_text_to_text_append(uint8_t *document, size_t capacity, size_t *used, + const uint8_t *material, size_t material_size) +{ + if (material_size > capacity - *used) + return 0; + memcpy(document + *used, material, material_size); + *used += material_size; + return 1; +} + +static int +cf_fuzz_text_to_text_append_byte(uint8_t *document, size_t capacity, + size_t *used, uint8_t value) +{ + return cf_fuzz_text_to_text_append(document, capacity, used, &value, 1U); +} + +static void +cf_fuzz_text_to_text_control(const uint8_t *selector, + cf_fuzz_control_t *control) +{ + control->ppd_profile = selector[0]; + control->page_size = selector[1]; + control->color_model = selector[2]; + control->resolution = selector[3]; + control->sides = selector[4]; + control->orientation = selector[5]; + control->scaling = selector[6]; + control->copies = selector[7]; + control->number_up = selector[8]; + control->position = selector[9]; + control->quality = selector[10]; + control->output_order = selector[11]; + control->media_type = selector[12]; + control->mirror = selector[13]; + control->route_mode = selector[14]; + control->reserved = selector[15]; +} + +static uint8_t * +cf_fuzz_build_text_to_text_document( + const uint8_t selector[CF_FUZZ_TEXT_TO_TEXT_SELECTORS], + const uint8_t *payload, size_t payload_size, + const cf_fuzz_texttotext_state_t *state, size_t *document_size) +{ + static const uint8_t euro[] = {0xe2U, 0x82U, 0xacU}; + static const uint8_t smile[] = {0xf0U, 0x9fU, 0x98U, 0x80U}; + static const uint8_t latin_e_acute[] = {0xc3U, 0xa9U}; + static const uint8_t ascii_z[] = {'Z'}; + const unsigned stream_mode = (selector[15] >> 1U) % 4U; + const int word_wrap = strcmp(state->overlong, "word-wrap") == 0; + const unsigned word_limit = word_wrap ? 1U : + cf_fuzz_texttotext_min(state->text_width > 1U ? state->text_width - 1U + : 1U, + 7U); + const size_t capacity = payload_size * 5U + 2304U; + uint8_t *document = (uint8_t *)malloc(capacity); + const uint8_t *stream_character = ascii_z; + size_t stream_character_size = sizeof(ascii_z); + size_t used = 0; + unsigned word_bytes = 0; + size_t index; + + if (!document) + return NULL; + if (strcmp(state->encoding, "UTF-8") == 0) + { + stream_character = stream_mode == 2U ? smile : euro; + stream_character_size = stream_mode == 2U ? sizeof(smile) : sizeof(euro); + } + else if (strcmp(state->encoding, "ISO-8859-1") == 0) + { + stream_character = latin_e_acute; + stream_character_size = sizeof(latin_e_acute); + } + else if (strcmp(state->encoding, "CP1252") == 0) + { + stream_character = euro; + stream_character_size = sizeof(euro); + } + if (!cf_fuzz_text_to_text_append(document, capacity, &used, + (const uint8_t *)"A ", 2U)) + goto fail; + + if (stream_mode == 1U || stream_mode == 2U) + { + while (used < CF_FUZZ_TEXT_TO_TEXT_CHUNK_BOUNDARY) + { + uint8_t value = word_bytes >= word_limit ? (uint8_t)' ' : + (uint8_t)('a' + used % 26U); + if (!cf_fuzz_text_to_text_append_byte(document, capacity, &used, value)) + goto fail; + word_bytes = value == ' ' ? 0U : word_bytes + 1U; + } + if (!cf_fuzz_text_to_text_append(document, capacity, &used, + stream_character, + stream_character_size)) + goto fail; + word_bytes += stream_character_size; + } + + for (index = 0; index < payload_size; index ++) + { + const uint8_t value = payload[index]; + + if (value % 53U == 0U && state->pagination[0] == 't' && + used && document[used - 1U] != '\f') + { + if (!cf_fuzz_text_to_text_append_byte(document, capacity, &used, '\f') || + !cf_fuzz_text_to_text_append(document, capacity, &used, + (const uint8_t *)"P ", 2U)) + goto fail; + word_bytes = 0; + } + else if (value % 31U == 0U) + { + /* Tab expansion can move a partial word to a fresh line and enter the + * already-known first-word under-read. Keep tab and word-wrap as + * separate state blocks so this lane can continue past that root. */ + if (!cf_fuzz_text_to_text_append_byte(document, capacity, &used, + word_wrap ? ' ' : '\t')) + goto fail; + word_bytes = 0; + } + else if (value % 29U == 0U) + { + static const uint8_t crlf[] = {'\r', '\n'}; + const uint8_t newline = selector[13] % 3U == 0U ? '\n' : '\r'; + + if (selector[13] % 3U == 2U) + { + if (!cf_fuzz_text_to_text_append(document, capacity, &used, crlf, + sizeof(crlf))) + goto fail; + } + else if (!cf_fuzz_text_to_text_append_byte(document, capacity, &used, + newline)) + goto fail; + word_bytes = 0; + } + else + { + const int use_multibyte = + stream_mode == 3U && !word_wrap && stream_character_size > 1U && + (value & 7U) == 0U; + + if (word_bytes >= word_limit) + { + if (!cf_fuzz_text_to_text_append_byte(document, capacity, &used, ' ')) + goto fail; + word_bytes = 0; + } + if (use_multibyte) + { + if (!cf_fuzz_text_to_text_append(document, capacity, &used, + stream_character, + stream_character_size)) + goto fail; + word_bytes += stream_character_size; + } + else + { + const uint8_t glyph = (uint8_t)('!' + value % 94U); + if (!cf_fuzz_text_to_text_append_byte(document, capacity, &used, glyph)) + goto fail; + word_bytes ++; + } + } + } + + if (!used || (document[used - 1U] != '\n' && document[used - 1U] != '\r')) + if (!cf_fuzz_text_to_text_append_byte(document, capacity, &used, '\n')) + goto fail; + *document_size = used; + return document; + +fail: + free(document); + return NULL; +} + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + const size_t fixed_size = + CF_FUZZ_TEXT_TO_TEXT_MAGIC_SIZE + CF_FUZZ_TEXT_TO_TEXT_SELECTORS; + const uint8_t *selector; + const uint8_t *payload; + size_t payload_size; + cf_fuzz_control_t control; + cf_fuzz_texttotext_state_t state; + cf_fuzz_run_result_t result; + uint8_t *document; + size_t document_size = 0; + + if (!data || size < fixed_size + 1U || + size > fixed_size + CF_FUZZ_TEXT_TO_TEXT_MAX_PAYLOAD || + memcmp(data, CF_FUZZ_TEXT_TO_TEXT_MAGIC, + CF_FUZZ_TEXT_TO_TEXT_MAGIC_SIZE) != 0) + return 0; + + selector = data + CF_FUZZ_TEXT_TO_TEXT_MAGIC_SIZE; + payload = data + fixed_size; + payload_size = size - fixed_size; + cf_fuzz_text_to_text_control(selector, &control); + cf_fuzz_decode_texttotext_state(&control, &state); + document = cf_fuzz_build_text_to_text_document( + selector, payload, payload_size, &state, &document_size); + if (!document) + return 0; + + (void)cf_fuzz_execute_direct(document, document_size, &control, 1, &result); + cf_fuzz_free_run_result(&result); + free(document); + return 0; +} diff --git a/parser-fuzzers/harnesses/job.h b/parser-fuzzers/harnesses/job.h new file mode 100644 index 0000000..67017db --- /dev/null +++ b/parser-fuzzers/harnesses/job.h @@ -0,0 +1,90 @@ +#ifndef CUPSFILTERS_FUZZ_JOB_H +#define CUPSFILTERS_FUZZ_JOB_H + +#include "control.h" + +#include +#include +#include + +#define CF_FUZZ_JOB_LENGTH_FIELDS 4U +#define CF_FUZZ_JOB_HEADER_SIZE (CF_FUZZ_JOB_LENGTH_FIELDS * 4U) +#define CF_FUZZ_JOB_FIXED_SIZE (CF_FUZZ_JOB_HEADER_SIZE + CF_FUZZ_CONTROL_SIZE) +#define CF_FUZZ_JOB_PPD_PREFIX "*PPD-Adobe:" +#define CF_FUZZ_JOB_PPD_PREFIX_SIZE 11U + +typedef struct cf_fuzz_job_input_s { + cf_fuzz_control_t control; + const uint8_t *ppd; + size_t ppd_size; + const uint8_t *options; + size_t options_size; + const uint8_t *title; + size_t title_size; + const uint8_t *document; + size_t document_size; +} cf_fuzz_job_input_t; + +static inline uint32_t cf_fuzz_job_load_u32le(const uint8_t *data) { + return (uint32_t)data[0] | ((uint32_t)data[1] << 8) | + ((uint32_t)data[2] << 16) | ((uint32_t)data[3] << 24); +} + +static inline void cf_fuzz_job_store_u32le(uint8_t *data, uint32_t value) { + data[0] = (uint8_t)value; + data[1] = (uint8_t)(value >> 8); + data[2] = (uint8_t)(value >> 16); + data[3] = (uint8_t)(value >> 24); +} + +static inline int cf_fuzz_parse_job_input( + const uint8_t *data, size_t size, size_t max_ppd, size_t max_options, + size_t max_title, size_t max_document, cf_fuzz_job_input_t *input) { + size_t lengths[CF_FUZZ_JOB_LENGTH_FIELDS]; + size_t offset = CF_FUZZ_JOB_FIXED_SIZE; + + if (!data || !input || size < CF_FUZZ_JOB_FIXED_SIZE) { + return 0; + } + for (size_t index = 0; index < CF_FUZZ_JOB_LENGTH_FIELDS; index++) { + lengths[index] = (size_t)cf_fuzz_job_load_u32le(data + index * 4U); + } + if (lengths[0] > max_ppd || lengths[1] > max_options || + lengths[2] > max_title || !lengths[3] || lengths[3] > max_document || + lengths[0] > size - offset) { + return 0; + } + input->ppd = data + offset; + input->ppd_size = lengths[0]; + offset += lengths[0]; + if (lengths[1] > size - offset) { + return 0; + } + input->options = data + offset; + input->options_size = lengths[1]; + offset += lengths[1]; + if (lengths[2] > size - offset) { + return 0; + } + input->title = data + offset; + input->title_size = lengths[2]; + offset += lengths[2]; + if (lengths[3] != size - offset) { + return 0; + } + input->document = data + offset; + input->document_size = lengths[3]; + + if ((input->ppd_size && + (input->ppd_size < CF_FUZZ_JOB_PPD_PREFIX_SIZE || + memcmp(input->ppd, CF_FUZZ_JOB_PPD_PREFIX, + CF_FUZZ_JOB_PPD_PREFIX_SIZE) != 0)) || + (input->options_size && memchr(input->options, 0, input->options_size))) { + return 0; + } + memcpy(&input->control, data + CF_FUZZ_JOB_HEADER_SIZE, CF_FUZZ_CONTROL_SIZE); + cf_fuzz_apply_control_policy(&input->control); + return 1; +} + +#endif diff --git a/parser-fuzzers/harnesses/lsan_coverage_stubs.c b/parser-fuzzers/harnesses/lsan_coverage_stubs.c new file mode 100644 index 0000000..5bee9af --- /dev/null +++ b/parser-fuzzers/harnesses/lsan_coverage_stubs.c @@ -0,0 +1,3 @@ +/* OSS-Fuzz coverage builds do not link LeakSanitizer. */ +void __lsan_disable(void) {} +void __lsan_enable(void) {} diff --git a/parser-fuzzers/harnesses/profiles.h b/parser-fuzzers/harnesses/profiles.h new file mode 100644 index 0000000..a88c41d --- /dev/null +++ b/parser-fuzzers/harnesses/profiles.h @@ -0,0 +1,405 @@ +#ifndef CUPSFILTERS_FUZZ_PROFILES_H +#define CUPSFILTERS_FUZZ_PROFILES_H + +#include "control.h" + +#include +#include + +#ifdef CF_FUZZ_TEXTTOTEXT_STATE_OPTIONS +typedef struct cf_fuzz_texttotext_state_s { + unsigned width; + unsigned height; + unsigned left; + unsigned right; + unsigned top; + unsigned bottom; + unsigned text_width; + unsigned text_height; + unsigned tab_width; + const char *encoding; + const char *overlong; + const char *pagination; + const char *send_ff; + const char *newline; + const char *page_ranges; + const char *page_set; + const char *output_order; + const char *collate; +} cf_fuzz_texttotext_state_t; + +static inline unsigned cf_fuzz_texttotext_min(unsigned first, + unsigned second) { + return first < second ? first : second; +} + +static inline void cf_fuzz_decode_texttotext_state( + const cf_fuzz_control_t *control, cf_fuzz_texttotext_state_t *state) { + static const char *const encodings[] = { + "ASCII", "UTF-8", "ISO-8859-1", "CP1252"}; + static const char *const overlong[] = { + "truncate", "word-wrap", "wrap-at-width"}; + static const char *const newlines[] = {"lf", "cr", "crlf"}; + static const char *const ranges[] = { + "1-1", "1-2", "2-3", "1-4", "2-4", "1,3", "1-99"}; + static const char *const page_sets[] = {"all", "odd", "even"}; + unsigned horizontal_room; + unsigned vertical_room; + + state->width = 8U + control->page_size % 41U; + state->height = 4U + control->sides % 17U; + + horizontal_room = state->width - 4U; + state->left = control->color_model % + (cf_fuzz_texttotext_min(horizontal_room, 6U) + 1U); + horizontal_room -= state->left; + state->right = control->resolution % + (cf_fuzz_texttotext_min(horizontal_room, 6U) + 1U); + + vertical_room = state->height - 2U; + state->top = control->orientation % + (cf_fuzz_texttotext_min(vertical_room, 3U) + 1U); + vertical_room -= state->top; + state->bottom = control->scaling % + (cf_fuzz_texttotext_min(vertical_room, 3U) + 1U); + + state->text_width = state->width - state->left - state->right; + state->text_height = state->height - state->top - state->bottom; + state->tab_width = 1U + control->position % + cf_fuzz_texttotext_min(state->text_width, 8U); + state->encoding = encodings[control->media_type % 4U]; + state->overlong = overlong[control->quality % 3U]; + state->pagination = control->route_mode & 1U ? "true" : "false"; + state->send_ff = control->route_mode & 2U ? "true" : "false"; + state->newline = newlines[control->mirror % 3U]; + state->page_ranges = ranges[control->number_up % 7U]; + state->page_set = page_sets[control->ppd_profile % 3U]; + state->output_order = control->output_order & 1U ? "reverse" : "normal"; + state->collate = control->reserved & 1U ? "true" : "false"; +} +#endif + +#ifdef CF_FUZZ_PS_DSC_STATE_OPTIONS +typedef struct cf_fuzz_ps_dsc_state_s { + unsigned page_count; + unsigned sheet_count; + unsigned number_up; + unsigned copies; + unsigned section_mode; + unsigned binary_mode; + unsigned trailer_mode; + const char *number_up_layout; + const char *page_set; + const char *output_order; + const char *collate; + const char *sides; + const char *fit_to_page; + const char *mirror; + const char *page_border; + unsigned orientation_requested; + char page_ranges[32]; +} cf_fuzz_ps_dsc_state_t; + +static inline void cf_fuzz_decode_ps_dsc_state( + const cf_fuzz_control_t *control, cf_fuzz_ps_dsc_state_t *state) { + static const unsigned number_up[] = {1U, 2U, 4U, 6U, 9U, 16U}; + static const char *const layouts[] = { + "lrtb", "lrbt", "rltb", "rlbt", "tblr", "tbrl", "btlr", "btrl"}; + static const char *const page_sets[] = {"all", "odd", "even"}; + static const char *const sides[] = { + "one-sided", "two-sided-long-edge", "two-sided-short-edge"}; + static const char *const borders[] = { + "none", "single", "single-thick", "double", "double-thick"}; + const unsigned range_mode = control->quality % 6U; + + state->page_count = 1U + control->page_size % 8U; + state->number_up = number_up[control->number_up % 6U]; + state->sheet_count = + (state->page_count + state->number_up - 1U) / state->number_up; + state->copies = 1U + control->copies % 4U; + state->section_mode = control->ppd_profile; + state->binary_mode = control->color_model % 4U; + state->trailer_mode = control->resolution % 4U; + state->number_up_layout = layouts[control->position % 8U]; + state->page_set = page_sets[control->media_type % 3U]; + state->output_order = control->output_order & 1U ? "Reverse" : "Normal"; + state->collate = control->reserved & 1U ? "true" : "false"; + state->sides = sides[control->sides % 3U]; + state->fit_to_page = control->scaling & 1U ? "true" : "false"; + state->mirror = control->mirror & 1U ? "true" : "false"; + state->page_border = borders[control->route_mode % 5U]; + state->orientation_requested = 3U + control->orientation % 4U; + + switch (range_mode) { + case 0U: + snprintf(state->page_ranges, sizeof(state->page_ranges), "1-%u", + state->sheet_count); + break; + case 1U: + snprintf(state->page_ranges, sizeof(state->page_ranges), "1-1"); + break; + case 2U: + snprintf(state->page_ranges, sizeof(state->page_ranges), "%u-%u", + state->sheet_count, state->sheet_count); + break; + case 3U: + snprintf(state->page_ranges, sizeof(state->page_ranges), "2-%u", + state->sheet_count > 1U ? state->sheet_count : 2U); + break; + case 4U: + snprintf(state->page_ranges, sizeof(state->page_ranges), "1,%u", + state->sheet_count > 1U ? state->sheet_count : 1U); + break; + default: + snprintf(state->page_ranges, sizeof(state->page_ranges), "1-99"); + break; + } +} +#endif + +static inline int cf_fuzz_build_options(char *buffer, size_t buffer_size, + const cf_fuzz_control_t *control) { + static const char *const page_sizes[] = {"A4", "Letter"}; + static const char *const color_models[] = {"Gray", "RGB", "CMYK"}; + static const char *const resolutions[] = {"300dpi", "600dpi"}; + static const char *const sides[] = { + "one-sided", "two-sided-long-edge", "two-sided-short-edge"}; + static const char *const scaling_modes[] = { + "auto-fit", "fit", "fill", "none"}; + static const char *const positions[] = { + "center", "top-left", "top-right", "bottom-left", "bottom-right"}; + static const char *const output_orders[] = {"normal", "reverse"}; + static const char *const media_types[] = {"Plain", "Glossy", "Transparency"}; +#ifdef CF_FUZZ_OPTIONS_PDF_NUP_BOUNDARY + static const unsigned number_up[] = {17, 18, 19, 20, 32, 255}; +#else + static const unsigned number_up[] = {1, 2, 4, 6, 9, 16}; +#endif + unsigned copies = 1U + control->copies % 4U; + unsigned scaling = 25U + control->scaling % 176U; + unsigned ppi = 72U + ((unsigned)control->quality * 1128U) / 255U; + int length; + int extra; + +#ifdef CF_FUZZ_TEXTTOTEXT_STATE_OPTIONS + { + cf_fuzz_texttotext_state_t state; + + cf_fuzz_decode_texttotext_state(control, &state); + length = snprintf( + buffer, buffer_size, + "PageWidth=%u PageHeight=%u PageLeft=%u PageRight=%u " + "PageTop=%u PageBottom=%u PrinterEncoding=%s " + "OverLongLines=%s TabWidth=%u Pagination=%s SendFF=%s " + "NewlineCharacters=%s page-ranges=%s page-set=%s " + "OutputOrder=%s Collate=%s", + state.width, state.height, state.left, state.right, state.top, + state.bottom, state.encoding, state.overlong, state.tab_width, + state.pagination, state.send_ff, state.newline, state.page_ranges, + state.page_set, state.output_order, state.collate); + return length < 0 || (size_t)length >= buffer_size ? -1 : 0; + } +#endif + +#ifdef CF_FUZZ_PS_DSC_STATE_OPTIONS + { + cf_fuzz_ps_dsc_state_t state; + + cf_fuzz_decode_ps_dsc_state(control, &state); + length = snprintf( + buffer, buffer_size, + "PageSize=A4 sides=%s orientation-requested=%u " + "number-up=%u number-up-layout=%s page-ranges=%s page-set=%s " + "OutputOrder=%s Collate=%s copies=%u emit-jcl=false " + "fit-to-page=%s mirror=%s page-border=%s", + state.sides, state.orientation_requested, state.number_up, + state.number_up_layout, state.page_ranges, state.page_set, + state.output_order, state.collate, state.copies, state.fit_to_page, + state.mirror, state.page_border); + return length < 0 || (size_t)length >= buffer_size ? -1 : 0; + } +#endif + +#if (defined(CF_FUZZ_IMAGE_SCALING_PRINT) + \ + defined(CF_FUZZ_IMAGE_SCALING_PPI) + \ + defined(CF_FUZZ_IMAGE_SCALING_PERCENT) + \ + defined(CF_FUZZ_IMAGE_SCALING_MULTIPAGE) + \ + defined(CF_FUZZ_IMAGE_SCALING_FITPLOT) + \ + defined(CF_FUZZ_IMAGE_SCALING_NATURAL) + \ + defined(CF_FUZZ_IMAGE_SCALING_FILL) + \ + defined(CF_FUZZ_IMAGE_SCALING_CROP) + \ + defined(CF_FUZZ_IMAGE_SCALING_DEFAULT)) > 1 +#error "Select at most one image scaling option contract" +#endif + + length = snprintf( + buffer, buffer_size, + "PageSize=%s ColorModel=%s Resolution=%s sides=%s " + "orientation-requested=%u " + "number-up=%u position=%s copies=%u output-order=%s MediaType=%s " + "mirror=%s emit-jcl=false", + page_sizes[control->page_size % 2U], + color_models[control->color_model % 3U], + resolutions[control->resolution % 2U], sides[control->sides % 3U], + 3U + control->orientation % 4U, + number_up[control->number_up % 6U], + positions[control->position % 5U], copies, + output_orders[control->output_order % 2U], + media_types[control->media_type % 3U], + control->mirror & 1U ? "true" : "false"); + if (length < 0 || (size_t)length >= buffer_size) { + return -1; + } +#if defined(CF_FUZZ_IMAGE_SCALING_PRINT) + extra = snprintf(buffer + length, buffer_size - (size_t)length, + " print-scaling=%s", + scaling_modes[control->scaling % 4U]); +#elif defined(CF_FUZZ_IMAGE_SCALING_PPI) + extra = snprintf(buffer + length, buffer_size - (size_t)length, + " ppi=%u", ppi); +#elif defined(CF_FUZZ_IMAGE_SCALING_PERCENT) + extra = snprintf(buffer + length, buffer_size - (size_t)length, + " scaling=%u", 25U + control->scaling % 76U); +#elif defined(CF_FUZZ_IMAGE_SCALING_MULTIPAGE) + extra = snprintf(buffer + length, buffer_size - (size_t)length, + " scaling=%u", 101U + control->scaling % 100U); +#elif defined(CF_FUZZ_IMAGE_SCALING_FITPLOT) + extra = snprintf(buffer + length, buffer_size - (size_t)length, + " fitplot=%s", control->scaling & 1U ? "true" : "false"); +#elif defined(CF_FUZZ_IMAGE_SCALING_NATURAL) + { + static const unsigned natural_scaling[] = {0U, 50U, 100U, 200U}; + extra = snprintf(buffer + length, buffer_size - (size_t)length, + " natural-scaling=%u", + natural_scaling[control->scaling % 4U]); + } +#elif defined(CF_FUZZ_IMAGE_SCALING_FILL) + extra = snprintf(buffer + length, buffer_size - (size_t)length, + " fill=%s", control->scaling & 1U ? "true" : "false"); +#elif defined(CF_FUZZ_IMAGE_SCALING_CROP) + extra = snprintf(buffer + length, buffer_size - (size_t)length, + " crop-to-fit=%s", + control->scaling & 1U ? "true" : "false"); +#elif defined(CF_FUZZ_IMAGE_SCALING_DEFAULT) + extra = 0; +#else + extra = snprintf(buffer + length, buffer_size - (size_t)length, + " print-scaling=%s scaling=%u ppi=%u", + scaling_modes[control->scaling % 4U], scaling, ppi); +#endif + if (extra < 0 || (size_t)extra >= buffer_size - (size_t)length) { + return -1; + } + length += extra; +#ifdef CF_FUZZ_OPTIONS_CM_CALIBRATION + extra = snprintf(buffer + length, buffer_size - (size_t)length, + " cm-calibration=true"); + if (extra < 0 || (size_t)extra >= buffer_size - (size_t)length) { + return -1; + } + length += extra; +#endif +#ifdef CF_FUZZ_OPTIONS_PDF_DEPTH + { + static const char *const page_sets[] = {"all", "odd", "even"}; + static const char *const page_ranges[] = { + "1-1", "1-2", "2-4", "1-16"}; + int booklet = control->route_mode % 2U; + const char *page_set = booklet ? "all" : page_sets[control->reserved % 3U]; + const char *page_range = + booklet ? "1-16" : page_ranges[(control->reserved / 3U) % 4U]; + int extra = snprintf(buffer + length, buffer_size - (size_t)length, + " page-set=%s page-ranges=%s%s", + page_set, page_range, + booklet ? " imposition-template=booklet" : ""); + if (extra < 0 || (size_t)extra >= buffer_size - (size_t)length) { + return -1; + } + } +#endif +#ifdef CF_FUZZ_OPTIONS_PDF_BOOKLET_EMPTY_BOUNDARY + { + int extra = snprintf(buffer + length, buffer_size - (size_t)length, + " page-set=even page-ranges=2-4 " + "imposition-template=booklet"); + if (extra < 0 || (size_t)extra >= buffer_size - (size_t)length) { + return -1; + } + } +#endif +#ifdef CF_FUZZ_TEXT_LAYOUT_OPTIONS + { + static const unsigned columns[] = {1U, 2U, 3U, 4U}; + static const unsigned cpi[] = {6U, 8U, 10U, 12U, 15U}; + static const unsigned lpi[] = {4U, 6U, 8U, 10U}; + int text_extra = snprintf( + buffer + length, buffer_size - (size_t)length, + " columns=%u cpi=%u lpi=%u wrap=%s prettyprint=%s", + columns[control->number_up % + (sizeof(columns) / sizeof(columns[0]))], + cpi[control->quality % (sizeof(cpi) / sizeof(cpi[0]))], + lpi[control->reserved % (sizeof(lpi) / sizeof(lpi[0]))], + control->scaling & 1U ? "true" : "false", + control->route_mode & 1U ? "true" : "false"); + if (text_extra < 0 || + (size_t)text_extra >= buffer_size - (size_t)length) { + return -1; + } + length += text_extra; + } +#endif + return 0; +} + +static inline int cf_fuzz_write_ppd(FILE *file, + const cf_fuzz_control_t *control, + const char *filter_name) { + static const int model_numbers[] = {0, 2, 0x1130, 0x40}; + unsigned profile = control->ppd_profile % 4U; + + return fprintf( + file, + "*PPD-Adobe: \"4.3\"\n" + "*FormatVersion: \"4.3\"\n" + "*FileVersion: \"2.0\"\n" + "*LanguageVersion: English\n" + "*LanguageEncoding: ISOLatin1\n" + "*Manufacturer: \"OpenPrinting\"\n" + "*ModelName: \"cups-filters fuzz\"\n" + "*ShortNickName: \"cups-filters fuzz\"\n" + "*NickName: \"cups-filters fuzz\"\n" + "*PCFileName: \"FUZZ.PPD\"\n" + "*Product: \"(cups-filters fuzz)\"\n" + "*PSVersion: \"(3010) 0\"\n" + "*cupsVersion: 2.0\n" + "*cupsModelNumber: %d\n" + "*cupsManualCopies: False\n" + "*cupsFilter: \"application/octet-stream 0 %s\"\n" + "*OpenUI *PageSize: PickOne\n" + "*DefaultPageSize: A4\n" + "*PageSize A4/A4: \"<>setpagedevice\"\n" + "*PageSize Letter/Letter: \"<>setpagedevice\"\n" + "*CloseUI: *PageSize\n" + "*DefaultImageableArea: A4\n" + "*ImageableArea A4: \"12 12 583 830\"\n" + "*ImageableArea Letter: \"18 36 594 756\"\n" + "*DefaultPaperDimension: A4\n" + "*PaperDimension A4: \"595 842\"\n" + "*PaperDimension Letter: \"612 792\"\n" + "*OpenUI *ColorModel: PickOne\n" + "*DefaultColorModel: Gray\n" + "*ColorModel Gray/Gray: \"<>setpagedevice\"\n" + "*ColorModel RGB/RGB: \"<>setpagedevice\"\n" + "*ColorModel CMYK/CMYK: \"<>setpagedevice\"\n" + "*CloseUI: *ColorModel\n" + "*OpenUI *Resolution: PickOne\n" + "*DefaultResolution: 300dpi\n" + "*Resolution 300dpi/300 dpi: \"<>setpagedevice\"\n" + "*Resolution 600dpi/600 dpi: \"<>setpagedevice\"\n" + "*CloseUI: *Resolution\n", + model_numbers[profile], filter_name) < 0 + ? -1 + : 0; +} + +#endif diff --git a/parser-fuzzers/harnesses/runtime.h b/parser-fuzzers/harnesses/runtime.h new file mode 100644 index 0000000..dd39193 --- /dev/null +++ b/parser-fuzzers/harnesses/runtime.h @@ -0,0 +1,60 @@ +#ifndef CUPSFILTERS_FUZZ_RUNTIME_H +#define CUPSFILTERS_FUZZ_RUNTIME_H + +#include +#include +#include +#include +#include +#include +#include + +static char cf_fuzz_data_directory[PATH_MAX]; + +static inline void cf_fuzz_init_runtime(void) { + char executable[PATH_MAX]; + char fontconfig_file[PATH_MAX]; + char *slash; + ssize_t length; + + if (cf_fuzz_data_directory[0]) { + return; + } + length = readlink("/proc/self/exe", executable, sizeof(executable) - 1U); + if (length <= 0 || (size_t)length >= sizeof(executable) - 1U) { + snprintf(cf_fuzz_data_directory, sizeof(cf_fuzz_data_directory), "."); + return; + } + executable[length] = '\0'; + slash = strrchr(executable, '/'); + if (!slash) { + snprintf(cf_fuzz_data_directory, sizeof(cf_fuzz_data_directory), "."); + return; + } + *slash = '\0'; + snprintf(cf_fuzz_data_directory, sizeof(cf_fuzz_data_directory), + "%s/cups-data", executable); + snprintf(fontconfig_file, sizeof(fontconfig_file), "%s/fonts.conf", + executable); + (void)setenv("FONTCONFIG_FILE", fontconfig_file, 1); +} + +static inline const char *cf_fuzz_data_dir(void) { + cf_fuzz_init_runtime(); + return cf_fuzz_data_directory; +} + +static inline int cf_fuzz_write_all(int fd, const uint8_t *data, size_t size) { + size_t offset = 0; + + while (offset < size) { + ssize_t written = write(fd, data + offset, size - offset); + if (written <= 0) { + return -1; + } + offset += (size_t)written; + } + return 0; +} + +#endif diff --git a/parser-fuzzers/harnesses/validity.h b/parser-fuzzers/harnesses/validity.h new file mode 100644 index 0000000..78c7dd8 --- /dev/null +++ b/parser-fuzzers/harnesses/validity.h @@ -0,0 +1,446 @@ +#ifndef CUPSFILTERS_FUZZ_VALIDITY_H +#define CUPSFILTERS_FUZZ_VALIDITY_H + +#include +#include +#include +#include +#include +#include +#include + +#define CF_FUZZ_RASTER_REQUIRE_PDF_COLORSPACE 1U +#define CF_FUZZ_RASTER_REQUIRE_ESCPX_WEAVE 2U +#define CF_FUZZ_RASTER_REQUIRE_COMPRESSION_1 4U +#define CF_FUZZ_RASTER_REQUIRE_COMPRESSION_2 8U +#define CF_FUZZ_RASTER_REQUIRE_COMPRESSION_3 16U +#define CF_FUZZ_RASTER_REQUIRE_COMPRESSION_10 32U +#define CF_FUZZ_RASTER_REJECT_COMPRESSION_3 64U +#define CF_FUZZ_RASTER_REQUIRE_MULTIROW 128U +#define CF_FUZZ_RASTER_REQUIRE_MODE10_RGB 256U + +#define CF_FUZZ_PDF_REJECT_INTERACTIVE 1U +#define CF_FUZZ_PDF_REQUIRE_INTERACTIVE 2U + +static inline uint32_t cf_fuzz_be32(const uint8_t *data) { + return ((uint32_t)data[0] << 24) | ((uint32_t)data[1] << 16) | + ((uint32_t)data[2] << 8) | data[3]; +} + +static inline uint32_t cf_fuzz_le32(const uint8_t *data) { + return (uint32_t)data[0] | ((uint32_t)data[1] << 8) | + ((uint32_t)data[2] << 16) | ((uint32_t)data[3] << 24); +} + +/* Exact raw pages keep output-state targets past the trailing-data frontier. */ +static inline int cf_fuzz_validate_simple_raster(const uint8_t *data, + size_t size) { + static const size_t header_size = 1796U; + size_t offset = 4U; + unsigned pages = 0; + + if (!data || size < 4U + header_size || memcmp(data, "3SaR", 4U) != 0) { + return 0; + } + while (offset < size) { + uint32_t height; + uint32_t bytes_per_line; + uint64_t page_bytes; + + if (size - offset < header_size || ++pages > 16U) { + return 0; + } + height = cf_fuzz_le32(data + offset + 376U); + bytes_per_line = cf_fuzz_le32(data + offset + 392U); + if (!height || !bytes_per_line || + cf_fuzz_le32(data + offset + 404U) != 0U) { + return 0; + } + page_bytes = (uint64_t)height * bytes_per_line; + if (page_bytes > size - offset - header_size) { + return 0; + } + offset += header_size + (size_t)page_bytes; + } + return pages > 0U && offset == size; +} + +/* imagetopdf/imagetops carry incomplete ASCII85 groups between rows but only + * reserve three padding bytes. State lanes use aligned rows to continue past + * that known boundary; faithful implementation lanes remain unrestricted. */ +static inline int cf_fuzz_png_width_aligned(const uint8_t *data, size_t size, + uint32_t alignment) { + static const uint8_t signature[] = {0x89, 0x50, 0x4e, 0x47, + 0x0d, 0x0a, 0x1a, 0x0a}; + + return data && alignment && size >= 24U && + memcmp(data, signature, sizeof(signature)) == 0 && + cf_fuzz_be32(data + 8U) == 13U && + memcmp(data + 12U, "IHDR", 4U) == 0 && + cf_fuzz_be32(data + 16U) != 0U && + cf_fuzz_be32(data + 16U) % alignment == 0U; +} + +static inline int cf_fuzz_validate_command_safe(const uint8_t *data, + size_t size) { + static const uint8_t header[] = "#CUPS-COMMAND"; + size_t line_length = 0; + + if (size <= sizeof(header) - 1U || + memcmp(data, header, sizeof(header) - 1U) != 0 || + (data[sizeof(header) - 1U] != '\n' && + data[sizeof(header) - 1U] != '\r')) { + return 0; + } + for (size_t offset = 0; offset < size; offset++) { + uint8_t value = data[offset]; + if (!value || (value != '\n' && value != '\r' && value != '\t' && + (value < 0x20U || value > 0x7eU))) { + return 0; + } + if (value == '\n') { + line_length = 0; + } else if (++line_length >= 1024U) { + return 0; + } + } + return 1; +} + +static inline int cf_fuzz_command_has_leading_nul_line(const uint8_t *data, + size_t size) { + if (!size) { + return 0; + } + if (!data[0]) { + return 1; + } + for (size_t offset = 1; offset < size; offset++) { + if (!data[offset] && data[offset - 1U] == '\n') { + return 1; + } + } + return 0; +} + +static inline int cf_fuzz_pdf_hex(uint8_t value) { + if (value >= '0' && value <= '9') { + return value - '0'; + } + if (value >= 'A' && value <= 'F') { + return value - 'A' + 10; + } + if (value >= 'a' && value <= 'f') { + return value - 'a' + 10; + } + return -1; +} + +static inline int cf_fuzz_pdf_name_delimiter(uint8_t value) { + return value <= 0x20U || value == '(' || value == ')' || value == '<' || + value == '>' || value == '[' || value == ']' || value == '{' || + value == '}' || value == '/' || value == '%'; +} + +/* Match PDF names while decoding #xx escapes. Object streams are excluded + * from the non-interactive depth lane so hidden dictionaries cannot recreate + * the known annotation root behind this lightweight preflight. */ +static inline int cf_fuzz_pdf_has_name(const uint8_t *data, size_t size, + const char *wanted) { + size_t wanted_size = strlen(wanted); + + for (size_t offset = 0; offset < size; offset++) { + size_t input = offset + 1U; + size_t output = 0; + int matched = 1; + if (data[offset] != '/') { + continue; + } + while (input < size && !cf_fuzz_pdf_name_delimiter(data[input])) { + uint8_t value = data[input++]; + if (value == '#' && input + 1U < size) { + int high = cf_fuzz_pdf_hex(data[input]); + int low = cf_fuzz_pdf_hex(data[input + 1U]); + if (high >= 0 && low >= 0) { + value = (uint8_t)((high << 4) | low); + input += 2U; + } + } + if (output >= wanted_size || value != (uint8_t)wanted[output]) { + matched = 0; + break; + } + output++; + } + if (matched && output == wanted_size && + (input == size || cf_fuzz_pdf_name_delimiter(data[input]))) { + return 1; + } + } + return 0; +} + +static inline int cf_fuzz_validate_pdf_policy(const uint8_t *data, size_t size, + unsigned flags) { + int interactive; + + if (size < 8U || memcmp(data, "%PDF-", 5U) != 0) { + return 0; + } + interactive = cf_fuzz_pdf_has_name(data, size, "AcroForm") || + cf_fuzz_pdf_has_name(data, size, "Annots") || + cf_fuzz_pdf_has_name(data, size, "Annot") || + cf_fuzz_pdf_has_name(data, size, "NeedAppearances"); + if ((flags & CF_FUZZ_PDF_REQUIRE_INTERACTIVE) && !interactive) { + return 0; + } + if ((flags & CF_FUZZ_PDF_REJECT_INTERACTIVE) && + (interactive || cf_fuzz_pdf_has_name(data, size, "ObjStm"))) { + return 0; + } + return 1; +} + +/* Validate a complete, bounded PNG before an integration route consumes it. */ +static inline int cf_fuzz_validate_png(const uint8_t *data, size_t size) { + static const uint8_t signature[] = {0x89, 0x50, 0x4e, 0x47, + 0x0d, 0x0a, 0x1a, 0x0a}; + uint8_t *compressed = NULL; + uint8_t *decoded = NULL; + size_t offset = sizeof(signature); + size_t compressed_size = 0; + size_t compressed_capacity = 0; + uint32_t width = 0; + uint32_t height = 0; + unsigned channels = 0; + int saw_header = 0; + int saw_data = 0; + int saw_end = 0; + int valid = 0; + + if (size < sizeof(signature) + 12U || + memcmp(data, signature, sizeof(signature)) != 0) { + return 0; + } + while (offset + 12U <= size && !saw_end) { + uint32_t chunk_size = cf_fuzz_be32(data + offset); + const uint8_t *kind = data + offset + 4U; + const uint8_t *chunk = data + offset + 8U; + uint32_t expected_crc; + uint32_t actual_crc; + + if ((size_t)chunk_size > size - offset - 12U) { + goto done; + } + expected_crc = cf_fuzz_be32(chunk + chunk_size); + actual_crc = (uint32_t)crc32(crc32(0L, Z_NULL, 0), kind, 4U); + actual_crc = (uint32_t)crc32(actual_crc, chunk, chunk_size); + if (actual_crc != expected_crc) { + goto done; + } + if (!saw_header) { + unsigned color_type; + if (memcmp(kind, "IHDR", 4U) != 0 || chunk_size != 13U) { + goto done; + } + width = cf_fuzz_be32(chunk); + height = cf_fuzz_be32(chunk + 4U); + color_type = chunk[9]; + if (!width || !height || width > 4096U || height > 4096U || + chunk[8] != 8U || + (color_type != 0U && color_type != 2U && color_type != 6U) || + chunk[10] != 0U || chunk[11] != 0U || chunk[12] != 0U) { + goto done; + } + channels = color_type == 0U ? 1U : (color_type == 2U ? 3U : 4U); + saw_header = 1; + } else if (memcmp(kind, "IDAT", 4U) == 0) { + size_t needed; + uint8_t *replacement; + if (chunk_size > 8U * 1024U * 1024U || + compressed_size > 8U * 1024U * 1024U - chunk_size) { + goto done; + } + needed = compressed_size + chunk_size; + if (needed > compressed_capacity) { + size_t capacity = compressed_capacity ? compressed_capacity : 1024U; + while (capacity < needed) { + capacity *= 2U; + } + replacement = (uint8_t *)realloc(compressed, capacity); + if (!replacement) { + goto done; + } + compressed = replacement; + compressed_capacity = capacity; + } + memcpy(compressed + compressed_size, chunk, chunk_size); + compressed_size = needed; + saw_data = 1; + } else if (memcmp(kind, "IEND", 4U) == 0) { + if (!saw_data || chunk_size != 0U || offset + 12U != size) { + goto done; + } + saw_end = 1; + } else { + goto done; + } + offset += 12U + chunk_size; + } + + if (saw_header && saw_data && saw_end) { + uint64_t decoded_size = + (uint64_t)height * (1U + (uint64_t)width * channels); + uLongf output_size; + if (!decoded_size || decoded_size > 8U * 1024U * 1024U) { + goto done; + } + decoded = (uint8_t *)malloc((size_t)decoded_size); + if (!decoded) { + goto done; + } + output_size = (uLongf)decoded_size; + valid = uncompress(decoded, &output_size, compressed, + (uLong)compressed_size) == Z_OK && + output_size == decoded_size; + } + +done: + free(decoded); + free(compressed); + return valid; +} + +/* The parser itself performs validation; only complete bounded pages proceed. */ +static inline int cf_fuzz_pdf_colorspace(unsigned color_space) { + switch (color_space) { + case CUPS_CSPACE_K: + case CUPS_CSPACE_SW: + case CUPS_CSPACE_RGB: + case CUPS_CSPACE_SRGB: + case CUPS_CSPACE_ADOBERGB: + case CUPS_CSPACE_CMYK: + case CUPS_CSPACE_DEVICE1: + case CUPS_CSPACE_DEVICE2: + case CUPS_CSPACE_DEVICE3: + case CUPS_CSPACE_DEVICE4: + case CUPS_CSPACE_DEVICE5: + case CUPS_CSPACE_DEVICE6: + case CUPS_CSPACE_DEVICE7: + case CUPS_CSPACE_DEVICE8: + case CUPS_CSPACE_DEVICE9: + case CUPS_CSPACE_DEVICEA: + case CUPS_CSPACE_DEVICEB: + case CUPS_CSPACE_DEVICEC: + case CUPS_CSPACE_DEVICED: + case CUPS_CSPACE_DEVICEE: + case CUPS_CSPACE_DEVICEF: + return 1; + default: + return 0; + } +} + +static inline int cf_fuzz_validate_raster_fd(int fd, unsigned flags) { + cups_page_header2_t header; + cups_raster_t *raster = NULL; + uint8_t *row = NULL; + uint64_t decoded = 0; + unsigned pages = 0; + int valid = 0; + + if (lseek(fd, 0, SEEK_SET) < 0) { + return 0; + } + raster = cupsRasterOpen(fd, CUPS_RASTER_READ); + if (!raster) { + goto done; + } + while (cupsRasterReadHeader2(raster, &header)) { + uint64_t minimum_bytes; + uint64_t page_size; + if (++pages > 16U || !header.cupsWidth || !header.cupsHeight || + header.cupsWidth > 4096U || header.cupsHeight > 4096U || + !header.cupsBytesPerLine || header.cupsBytesPerLine > 16384U || + !header.cupsBitsPerColor || header.cupsBitsPerColor > 16U || + !header.cupsBitsPerPixel || header.cupsBitsPerPixel > 64U || + !header.cupsNumColors || header.cupsNumColors > 16U || + !header.HWResolution[0] || !header.HWResolution[1] || + header.HWResolution[0] > 9600U || header.HWResolution[1] > 9600U) { + goto done; + } + minimum_bytes = ((uint64_t)header.cupsWidth * header.cupsBitsPerPixel + + 7U) / + 8U; + if (header.cupsBytesPerLine < minimum_bytes || + ((flags & CF_FUZZ_RASTER_REQUIRE_PDF_COLORSPACE) && + !cf_fuzz_pdf_colorspace(header.cupsColorSpace))) { + goto done; + } + if (((flags & CF_FUZZ_RASTER_REQUIRE_COMPRESSION_1) && + header.cupsCompression != 1U) || + ((flags & CF_FUZZ_RASTER_REQUIRE_COMPRESSION_2) && + header.cupsCompression != 2U) || + ((flags & CF_FUZZ_RASTER_REQUIRE_COMPRESSION_3) && + header.cupsCompression != 3U) || + ((flags & CF_FUZZ_RASTER_REQUIRE_COMPRESSION_10) && + header.cupsCompression != 10U) || + ((flags & CF_FUZZ_RASTER_REJECT_COMPRESSION_3) && + header.cupsCompression == 3U) || + ((flags & CF_FUZZ_RASTER_REQUIRE_MULTIROW) && + header.cupsHeight < 2U) || + ((flags & CF_FUZZ_RASTER_REQUIRE_MODE10_RGB) && + (header.cupsCompression != 10U || + header.cupsBitsPerColor != 8U || + header.cupsBitsPerPixel != 24U || + header.cupsNumColors != 3U || + (header.cupsColorSpace != CUPS_CSPACE_RGB && + header.cupsColorSpace != CUPS_CSPACE_SRGB && + header.cupsColorSpace != CUPS_CSPACE_ADOBERGB) || + header.cupsBytesPerLine != header.cupsWidth * 3U))) { + goto done; + } + if ((flags & CF_FUZZ_RASTER_REQUIRE_ESCPX_WEAVE) && + header.cupsRowCount > 1U) { + unsigned row_step = header.cupsRowStep % 100U; + unsigned column_step = header.cupsRowStep / 100U; + uint64_t complexity; + + if (!row_step || !column_step || header.cupsRowCount > 64U || + header.cupsRowFeed > 4096U) { + goto done; + } + complexity = (uint64_t)header.cupsRowCount * row_step * column_step; + if (complexity > 128U) { + goto done; + } + } + page_size = (uint64_t)header.cupsBytesPerLine * header.cupsHeight; + if (decoded + page_size > 8U * 1024U * 1024U) { + goto done; + } + row = (uint8_t *)malloc(header.cupsBytesPerLine); + if (!row) { + goto done; + } + for (unsigned y = 0; y < header.cupsHeight; y++) { + if (cupsRasterReadPixels(raster, row, header.cupsBytesPerLine) != + header.cupsBytesPerLine) { + goto done; + } + } + free(row); + row = NULL; + decoded += page_size; + } + valid = pages > 0U; + +done: + free(row); + if (raster) { + cupsRasterClose(raster); + } + return lseek(fd, 0, SEEK_SET) >= 0 && valid; +} + +#endif diff --git a/parser-fuzzers/parser_fuzzers/__init__.py b/parser-fuzzers/parser_fuzzers/__init__.py deleted file mode 100644 index dc4f2ef..0000000 --- a/parser-fuzzers/parser_fuzzers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Source-tree compatibility package. - -This shim lets `python3 -m parser_fuzzers.cli` work from a checkout without an -editable install. Packaged installs use the real package under `src/`. -""" - -from __future__ import annotations - -from pathlib import Path - -_SRC_PACKAGE = Path(__file__).resolve().parents[1] / "src" / "parser_fuzzers" -if _SRC_PACKAGE.exists(): - __path__.append(str(_SRC_PACKAGE)) # type: ignore[name-defined] diff --git a/parser-fuzzers/project.yaml b/parser-fuzzers/project.yaml new file mode 100644 index 0000000..97906e4 --- /dev/null +++ b/parser-fuzzers/project.yaml @@ -0,0 +1,18 @@ +homepage: "https://github.com/OpenPrinting/cups-filters" +main_repo: "https://github.com/OpenPrinting/cups-filters.git" +language: c++ + +primary_contact: "jiongchiyu@gmail.com" +auto_ccs: + - "till.kamppeter@gmail.com" + - "ossfuzz@iosifache.me" + - "msweet@msweet.org" + +architectures: + - x86_64 + +sanitizers: + - address + +fuzzing_engines: + - libfuzzer diff --git a/parser-fuzzers/pyproject.toml b/parser-fuzzers/pyproject.toml deleted file mode 100644 index 3f482e4..0000000 --- a/parser-fuzzers/pyproject.toml +++ /dev/null @@ -1,20 +0,0 @@ -[build-system] -requires = ["setuptools>=69"] -build-backend = "setuptools.build_meta" - -[project] -name = "parser-fuzzers" -version = "0.1.0" -description = "OpenPrinting parser and filter fuzzing toolkit with SMT-assisted templates" -readme = "README.md" -requires-python = ">=3.10" -dependencies = [ - "PyYAML>=6.0", - "z3-solver>=4.12", -] - -[project.scripts] -parser-fuzzers = "parser_fuzzers.cli:main" - -[tool.setuptools.packages.find] -where = ["src"] diff --git a/parser-fuzzers/requirements.txt b/parser-fuzzers/requirements.txt deleted file mode 100644 index 6d62e04..0000000 --- a/parser-fuzzers/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -PyYAML>=6.0 -z3-solver>=4.12 diff --git a/parser-fuzzers/scripts/README.md b/parser-fuzzers/scripts/README.md deleted file mode 100644 index ba8ff86..0000000 --- a/parser-fuzzers/scripts/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# Script Map - -Scripts are grouped by layer. Most scripts are thin wrappers around the Python -CLI and keep output under `work/` or `findings/`. - -## Smoke And Environment - -- `check_env.sh`: report Python/AFL++ tool availability. -- `run_smoke.sh`: clone-only solver-to-patch smoke test. -- `afl_build_env.sh`: print AFL++ compiler environment. -- `afl_coverage_env.sh`: print AFL++/ASan-oriented build environment. -- `llvm_coverage_env.sh`: print LLVM source-coverage build environment. -- `merge_llvm_coverage.sh`: merge LLVM profile output. - -## AFL++ Boundary - -- `run_afl.sh`: standard AFL++ command builder/runner. -- `build_afl_template_probe.sh`: build the clone-only AFL++ instrumented probe harness. -- `run_template_afl_loop.sh`: run template generation, standard AFL++, frontier import, and feedback-template generation in one file-backed cycle. -- `run_afl_pwg_frontier.sh`: AFL++ frontier campaign for PWG-derived targets. -- `afl_direct_filter_target.sh`: direct filter wrapper for AFL++ `@@`. -- `prepare_afl_frontier_corpus.py`: export retained documents as AFL++ seeds. -- `import_afl_frontier_feedback.py`: import AFL++ queue/crashes as feedback. -- `afl_stats_snapshot.py`: compact AFL++ stats reader. -- `run_afl_feedback_smt_round.sh`: run a template round from AFL++ feedback. -- `archive_afl_crashes.py`: archive AFL++ crashes/hangs and optionally delete - raw crash/hang files after the archive succeeds. -- `run_afl_pwg_bundle.sh`: standard PWG bundle AFL++ runner. Set - `SMT_AFL_BUNDLE_MAX_GB=` to print campaign size and stop AFL++ when the - campaign directory reaches that limit. - -## Template And Runner Campaigns - -- `run_template_runner_campaign.sh`: template/runner campaign that asks for - the filter root before executing configured direct-filter targets. -- `run_multitarget_ppd_fuzz.sh`: multi-target PPD/document monitor. -- `run_explore_ppd_fuzz.sh`: less-directed parser exploration. -- `run_general_parser_campaign.sh`: general parser campaign. -- `run_coverage_discovery_campaign.sh`: coverage-discovery campaign. -- `run_deep_coverage_campaign.sh`: deeper coverage-discovery campaign. -- `run_cold_semantic_campaign.sh`: cold parser semantic campaign. -- `run_feedback_template_campaign.sh`: feedback-driven template run. -- `run_structural_template_campaign.sh`: structural template run. -- `run_arithmetic_explore.sh`: cross-input arithmetic exploration. - -Longer campaigns should be launched with explicit `parser_fuzzers.cli` -command lines so duration, disk limits, configs, and targets are visible in the -run log. - -## Image Campaigns - -- `run_image_feedback_campaign.sh`: image parser feedback campaign. -- `run_image_deep_campaign.sh`: focused image parser campaign. -- `run_image_cycle_campaign.sh`: multi-round image parser cycle. - -## Triage - -- `replay_asan_filter.sh`: replay a retained case under ASan-built filters. -- `gdb_crash_filter.sh`: run a case under GDB. -- `run_baseline_comparison.sh`: local baseline comparison helper. -- `build_historical_reachability_matrix.py`: summarize private replay evidence - against configured parser/harness coverage. Keep generated reports private - unless the contents are safe to disclose. diff --git a/parser-fuzzers/scripts/afl_build_env.sh b/parser-fuzzers/scripts/afl_build_env.sh deleted file mode 100755 index 8970901..0000000 --- a/parser-fuzzers/scripts/afl_build_env.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" - -export PYTHONPATH="$ROOT/src" -python3 -m parser_fuzzers.cli afl-build-env --configs configs diff --git a/parser-fuzzers/scripts/afl_coverage_env.sh b/parser-fuzzers/scripts/afl_coverage_env.sh deleted file mode 100755 index 3259405..0000000 --- a/parser-fuzzers/scripts/afl_coverage_env.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -cat <<'EOF' -# Source this before configuring an AFL++ instrumented build. -# Example: -# source scripts/afl_coverage_env.sh -# cd /data/pre-gsoc/cups-filters -# make clean -# ./configure --disable-shared --enable-static --enable-individual-cups-filters -# make -j"$(nproc)" pwgtoraster rastertoescpx rastertopclx -export CC="${CC:-afl-clang-fast}" -export CXX="${CXX:-afl-clang-fast++}" -export AFL_USE_ASAN="${AFL_USE_ASAN:-1}" -export AFL_LLVM_CMPLOG="${AFL_LLVM_CMPLOG:-1}" -export CFLAGS="-O1 -g -fno-omit-frame-pointer ${CFLAGS:-}" -export CXXFLAGS="-O1 -g -fno-omit-frame-pointer ${CXXFLAGS:-}" -export ASAN_OPTIONS="${ASAN_OPTIONS:-abort_on_error=0:detect_leaks=0:symbolize=1:exitcode=86}" -EOF diff --git a/parser-fuzzers/scripts/afl_direct_filter_target.sh b/parser-fuzzers/scripts/afl_direct_filter_target.sh deleted file mode 100755 index 5b44a73..0000000 --- a/parser-fuzzers/scripts/afl_direct_filter_target.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [[ "$#" -ne 1 ]]; then - echo "usage: $0 " >&2 - exit 2 -fi - -input_document="$1" -filter_binary="${SMT_AFL_DIRECT_FILTER_BINARY:-${AFL_DIRECT_FILTER_BINARY:-}}" -ppd_path="${SMT_AFL_DIRECT_PPD:-${AFL_DIRECT_PPD:-}}" -job_options="${SMT_AFL_DIRECT_JOB_OPTIONS:-${AFL_DIRECT_JOB_OPTIONS:-}}" - -if [[ -z "$filter_binary" || -z "$ppd_path" ]]; then - echo "SMT_AFL_DIRECT_FILTER_BINARY and SMT_AFL_DIRECT_PPD must be set" >&2 - exit 2 -fi - -tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/smt-afl-direct.XXXXXX")" -stderr_file="$tmpdir/stderr.txt" -stdout_file="$tmpdir/stdout.bin" - -cleanup() { - rm -rf "$tmpdir" -} -trap cleanup EXIT - -export PPD="$ppd_path" -direct_ld_library_path="${SMT_AFL_DIRECT_LD_LIBRARY_PATH:-${AFL_DIRECT_LD_LIBRARY_PATH:-}}" -if [[ -n "$direct_ld_library_path" ]]; then - export LD_LIBRARY_PATH="$direct_ld_library_path${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -fi -export ASAN_OPTIONS="${ASAN_OPTIONS:-abort_on_error=1:detect_leaks=0:symbolize=0}" - -set +e -"$filter_binary" 1 afl afl 1 "$job_options" "$input_document" >"$stdout_file" 2>"$stderr_file" -status="$?" -set -e - -if [[ "$status" == "86" ]]; then - cat "$stderr_file" >&2 - exit 86 -fi - -if (( status >= 128 )); then - cat "$stderr_file" >&2 - exit 86 -fi - -if grep -Eq "AddressSanitizer|UndefinedBehaviorSanitizer|ERROR: LeakSanitizer|runtime error:" "$stderr_file"; then - cat "$stderr_file" >&2 - exit 86 -fi - -exit 0 diff --git a/parser-fuzzers/scripts/afl_stats_snapshot.py b/parser-fuzzers/scripts/afl_stats_snapshot.py deleted file mode 100755 index b86b732..0000000 --- a/parser-fuzzers/scripts/afl_stats_snapshot.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -from pathlib import Path - - -FIELDS = [ - "run_time", - "execs_done", - "execs_per_sec", - "corpus_count", - "corpus_favored", - "corpus_found", - "saved_crashes", - "saved_hangs", - "cycles_done", - "bitmap_cvg", - "edges_found", - "stability", -] - - -def main() -> int: - parser = argparse.ArgumentParser(description="print a compact AFL++ fuzzer_stats snapshot") - parser.add_argument("output_dir") - parser.add_argument("--label", default="") - args = parser.parse_args() - - stats_path = _stats_path(Path(args.output_dir)) - stats = _read_stats(stats_path) if stats_path is not None else _read_plot_data(Path(args.output_dir)) - if not stats: - stats = _count_artifacts(Path(args.output_dir)) - prefix = f"{args.label} " if args.label else "" - parts = [f"{field}={stats.get(field, 'n/a')}" for field in FIELDS if field in stats] - for field in ("queue_files", "crash_files", "hang_files"): - if field in stats and field not in FIELDS: - parts.append(f"{field}={stats[field]}") - print(prefix + " ".join(parts)) - return 0 - - -def _stats_path(output_dir: Path) -> Path | None: - candidates = [ - output_dir / "default" / "fuzzer_stats", - output_dir / "fuzzer_stats", - ] - candidates.extend(sorted(output_dir.glob("*/fuzzer_stats"))) - for path in candidates: - if path.exists(): - return path - return None - - -def _read_stats(path: Path) -> dict[str, str]: - stats: dict[str, str] = {} - for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): - if ":" not in line: - continue - key, value = line.split(":", 1) - stats[key.strip()] = value.strip() - return stats - - -def _read_plot_data(output_dir: Path) -> dict[str, str]: - path = output_dir / "plot_data" - if not path.exists(): - return {} - lines = [line.strip() for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.strip()] - if len(lines) < 2: - return _count_artifacts(output_dir) - headers = [item.strip() for item in lines[0].lstrip("#").split(",")] - values = [item.strip() for item in lines[-1].split(",")] - row = dict(zip(headers, values)) - stats = _count_artifacts(output_dir) - if "unix_time" in row: - stats["run_time"] = row.get("run_time", row["unix_time"]) - if "total_execs" in row: - stats["execs_done"] = row["total_execs"] - if "exec_speed" in row: - stats["execs_per_sec"] = row["exec_speed"] - if "cycles_done" in row: - stats["cycles_done"] = row["cycles_done"] - if "saved_crashes" in row: - stats["saved_crashes"] = row["saved_crashes"] - if "saved_hangs" in row: - stats["saved_hangs"] = row["saved_hangs"] - return stats - - -def _count_artifacts(output_dir: Path) -> dict[str, str]: - def count_files(path: Path) -> int: - if not path.exists(): - return 0 - return sum(1 for item in path.iterdir() if item.is_file() and not item.name.startswith("README")) - - return { - "queue_files": str(count_files(output_dir / "queue")), - "crash_files": str(count_files(output_dir / "crashes")), - "hang_files": str(count_files(output_dir / "hangs")), - } - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/parser-fuzzers/scripts/archive_afl_crashes.py b/parser-fuzzers/scripts/archive_afl_crashes.py deleted file mode 100755 index 3c3fad4..0000000 --- a/parser-fuzzers/scripts/archive_afl_crashes.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import shutil -import tarfile -import time -from pathlib import Path - - -def main() -> int: - parser = argparse.ArgumentParser(description="archive AFL++ crash/hang artifacts and optionally prune raw files") - parser.add_argument("--run-dir", required=True, help="AFL++ campaign directory") - parser.add_argument("--output-dir", default="findings", help="directory for the archive") - parser.add_argument("--label", default="", help="archive label; defaults to run-dir name") - parser.add_argument("--delete-after-archive", action="store_true", help="remove raw crash/hang files after archive succeeds") - parser.add_argument("--include-queue", action="store_true", help="also include AFL++ queue files") - args = parser.parse_args() - - run_dir = Path(args.run_dir) - if not run_dir.exists(): - raise SystemExit(f"run dir not found: {run_dir}") - - label = args.label or run_dir.name - stamp = time.strftime("%Y%m%d-%H%M%S") - archive_root = Path(args.output_dir) - archive_root.mkdir(parents=True, exist_ok=True) - staging = archive_root / f"{label}-afl-artifacts-{stamp}" - if staging.exists(): - shutil.rmtree(staging) - staging.mkdir(parents=True) - - copied: list[dict[str, str | int]] = [] - for src in _artifact_paths(run_dir, include_queue=args.include_queue): - rel = src.relative_to(run_dir) - dst = staging / rel - dst.parent.mkdir(parents=True, exist_ok=True) - if src.is_dir(): - shutil.copytree(src, dst, dirs_exist_ok=True) - size = _dir_size(src) - kind = "dir" - else: - shutil.copy2(src, dst) - size = src.stat().st_size - kind = "file" - copied.append({"kind": kind, "source": str(src), "archive_path": str(dst), "bytes": size}) - - manifest = { - "run_dir": str(run_dir), - "staging_dir": str(staging), - "include_queue": args.include_queue, - "delete_after_archive": args.delete_after_archive, - "copied": copied, - "crash_files": len(_raw_files(run_dir, "crashes")), - "hang_files": len(_raw_files(run_dir, "hangs")), - "run_dir_bytes_before": _dir_size(run_dir), - } - (staging / "archive_manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - - archive_path = staging.with_suffix(".tar.gz") - if archive_path.exists(): - archive_path.unlink() - with tarfile.open(archive_path, "w:gz") as tar: - tar.add(staging, arcname=staging.name) - - deleted = [] - if args.delete_after_archive: - for subdir in ("crashes", "hangs"): - for path in _raw_files(run_dir, subdir): - if path.name == "README.txt": - continue - deleted.append({"path": str(path), "bytes": path.stat().st_size}) - path.unlink() - manifest["deleted"] = deleted - manifest["run_dir_bytes_after"] = _dir_size(run_dir) - (staging / "archive_manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - with tarfile.open(archive_path, "w:gz") as tar: - tar.add(staging, arcname=staging.name) - - print( - json.dumps( - { - "archive": str(archive_path), - "staging_dir": str(staging), - "copied_items": len(copied), - "deleted_files": len(deleted), - "run_dir_bytes_before": manifest["run_dir_bytes_before"], - "run_dir_bytes_after": manifest.get("run_dir_bytes_after", manifest["run_dir_bytes_before"]), - "archive_bytes": archive_path.stat().st_size, - }, - indent=2, - sort_keys=True, - ) - ) - return 0 - - -def _artifact_paths(run_dir: Path, *, include_queue: bool) -> list[Path]: - paths: list[Path] = [] - for relative in [ - "standard-metrics.json", - "manifest.json", - "fuzzer_stats", - "afl.log", - "dynamic-pwg-bundle.dict", - "dynamic-dict-export.json", - "dynamic-seed-augment.json", - "bundle-seed-export.json", - "bundle_seed_manifest.json", - "fallback.ppd", - ]: - path = run_dir / relative - if path.exists(): - paths.append(path) - for pattern in ["out/*/fuzzer_stats", "out/*/crashes", "out/*/hangs"]: - paths.extend(sorted(run_dir.glob(pattern))) - if include_queue: - paths.extend(sorted(run_dir.glob("out/*/queue"))) - return _dedupe(paths) - - -def _raw_files(run_dir: Path, subdir: str) -> list[Path]: - files: list[Path] = [] - for root in run_dir.glob(f"out/*/{subdir}"): - files.extend(path for path in root.iterdir() if path.is_file()) - return sorted(files) - - -def _dedupe(paths: list[Path]) -> list[Path]: - seen: set[Path] = set() - deduped: list[Path] = [] - for path in paths: - resolved = path.resolve() - if resolved in seen: - continue - seen.add(resolved) - deduped.append(path) - return deduped - - -def _dir_size(path: Path) -> int: - if path.is_file(): - return path.stat().st_size - total = 0 - for item in path.rglob("*"): - if item.is_file(): - total += item.stat().st_size - return total - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/parser-fuzzers/scripts/augment_pwg_bundle_seeds_from_dynamic_profile.py b/parser-fuzzers/scripts/augment_pwg_bundle_seeds_from_dynamic_profile.py deleted file mode 100755 index a9426c0..0000000 --- a/parser-fuzzers/scripts/augment_pwg_bundle_seeds_from_dynamic_profile.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json - -from parser_fuzzers.afl_dynamic_bridge import augment_pwg_bundle_seed_dir - - -def main() -> int: - parser = argparse.ArgumentParser(description="augment PWG bundle AFL++ seeds with dynamic compare profile tokens") - parser.add_argument("--seed-dir", required=True) - parser.add_argument("--profile", required=True) - parser.add_argument("--output-dir", default="") - parser.add_argument("--limit", type=int, default=64) - args = parser.parse_args() - - manifest = augment_pwg_bundle_seed_dir( - args.seed_dir, - args.profile, - output_dir=args.output_dir or None, - limit=args.limit, - ) - print(json.dumps(manifest, indent=2, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/parser-fuzzers/scripts/build_afl_cupsfilters_stack.sh b/parser-fuzzers/scripts/build_afl_cupsfilters_stack.sh deleted file mode 100755 index 28e955f..0000000 --- a/parser-fuzzers/scripts/build_afl_cupsfilters_stack.sh +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" - -SRC_ROOT="${SMT_AFL_SRC_ROOT:-/data/pre-gsoc}" -BUILD_ROOT="${SMT_AFL_BUILD_ROOT:-$ROOT/work/afl-builds}" -INSTALL_ROOT="${SMT_AFL_INSTALL_ROOT:-$ROOT/work/afl-install}" -SRC_COPY_ROOT="${SMT_AFL_SRC_COPY_ROOT:-$ROOT/work/afl-src}" -LOG_ROOT="${SMT_AFL_BUILD_LOG_ROOT:-$ROOT/work/build-afl-cupsfilters}" -JOBS="${JOBS:-$(nproc)}" - -LIBPPD_SRC="${SMT_AFL_LIBPPD_SRC:-$SRC_ROOT/libppd-origin-latest}" -LIBCUPSFILTERS_SRC="${SMT_AFL_LIBCUPSFILTERS_SRC:-$SRC_ROOT/libcupsfilters}" -CUPSFILTERS_SRC="${SMT_AFL_CUPSFILTERS_SRC:-$SRC_ROOT/cups-filters}" - -LIBPPD_PREFIX="$INSTALL_ROOT/libppd" -LIBCUPSFILTERS_PREFIX="$INSTALL_ROOT/libcupsfilters" -CUPSFILTERS_PREFIX="$INSTALL_ROOT/cups-filters" -PDFIO_PREFIX="${SMT_AFL_PDFIO_PREFIX:-/data/pre-gsoc/env/pdfio-install}" - -COMMON_CFLAGS="-O1 -g -fno-omit-frame-pointer ${CFLAGS:-}" -COMMON_CXXFLAGS="-O1 -g -fno-omit-frame-pointer ${CXXFLAGS:-}" -COMMON_LDFLAGS="-fsanitize=address ${LDFLAGS:-}" - -export CC="${CC:-afl-clang-fast}" -export CXX="${CXX:-afl-clang-fast++}" -export AFL_USE_ASAN="${AFL_USE_ASAN:-1}" -export AFL_LLVM_CMPLOG="${AFL_LLVM_CMPLOG:-1}" -export ASAN_OPTIONS="${ASAN_OPTIONS:-abort_on_error=1:detect_leaks=0:symbolize=0}" - -mkdir -p "$BUILD_ROOT" "$INSTALL_ROOT" "$SRC_COPY_ROOT" "$LOG_ROOT" -rm -f "$LOG_ROOT"/*.log - -copy_source_tree() { - local name="$1" - local src="$2" - local dest="$SRC_COPY_ROOT/$name" - - rm -rf "$dest" - mkdir -p "$dest" - ( - cd "$src" - tar \ - --exclude=.git \ - --exclude=autom4te.cache \ - --exclude=.libs \ - --exclude='*.o' \ - --exclude='*.lo' \ - --exclude='*.la' \ - --exclude='*.log' \ - --exclude='*.trs' \ - --exclude=config.status \ - --exclude=config.log \ - --exclude=config.cache \ - --exclude=config.h \ - --exclude=stamp-h1 \ - --exclude=Makefile \ - --exclude=libtool \ - -cf - . - ) | ( - cd "$dest" - tar --no-same-owner -xf - - ) - if [[ -f "$dest/charset/pdf.utf-8.heavy" && ! -f "$dest/charset/pdf.utf-8.heavy.in" ]]; then - cp "$dest/charset/pdf.utf-8.heavy" "$dest/charset/pdf.utf-8.heavy.in" - fi - if [[ -f "$dest/charset/pdf.utf-8.simple" && ! -f "$dest/charset/pdf.utf-8.simple.in" ]]; then - cp "$dest/charset/pdf.utf-8.simple" "$dest/charset/pdf.utf-8.simple.in" - fi - echo "$dest" -} - -build_autotools() { - local name="$1" - local src="$2" - local build="$3" - local prefix="$4" - shift 4 - - if [[ ! -x "$src/configure" ]]; then - echo "missing configure script: $src/configure" >&2 - return 2 - fi - - rm -rf "$build" "$prefix" - mkdir -p "$build" - ( - cd "$src" - find . -type d \ - ! -path './.git*' \ - ! -path './autom4te.cache*' \ - ! -path './.libs*' \ - -exec mkdir -p "$build/{}" \; - ) - ( - cd "$build" - echo "[build:$name] configure" - "$src/configure" \ - --prefix="$prefix" \ - --disable-dependency-tracking \ - "$@" \ - >"$LOG_ROOT/$name.configure.log" 2>&1 - echo "[build:$name] make -j$JOBS" - make -j"$JOBS" >"$LOG_ROOT/$name.make.log" 2>&1 - echo "[build:$name] make install" - make install >"$LOG_ROOT/$name.install.log" 2>&1 - ) -} - -export CFLAGS="$COMMON_CFLAGS" -export CXXFLAGS="$COMMON_CXXFLAGS" -export LDFLAGS="$COMMON_LDFLAGS" - -LIBPPD_BUILD_SRC="$(copy_source_tree libppd "$LIBPPD_SRC")" -LIBCUPSFILTERS_BUILD_SRC="$(copy_source_tree libcupsfilters "$LIBCUPSFILTERS_SRC")" -CUPSFILTERS_BUILD_SRC="$(copy_source_tree cups-filters "$CUPSFILTERS_SRC")" - -build_autotools \ - libppd \ - "$LIBPPD_BUILD_SRC" \ - "$BUILD_ROOT/libppd" \ - "$LIBPPD_PREFIX" \ - --enable-shared \ - --enable-static - -export PKG_CONFIG_PATH="$LIBPPD_PREFIX/lib/pkgconfig:$PDFIO_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" - -LIBCUPSFILTERS_STATUS="afl-built" -if build_autotools \ - libcupsfilters \ - "$LIBCUPSFILTERS_BUILD_SRC" \ - "$BUILD_ROOT/libcupsfilters" \ - "$LIBCUPSFILTERS_PREFIX" \ - --enable-shared \ - --enable-static; then - LIBCUPSFILTERS_INCLUDE_ROOT="$LIBCUPSFILTERS_PREFIX/include" - LIBCUPSFILTERS_LIB_DIR="$LIBCUPSFILTERS_PREFIX/lib" - export PKG_CONFIG_PATH="$LIBCUPSFILTERS_PREFIX/lib/pkgconfig:$LIBPPD_PREFIX/lib/pkgconfig:$PDFIO_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" -else - if [[ -f "$LIBCUPSFILTERS_PREFIX/lib/libcupsfilters.so.2.0.0" ]]; then - LIBCUPSFILTERS_STATUS="afl-built-partial-install" - LIBCUPSFILTERS_INCLUDE_ROOT="$LIBCUPSFILTERS_BUILD_SRC" - LIBCUPSFILTERS_LIB_DIR="$LIBCUPSFILTERS_PREFIX/lib" - { - echo "libcupsfilters AFL++ library was installed, but data install failed." - echo "This is acceptable for fuzzing because the project-local library is present." - echo "install_log=$LOG_ROOT/libcupsfilters.install.log" - echo "include_root=$LIBCUPSFILTERS_INCLUDE_ROOT" - echo "lib_dir=$LIBCUPSFILTERS_LIB_DIR" - } >"$LOG_ROOT/libcupsfilters.partial-install.log" - export PKG_CONFIG_PATH="$LIBPPD_PREFIX/lib/pkgconfig:$PDFIO_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" - elif [[ -f "$BUILD_ROOT/libcupsfilters/.libs/libcupsfilters.so.2.0.0" ]]; then - LIBCUPSFILTERS_STATUS="afl-built-builddir" - LIBCUPSFILTERS_INCLUDE_ROOT="$LIBCUPSFILTERS_BUILD_SRC" - LIBCUPSFILTERS_LIB_DIR="$BUILD_ROOT/libcupsfilters/.libs" - { - echo "libcupsfilters AFL++ library was built but not installed; using builddir library." - echo "install_log=$LOG_ROOT/libcupsfilters.install.log" - echo "include_root=$LIBCUPSFILTERS_INCLUDE_ROOT" - echo "lib_dir=$LIBCUPSFILTERS_LIB_DIR" - } >"$LOG_ROOT/libcupsfilters.builddir.log" - export PKG_CONFIG_PATH="$LIBPPD_PREFIX/lib/pkgconfig:$PDFIO_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" - else - LIBCUPSFILTERS_STATUS="fallback-local-asan" - LIBCUPSFILTERS_INCLUDE_ROOT="${SMT_AFL_LIBCUPSFILTERS_INCLUDE:-$LIBCUPSFILTERS_SRC}" - LIBCUPSFILTERS_LIB_DIR="${SMT_AFL_LIBCUPSFILTERS_LIB:-$LIBCUPSFILTERS_SRC/.libs}" - if [[ ! -d "$LIBCUPSFILTERS_INCLUDE_ROOT" || ! -d "$LIBCUPSFILTERS_LIB_DIR" ]]; then - echo "libcupsfilters AFL++ build failed and fallback path is missing" >&2 - echo "include_root=$LIBCUPSFILTERS_INCLUDE_ROOT" >&2 - echo "lib_dir=$LIBCUPSFILTERS_LIB_DIR" >&2 - exit 1 - fi - { - echo "libcupsfilters AFL++ build failed; using existing local ASan build for now." - echo "configure_log=$LOG_ROOT/libcupsfilters.configure.log" - echo "make_log=$LOG_ROOT/libcupsfilters.make.log" - echo "install_log=$LOG_ROOT/libcupsfilters.install.log" - echo "include_root=$LIBCUPSFILTERS_INCLUDE_ROOT" - echo "lib_dir=$LIBCUPSFILTERS_LIB_DIR" - } >"$LOG_ROOT/libcupsfilters.fallback.log" - export PKG_CONFIG_PATH="$LIBPPD_PREFIX/lib/pkgconfig:$PDFIO_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" - fi -fi - -export LIBCUPSFILTERS_CFLAGS="-I$LIBCUPSFILTERS_INCLUDE_ROOT -I$LIBCUPSFILTERS_INCLUDE_ROOT/cupsfilters" -export LIBCUPSFILTERS_LIBS="-L$LIBCUPSFILTERS_LIB_DIR -Wl,-rpath,$LIBCUPSFILTERS_LIB_DIR -lcupsfilters -L$PDFIO_PREFIX/lib -Wl,-rpath,$PDFIO_PREFIX/lib" -export LIBPPD_CFLAGS="-I$LIBPPD_PREFIX/include/ppd -I$LIBPPD_PREFIX/include" -export LIBPPD_LIBS="-L$LIBPPD_PREFIX/lib -Wl,-rpath,$LIBPPD_PREFIX/lib -lppd" -export LDFLAGS="$COMMON_LDFLAGS -Wl,-rpath,$LIBCUPSFILTERS_LIB_DIR -Wl,-rpath,$LIBPPD_PREFIX/lib -Wl,-rpath,$PDFIO_PREFIX/lib" - -CUPSFILTERS_INSTALL_STATUS="make-install" -if build_autotools \ - cups-filters \ - "$CUPSFILTERS_BUILD_SRC" \ - "$BUILD_ROOT/cups-filters" \ - "$CUPSFILTERS_PREFIX" \ - --enable-individual-cups-filters \ - --disable-shared \ - --enable-static; then - CUPSFILTERS_INSTALL_STATUS="make-install" -else - if [[ ! -x "$BUILD_ROOT/cups-filters/pwgtopdf" ]]; then - echo "cups-filters build failed before producing pwgtopdf" >&2 - echo "make_log=$LOG_ROOT/cups-filters.make.log" >&2 - echo "install_log=$LOG_ROOT/cups-filters.install.log" >&2 - exit 1 - fi - CUPSFILTERS_INSTALL_STATUS="manual-copy-after-install-hook-failure" -fi - -mkdir -p "$CUPSFILTERS_PREFIX/lib/cups/filter" -find "$BUILD_ROOT/cups-filters" -maxdepth 1 -type f -perm -111 \ - ! -name config.status \ - ! -name libtool \ - -exec cp -a {} "$CUPSFILTERS_PREFIX/lib/cups/filter/" \; - -cat >"$INSTALL_ROOT/afl-env.sh" < pwgtopdf", - ("pwg_to_pdf", "pwgtopdf", "pwg-bundle", "cfimagergbtowhite", "cluster-03"), - "PWG document plus PPD/job options, standard AFL++ seedable", - "Good fit for template-generated PWG seeds and AFL++ CmpLog/dictionary.", - ), - Family( - "pwg_pwgtopclm", - "PWG Raster -> pwgtopclm", - ("pwg_to_pclm", "pwgtopclm", "pclm"), - "PWG document plus PPD/job options, standard AFL++ seedable", - "Sibling path to pwgtopdf; useful for format-depth comparison.", - ), - Family( - "image_imagetoraster", - "Image -> imagetoraster", - ("imagetoraster", "image_to_imagetoraster"), - "Image document plus PPD/job options", - "Needs image templates, dimensions, color models, and valid/near-valid headers.", - ), - Family( - "image_imagetops", - "Image -> imagetops", - ("imagetops", "image_to_imagetops"), - "Image document plus PPD/job options", - "Historically produced several shallow and mid-depth image conversion crashes.", - ), - Family( - "image_imagetopdf", - "Image -> imagetopdf", - ("imagetopdf", "image_to_imagetopdf"), - "Image document plus PPD/job options", - "Useful as a control target for image parsing without PostScript output paths.", - ), - Family( - "cups_raster_escpx", - "CUPS Raster -> rastertoescpx/commandtoescpx", - ("rastertoescpx", "commandtoescpx", "escpx", "cups_raster_to_rastertoescpx"), - "CUPS raster/command document plus PPD/job options", - "Requires raster-specific structural templates; generic image seeds are not enough.", - ), - Family( - "cups_raster_pclx", - "CUPS Raster -> rastertopclx/commandtopclx", - ("rastertopclx", "commandtopclx", "pclx", "cups_raster_to_rastertopclx"), - "CUPS raster/command document plus PPD/job options", - "Requires row/plane/header consistency to get past early parser checks.", - ), - Family( - "pdf_filters", - "PDF filters", - ("pdftopdf", "pdftops", "pdftoraster", "mupdftopwg", "qpdf", ".pdf"), - "PDF document plus PPD/job options", - "Coverage can be lower unless seeds contain enough valid PDF structure.", - ), - Family( - "postscript_gs", - "PostScript / Ghostscript wrappers", - ("postscript", "gstoraster", "gstopdf", "gstopxl"), - "PostScript document plus wrapper environment", - "Often depends on external Ghostscript behavior and wrapper argument shape.", - ), - Family( - "text_filters", - "Text filters", - ("texttopdf", "texttotext", "text_to_"), - "Text document plus PPD/job options", - "Good low-cost parser lane; lower historical crash density so far.", - ), -) - - -def main() -> int: - parser = argparse.ArgumentParser(description="build a heuristic historical crash reachability matrix") - parser.add_argument("--project-root", default=".") - parser.add_argument("--archive", default="", help="historical crash archive tar.gz; latest archive is used when omitted") - parser.add_argument("--configs", default="configs") - parser.add_argument("--output-json", required=True) - parser.add_argument("--output-md", required=True) - args = parser.parse_args() - - root = Path(args.project_root).resolve() - archive = Path(args.archive).resolve() if args.archive else _latest_archive(root / "findings") - if not archive or not archive.exists(): - raise SystemExit("historical crash archive not found") - - evidence_paths = _archive_names(archive) - targets = _load_targets(root / args.configs) - rows = [_family_row(root, family, evidence_paths, targets) for family in FAMILIES] - payload = { - "project_root": str(root), - "archive": str(archive), - "evidence_path_count": len(evidence_paths), - "rows": rows, - } - - Path(args.output_json).parent.mkdir(parents=True, exist_ok=True) - Path(args.output_json).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - Path(args.output_md).write_text(_render_md(payload), encoding="utf-8") - print(json.dumps({"archive": str(archive), "rows": len(rows), "output_json": args.output_json, "output_md": args.output_md}, indent=2)) - return 0 - - -def _latest_archive(findings: Path) -> Path | None: - archives = sorted(findings.glob("2026-06-06-historical-crash-archive-*.tar.gz"), key=lambda path: path.stat().st_mtime) - return archives[-1] if archives else None - - -def _archive_names(archive: Path) -> list[str]: - with tarfile.open(archive, "r:gz") as tar: - return [member.name for member in tar.getmembers()] - - -def _load_targets(configs: Path) -> list[dict[str, Any]]: - targets: list[dict[str, Any]] = [] - for path in sorted(configs.glob("parser_targets*.yaml")) + sorted(configs.glob("targets.yaml")): - try: - data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - except Exception: - continue - for item in data.get("targets", []) or []: - record = dict(item) - record["config_path"] = str(path) - targets.append(record) - return targets - - -def _family_row(root: Path, family: Family, evidence_paths: list[str], targets: list[dict[str, Any]]) -> dict[str, Any]: - needles = tuple(needle.lower() for needle in family.needles) - matches = [path for path in evidence_paths if any(needle in path.lower() for needle in needles)] - target_matches = [ - target for target in targets if _target_matches(target, needles) - ] - afl_binaries = sorted(_afl_binaries_for(root, target_matches)) - reachability = "low" - if target_matches and afl_binaries: - reachability = "high" - elif target_matches: - reachability = "medium" - elif matches: - reachability = "needs-harness" - return { - "family_id": family.family_id, - "title": family.title, - "historical_evidence_count": len(matches), - "historical_evidence_examples": matches[:8], - "configured_targets": [ - { - "id": target.get("id", ""), - "config_path": target.get("config_path", ""), - "filter_binary": target.get("filter_binary", ""), - "document_kind": target.get("document_kind", ""), - "ppd_kind": target.get("ppd_kind", ""), - } - for target in target_matches[:12] - ], - "afl_binaries": afl_binaries, - "reachability": reachability, - "required_harness": family.required_harness, - "notes": family.notes, - } - - -def _target_matches(target: dict[str, Any], needles: tuple[str, ...]) -> bool: - text = " ".join( - str(target.get(key, "")) - for key in ("id", "description", "filter_binary", "document_kind", "ppd_kind", "input_mime") - ).lower() - return any(needle in text for needle in needles) - - -def _afl_binaries_for(root: Path, targets: list[dict[str, Any]]) -> set[str]: - binaries: set[str] = set() - filter_root = root / "work" / "afl-builds" / "cups-filters" - for target in targets: - binary = str(target.get("filter_binary", "")) - name = Path(binary).name - if name and (filter_root / name).exists(): - binaries.add(str(filter_root / name)) - if (root / "work" / "afl" / "bin" / "pwg_bundle_harness").exists(): - for target in targets: - if "pwg" in str(target.get("id", "")): - binaries.add(str(root / "work" / "afl" / "bin" / "pwg_bundle_harness")) - return binaries - - -def _render_md(payload: dict[str, Any]) -> str: - lines = [ - "# Historical Crash Reachability Matrix", - "", - f"- Archive: `{payload['archive']}`", - f"- Evidence paths scanned: `{payload['evidence_path_count']}`", - "", - "| Family | Evidence | Configured Targets | AFL++ Binaries | Reachability | Harness |", - "|---|---:|---:|---:|---|---|", - ] - for row in payload["rows"]: - lines.append( - "| {title} | {evidence} | {targets} | {binaries} | {reachability} | {harness} |".format( - title=row["title"], - evidence=row["historical_evidence_count"], - targets=len(row["configured_targets"]), - binaries=len(row["afl_binaries"]), - reachability=row["reachability"], - harness=row["required_harness"], - ) - ) - lines.extend(["", "## Notes", ""]) - for row in payload["rows"]: - lines.append(f"### {row['title']}") - lines.append("") - lines.append(f"- Reachability: `{row['reachability']}`") - lines.append(f"- Notes: {row['notes']}") - if row["historical_evidence_examples"]: - lines.append("- Evidence examples:") - for example in row["historical_evidence_examples"][:5]: - lines.append(f" - `{example}`") - if row["configured_targets"]: - lines.append("- Target examples:") - for target in row["configured_targets"][:5]: - lines.append(f" - `{target['id']}` via `{target['filter_binary']}`") - lines.append("") - return "\n".join(lines) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/parser-fuzzers/scripts/check_cups_filters_targets.sh b/parser-fuzzers/scripts/check_cups_filters_targets.sh deleted file mode 100755 index 2630628..0000000 --- a/parser-fuzzers/scripts/check_cups_filters_targets.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash -set -u - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -mode="minimal" - -if [[ "${1:-}" == "--coverage" ]]; then - mode="coverage" - shift -fi - -FILTER_ROOT="${1:-${SMT_FUZZER_FILTER_ROOT:-/usr/lib/cups/filter}}" -missing=0 - -check_cmd() { - local name="$1" - if command -v "$name" >/dev/null 2>&1; then - echo "[ok] $name: $(command -v "$name")" - else - echo "[missing] $name" - missing=1 - fi -} - -check_filter() { - local name="$1" - local path="$FILTER_ROOT/$name" - if [[ -x "$path" ]]; then - echo "[ok] $name: $path" - else - echo "[missing] $name: $path" - missing=1 - fi -} - -minimal_filters=( - rastertopclx - rastertoescpx - pwgtoraster -) - -coverage_filters=( - rastertopclx - rastertoescpx - rastertops - pwgtoraster - pwgtopdf - pdftopdf - pdftops - pdftoraster - mupdftopwg - imagetoraster - imagetopdf - imagetops - texttopdf - texttotext - gstoraster - gstopdf - gstopxl - pwgtopclm - commandtoescpx - commandtopclx -) - -echo "[info] project: $ROOT" -echo "[info] filter root: $FILTER_ROOT" -echo "[info] mode: $mode" - -check_cmd python3 -check_cmd cupstestppd -check_cmd cupsfilter - -if [[ "$mode" == "coverage" ]]; then - filters=("${coverage_filters[@]}") -else - filters=("${minimal_filters[@]}") -fi - -for filter in "${filters[@]}"; do - check_filter "$filter" -done - -if [[ "$missing" == "0" ]]; then - echo "[ok] CUPS filter target check passed" - echo "export SMT_FUZZER_FILTER_ROOT=$(printf '%q' "$FILTER_ROOT")" -else - echo "[note] missing tools or filters above; install cups-filters or point SMT_FUZZER_FILTER_ROOT at a build tree" -fi - -exit "$missing" diff --git a/parser-fuzzers/scripts/check_env.sh b/parser-fuzzers/scripts/check_env.sh deleted file mode 100755 index c8ba47d..0000000 --- a/parser-fuzzers/scripts/check_env.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -u - -strict=0 -if [[ "${1:-}" == "--strict" ]]; then - strict=1 -fi - -missing_required=0 - -check_cmd() { - local name="$1" - local required="$2" - if command -v "$name" >/dev/null 2>&1; then - echo "[ok] $name: $(command -v "$name")" - else - if [[ "$required" == "yes" ]]; then - echo "[missing] $name is required" - missing_required=1 - else - echo "[optional-missing] $name is not in PATH" - fi - fi -} - -check_python_module() { - local module="$1" - local required="$2" - if python3 -c "import ${module}" >/dev/null 2>&1; then - echo "[ok] python module ${module}" - else - if [[ "$required" == "yes" ]]; then - echo "[missing] python module ${module}; run: python3 -m pip install -r requirements.txt" - missing_required=1 - else - echo "[optional-missing] python module ${module}" - fi - fi -} - -check_cmd python3 yes -check_cmd clang no -check_cmd llvm-cov no -check_cmd afl-fuzz no -check_cmd afl-clang-fast no -check_python_module yaml yes -check_python_module z3 yes - -if [[ "$strict" == "1" && "$missing_required" != "0" ]]; then - exit 1 -fi - -if [[ "$missing_required" != "0" ]]; then - echo "[note] required runtime dependencies are missing for strict SMT runs; default smoke can still test the wiring." -fi - -exit 0 diff --git a/parser-fuzzers/scripts/dynamic_profile_to_afl_dict.py b/parser-fuzzers/scripts/dynamic_profile_to_afl_dict.py deleted file mode 100755 index 72a7747..0000000 --- a/parser-fuzzers/scripts/dynamic_profile_to_afl_dict.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json - -from parser_fuzzers.afl_dynamic_bridge import write_dynamic_afl_dictionary - - -def main() -> int: - parser = argparse.ArgumentParser(description="convert dynamic compare profile tokens into an AFL++ dictionary") - parser.add_argument("--profile", required=True) - parser.add_argument("--output", required=True) - parser.add_argument("--base-dictionary", default="") - parser.add_argument("--max-tokens", type=int, default=512) - args = parser.parse_args() - - manifest = write_dynamic_afl_dictionary( - args.profile, - args.output, - base_dictionary=args.base_dictionary or None, - max_tokens=args.max_tokens, - ) - print(json.dumps(manifest, indent=2, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/parser-fuzzers/scripts/export_pwg_bundle_seeds.py b/parser-fuzzers/scripts/export_pwg_bundle_seeds.py deleted file mode 100755 index 69ba358..0000000 --- a/parser-fuzzers/scripts/export_pwg_bundle_seeds.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import shutil -from pathlib import Path - - -MAGIC = b"SMT_PWG_BUNDLE_V1\n" -PPD_MARK = b"--SMT-PPD--\n" -OPTIONS_MARK = b"--SMT-OPTIONS--\n" -DOCUMENT_MARK = b"--SMT-DOCUMENT--\n" - - -def main() -> int: - parser = argparse.ArgumentParser(description="export AFL++ PWG bundle seeds with PPD, job options, and document") - parser.add_argument("--run-dir", required=True) - parser.add_argument("--target-id", required=True) - parser.add_argument("--output-dir", required=True) - parser.add_argument("--limit", type=int, default=512) - args = parser.parse_args() - - run_dir = Path(args.run_dir) - interesting = run_dir / "corpus" / "interesting" / args.target_id - if not interesting.exists(): - raise SystemExit(f"interesting corpus not found: {interesting}") - - output_dir = Path(args.output_dir) - if output_dir.exists(): - shutil.rmtree(output_dir) - output_dir.mkdir(parents=True) - - exported = [] - for case_dir in sorted(interesting.glob("case-*")): - if args.limit > 0 and len(exported) >= args.limit: - break - ppd = case_dir / "candidate.ppd" - doc = case_dir / "document.pwg" - if not ppd.exists() or not doc.exists(): - continue - options = _job_options(case_dir) - out = output_dir / f"{args.target_id}-{case_dir.name}-{len(exported):06d}.pwg-bundle" - out.write_bytes( - MAGIC - + PPD_MARK - + ppd.read_bytes() - + b"\n" - + OPTIONS_MARK - + options.encode("utf-8", errors="replace") - + b"\n" - + DOCUMENT_MARK - + doc.read_bytes() - ) - exported.append( - { - "source_case_dir": str(case_dir), - "source_ppd": str(ppd), - "source_document": str(doc), - "job_options": options, - "output_path": str(out), - } - ) - - if not exported: - raise SystemExit(f"no bundle seeds exported from {interesting}") - - manifest = { - "run_dir": str(run_dir), - "target_id": args.target_id, - "output_dir": str(output_dir), - "exported": len(exported), - "format": "SMT_PWG_BUNDLE_V1", - "seeds": exported, - } - (output_dir / "bundle_seed_manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - print(json.dumps(manifest, indent=2, sort_keys=True)) - return 0 - - -def _job_options(case_dir: Path) -> str: - meta = case_dir / "meta.json" - if not meta.exists(): - return "" - try: - payload = json.loads(meta.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return "" - return str(payload.get("job_options") or "") - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/parser-fuzzers/scripts/gdb_crash_filter.sh b/parser-fuzzers/scripts/gdb_crash_filter.sh deleted file mode 100755 index e5589a5..0000000 --- a/parser-fuzzers/scripts/gdb_crash_filter.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -FILTER="${1:-}" -CASE_DIR="${2:-}" -BREAKPOINT="${3:-}" -OUT_DIR="${4:-}" - -if [[ -z "$FILTER" || -z "$CASE_DIR" ]]; then - echo "usage: $0 [breakpoint] [out-dir]" >&2 - echo "example: $0 pwgtoraster work/arithmetic-explore//cases/case-000001 cupsfilters/pwgtoraster.c:1906" >&2 - exit 2 -fi - -if [[ "$FILTER" = /* ]]; then - FILTER_BIN="$FILTER" - FILTER_NAME="$(basename "$FILTER")" -else - FILTER_NAME="$FILTER" - FILTER_BIN="/data/pre-gsoc/cups-filters/$FILTER_NAME" -fi - -CASE_DIR="$(cd "$CASE_DIR" && pwd)" -OUT_DIR="${OUT_DIR:-$ROOT/work/asan-replay/$(basename "$CASE_DIR")-$FILTER_NAME-gdb}" -mkdir -p "$OUT_DIR" - -PPD_FILE="$CASE_DIR/candidate.ppd" -if [[ ! -f "$PPD_FILE" ]]; then - echo "missing PPD: $PPD_FILE" >&2 - exit 2 -fi - -DOC_FILE="" -for candidate in \ - "$CASE_DIR"/document.pwg \ - "$CASE_DIR"/document.ras \ - "$CASE_DIR"/document.pdf \ - "$CASE_DIR"/document.ps \ - "$CASE_DIR"/document.txt \ - "$CASE_DIR"/document.ppm \ - "$CASE_DIR"/document.pgm \ - "$CASE_DIR"/document.pbm \ - "$CASE_DIR"/document.png \ - "$CASE_DIR"/document.cmd \ - "$CASE_DIR"/document.bin; do - if [[ -f "$candidate" ]]; then - DOC_FILE="$candidate" - break - fi -done - -if [[ -z "$DOC_FILE" ]]; then - echo "missing document input in $CASE_DIR" >&2 - exit 2 -fi - -if [[ ! -x "$FILTER_BIN" ]]; then - echo "missing executable filter: $FILTER_BIN" >&2 - exit 2 -fi - -LIBPPD_ASAN="${LIBPPD_ASAN:-${SMT_FUZZER_LIBPPD_ASAN:-/data/pre-gsoc/libppd-origin-latest/.libs}}" -LIBCUPSFILTERS_ASAN="${LIBCUPSFILTERS_ASAN:-${SMT_FUZZER_LIBCUPSFILTERS_ASAN:-/data/pre-gsoc/libcupsfilters-master-asan/.libs}}" -if [[ ! -d "$LIBCUPSFILTERS_ASAN" ]]; then - LIBCUPSFILTERS_ASAN="/data/pre-gsoc/libcupsfilters/.libs" -fi -PDFIO_LIB="${PDFIO_LIB:-${SMT_FUZZER_PDFIO_LIB:-/data/pre-gsoc/env/pdfio-install/lib}}" -LD_LIBRARY_PATH_VALUE="$LIBPPD_ASAN:$LIBCUPSFILTERS_ASAN:$PDFIO_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -ASAN_OPTIONS_VALUE="${ASAN_OPTIONS:-abort_on_error=1:detect_leaks=0:symbolize=1}" - -GDB_ARGS=( - --batch - -ex "set debuginfod enabled off" - -ex "set breakpoint pending on" - -ex "set env LD_LIBRARY_PATH=$LD_LIBRARY_PATH_VALUE" - -ex "set env ASAN_OPTIONS=$ASAN_OPTIONS_VALUE" - -ex "set env PPD=$PPD_FILE" -) - -if [[ -n "$BREAKPOINT" ]]; then - GDB_ARGS+=(-ex "break $BREAKPOINT") -fi - -GDB_ARGS+=( - -ex "run" - -ex "bt full" - -ex "frame 0" - -ex "info locals" - --args "$FILTER_BIN" 1 smt smt 1 "" "$DOC_FILE" -) - -set +e -gdb "${GDB_ARGS[@]}" > "$OUT_DIR/gdb.txt" 2>&1 -status=$? -set -e - -{ - echo "status=$status" - echo "filter=$FILTER_BIN" - echo "ppd=$PPD_FILE" - echo "document=$DOC_FILE" - echo "breakpoint=${BREAKPOINT:-}" - echo "gdb=$OUT_DIR/gdb.txt" -} > "$OUT_DIR/summary.txt" - -cat "$OUT_DIR/summary.txt" -exit "$status" diff --git a/parser-fuzzers/scripts/import_afl_frontier_feedback.py b/parser-fuzzers/scripts/import_afl_frontier_feedback.py deleted file mode 100755 index 006b7cc..0000000 --- a/parser-fuzzers/scripts/import_afl_frontier_feedback.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json - -from parser_fuzzers.afl_feedback import import_afl_artifacts - - -def main() -> int: - parser = argparse.ArgumentParser(description="import AFL++ queue/crashes into a runner-style feedback run") - parser.add_argument("--afl-out", required=True, help="AFL++ output dir, instance dir, or campaign dir") - parser.add_argument("--target-id", required=True) - parser.add_argument("--output-run-dir", required=True) - parser.add_argument("--extension", default=".pwg") - parser.add_argument("--queue-limit", type=int, default=512) - parser.add_argument("--crash-limit", type=int, default=128) - parser.add_argument( - "--queue-mode", - choices=["new", "all", "none"], - default="new", - help="new imports AFL-discovered queue entries; all also imports original seeds", - ) - args = parser.parse_args() - - summary = import_afl_artifacts( - afl_out=args.afl_out, - target_id=args.target_id, - output_run_dir=args.output_run_dir, - extension=args.extension, - queue_limit=args.queue_limit, - crash_limit=args.crash_limit, - queue_mode=args.queue_mode, - ) - print(json.dumps(summary.to_dict(), indent=2, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/parser-fuzzers/scripts/install_ubuntu_deps.sh b/parser-fuzzers/scripts/install_ubuntu_deps.sh deleted file mode 100755 index 229f22e..0000000 --- a/parser-fuzzers/scripts/install_ubuntu_deps.sh +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - cat <<'EOF' -usage: scripts/install_ubuntu_deps.sh [mode] [options] - -Modes: - --minimal Python smoke dependencies only - --system-filters minimal + Ubuntu CUPS/cups-filters runtime - --asan-build minimal + local OpenPrinting ASan build tools/deps - --afl minimal + AFL++ tooling - --all install all groups (default) - -Options: - --dry-run print apt-get commands without installing - --no-update skip apt-get update - -y, --yes pass -y to apt-get install - -h, --help show this help - -Examples: - scripts/install_ubuntu_deps.sh --dry-run - scripts/install_ubuntu_deps.sh --minimal - scripts/install_ubuntu_deps.sh --asan-build -y - scripts/install_ubuntu_deps.sh --all -y -EOF -} - -print_shell_command() { - printf '[dry-run]' - for arg in "$@"; do - printf ' %q' "$arg" - done - printf '\n' -} - -mode="all" -dry_run=0 -apt_update=1 -assume_yes=0 - -while [[ "$#" -gt 0 ]]; do - case "$1" in - --minimal|--system-filters|--asan-build|--afl|--all) - mode="${1#--}" - ;; - --dry-run) - dry_run=1 - ;; - --no-update) - apt_update=0 - ;; - -y|--yes) - assume_yes=1 - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "unknown argument: $1" >&2 - usage >&2 - exit 2 - ;; - esac - shift -done - -if [[ -r /etc/os-release ]]; then - # shellcheck disable=SC1091 - . /etc/os-release - if [[ "${ID:-}" != "ubuntu" && "${ID_LIKE:-}" != *"ubuntu"* && "${ID_LIKE:-}" != *"debian"* ]]; then - echo "[warn] this script is tuned for Ubuntu/Debian; detected ID=${ID:-unknown}" >&2 - fi -fi - -sudo_cmd=() -if [[ "${EUID:-$(id -u)}" -ne 0 ]]; then - if command -v sudo >/dev/null 2>&1; then - sudo_cmd=(sudo) - else - echo "sudo is required when not running as root" >&2 - exit 2 - fi -fi - -minimal_packages=( - ca-certificates - curl - git - make - pkg-config - python3 - python3-dev - python3-pip - python3-venv -) - -system_filter_packages=( - cups - cups-client - cups-filters - cups-ppdc - ghostscript - poppler-utils - qpdf -) - -asan_build_packages=( - autoconf - automake - autopoint - build-essential - clang - dbus - gettext - gdb - libavahi-client-dev - libavahi-common-dev - libcups2-dev - libcupsimage2-dev - libdbus-1-dev - libexif-dev - libfontconfig1-dev - libfreetype6-dev - libglib2.0-dev - libijs-dev - libjpeg-dev - liblcms2-dev - libldap2-dev - libnss-mdns - libpam0g-dev - libpaper-dev - libpng-dev - libpoppler-cpp-dev - libpoppler-glib-dev - libqpdf-dev - libssl-dev - libtiff-dev - libtool - libxml2-dev - lld - llvm - mupdf-tools - zlib1g-dev -) - -afl_packages=( - afl++ - clang - lld - llvm -) - -packages=("${minimal_packages[@]}") -case "$mode" in - minimal) - ;; - system-filters) - packages+=("${system_filter_packages[@]}") - ;; - asan-build) - packages+=("${asan_build_packages[@]}") - ;; - afl) - packages+=("${afl_packages[@]}") - ;; - all) - packages+=("${system_filter_packages[@]}" "${asan_build_packages[@]}" "${afl_packages[@]}") - ;; - *) - echo "internal error: unknown mode $mode" >&2 - exit 2 - ;; -esac - -mapfile -t packages < <( - for package in "${packages[@]}"; do - echo "$package" - done | sort -u -) - -available_packages=() -missing_packages=() -for package in "${packages[@]}"; do - if apt-cache show "$package" >/dev/null 2>&1; then - available_packages+=("$package") - else - missing_packages+=("$package") - fi -done - -install_args=(install) -if [[ "$assume_yes" == "1" ]]; then - install_args+=(-y) -fi -install_args+=("${available_packages[@]}") - -echo "[info] mode: $mode" -echo "[info] packages: ${#available_packages[@]} available" -if [[ "${#missing_packages[@]}" -gt 0 ]]; then - echo "[warn] unavailable package names on this apt index: ${missing_packages[*]}" >&2 -fi - -if [[ "$dry_run" == "1" ]]; then - if [[ "$apt_update" == "1" ]]; then - print_shell_command "${sudo_cmd[@]}" apt-get update - fi - print_shell_command "${sudo_cmd[@]}" apt-get "${install_args[@]}" - exit 0 -fi - -if [[ "$apt_update" == "1" ]]; then - "${sudo_cmd[@]}" apt-get update -fi - -"${sudo_cmd[@]}" apt-get "${install_args[@]}" - -cat <<'EOF' - -[next] - python3 -m venv .venv - . .venv/bin/activate - python3 -m pip install -U pip - python3 -m pip install -e . -EOF diff --git a/parser-fuzzers/scripts/llvm_coverage_env.sh b/parser-fuzzers/scripts/llvm_coverage_env.sh deleted file mode 100755 index 4fdef3f..0000000 --- a/parser-fuzzers/scripts/llvm_coverage_env.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -cat <<'EOF' -# Source this before configuring an out-of-tree coverage build. -# Example: -# source scripts/llvm_coverage_env.sh -# cd /data/pre-gsoc/cups-filters -# make clean -# ./configure --disable-shared --enable-static --enable-individual-cups-filters -# make -j"$(nproc)" pwgtoraster rastertoescpx rastertopclx -export CC="${CC:-clang}" -export CXX="${CXX:-clang++}" -export CFLAGS="-O1 -g -fno-omit-frame-pointer -fsanitize=address -fprofile-instr-generate -fcoverage-mapping ${CFLAGS:-}" -export CXXFLAGS="-O1 -g -fno-omit-frame-pointer -fsanitize=address -fprofile-instr-generate -fcoverage-mapping ${CXXFLAGS:-}" -export LDFLAGS="-fsanitize=address -fprofile-instr-generate ${LDFLAGS:-}" -export ASAN_OPTIONS="${ASAN_OPTIONS:-abort_on_error=0:detect_leaks=0:symbolize=1:exitcode=86}" -EOF diff --git a/parser-fuzzers/scripts/merge_llvm_coverage.sh b/parser-fuzzers/scripts/merge_llvm_coverage.sh deleted file mode 100755 index 140bb34..0000000 --- a/parser-fuzzers/scripts/merge_llvm_coverage.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [[ $# -lt 3 ]]; then - echo "usage: $0 " >&2 - exit 2 -fi - -PROFRAW_DIR="$1" -BINARY="$2" -OUT_DIR="$3" -mkdir -p "$OUT_DIR" - -if ! command -v llvm-profdata >/dev/null 2>&1; then - echo "missing llvm-profdata in PATH" >&2 - exit 2 -fi -if ! command -v llvm-cov >/dev/null 2>&1; then - echo "missing llvm-cov in PATH" >&2 - exit 2 -fi - -llvm-profdata merge -sparse "$PROFRAW_DIR"/*.profraw -o "$OUT_DIR/coverage.profdata" -llvm-cov export "$BINARY" -instr-profile="$OUT_DIR/coverage.profdata" > "$OUT_DIR/coverage.json" -llvm-cov report "$BINARY" -instr-profile="$OUT_DIR/coverage.profdata" > "$OUT_DIR/coverage.txt" - -echo "$OUT_DIR/coverage.json" -echo "$OUT_DIR/coverage.txt" diff --git a/parser-fuzzers/scripts/prepare_afl_frontier_corpus.py b/parser-fuzzers/scripts/prepare_afl_frontier_corpus.py deleted file mode 100755 index 565aa13..0000000 --- a/parser-fuzzers/scripts/prepare_afl_frontier_corpus.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import shutil -from pathlib import Path - - -def main() -> int: - parser = argparse.ArgumentParser(description="prepare AFL++ seeds from retained multitarget documents") - parser.add_argument("--run-dir", required=True) - parser.add_argument("--target-id", required=True) - parser.add_argument("--output-dir", required=True) - parser.add_argument("--extension", default=".pwg") - parser.add_argument("--limit", type=int, default=256) - args = parser.parse_args() - - run_dir = Path(args.run_dir) - interesting = run_dir / "corpus" / "interesting" / args.target_id - if not interesting.exists(): - raise SystemExit(f"interesting corpus not found: {interesting}") - - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - copied = 0 - for case_dir in sorted(interesting.glob("case-*")): - if not case_dir.is_dir(): - continue - documents = sorted(case_dir.glob(f"document*{args.extension}")) - if not documents: - continue - source = documents[0] - destination = output_dir / f"{case_dir.name}{args.extension}" - shutil.copy2(source, destination) - copied += 1 - if copied >= args.limit: - break - - if copied == 0: - raise SystemExit(f"no document*{args.extension} seeds found under {interesting}") - print(f"prepared {copied} AFL++ seeds in {output_dir}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/parser-fuzzers/scripts/print_cups_filters_build_plan.sh b/parser-fuzzers/scripts/print_cups_filters_build_plan.sh deleted file mode 100755 index 0280516..0000000 --- a/parser-fuzzers/scripts/print_cups_filters_build_plan.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -ASAN_ROOT="${1:-$ROOT/work/openprinting-asan}" -PREFIX="${2:-$ASAN_ROOT/prefix}" -SRC_ROOT="${3:-$ASAN_ROOT/src}" -JOBS="${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)}" - -cat <&2 - exit 2 - ;; -esac - -mkdir -p "\$SRC_ROOT" "\$PREFIX" - -export CC=clang -export CXX=clang++ -export CFLAGS="-g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer" -export CXXFLAGS="-g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer" -export LDFLAGS="-fsanitize=address,undefined" -export PKG_CONFIG_PATH="\$PREFIX/lib/pkgconfig:\$PREFIX/lib64/pkgconfig:\${PKG_CONFIG_PATH:-}" - -clone_or_update() { - repo="\$1" - url="\$2" - if [[ -d "\$SRC_ROOT/\$repo/.git" ]]; then - git -C "\$SRC_ROOT/\$repo" pull --ff-only - else - git clone "\$url" "\$SRC_ROOT/\$repo" - fi -} - -clone_or_update libcupsfilters https://github.com/OpenPrinting/libcupsfilters.git -clone_or_update libppd https://github.com/OpenPrinting/libppd.git -clone_or_update cups-filters https://github.com/OpenPrinting/cups-filters.git - -cd "\$SRC_ROOT/libcupsfilters" -./autogen.sh -./configure --prefix="\$PREFIX" -make -j"\$JOBS" -make install - -cd "\$SRC_ROOT/libppd" -./autogen.sh -./configure --prefix="\$PREFIX" -make -j"\$JOBS" -make install - -cd "\$SRC_ROOT/cups-filters" -./autogen.sh -./configure --prefix="\$PREFIX" -make -j"\$JOBS" - -cd "\$SMT_FUZZER_ROOT" -export SMT_FUZZER_FILTER_ROOT="\$SRC_ROOT/cups-filters" -export SMT_FUZZER_LD_LIBRARY_PATH="\$SRC_ROOT/libcupsfilters/.libs:\$SRC_ROOT/libppd/.libs:\$PREFIX/lib:\$PREFIX/lib64" -export SMT_FUZZER_ASSUME_ASAN=1 -scripts/check_cups_filters_targets.sh "\$SMT_FUZZER_FILTER_ROOT" -scripts/run_asan_cups_filters_campaign.sh "\$ASAN_ROOT" 60 4 5 -EOF diff --git a/parser-fuzzers/scripts/replay_asan_filter.sh b/parser-fuzzers/scripts/replay_asan_filter.sh deleted file mode 100755 index 2c35427..0000000 --- a/parser-fuzzers/scripts/replay_asan_filter.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -FILTER="${1:-}" -CASE_DIR="${2:-}" -OUT_DIR="${3:-}" - -if [[ -z "$FILTER" || -z "$CASE_DIR" ]]; then - echo "usage: $0 [out-dir]" >&2 - exit 2 -fi - -if [[ "$FILTER" = /* ]]; then - FILTER_BIN="$FILTER" - FILTER_NAME="$(basename "$FILTER")" -else - FILTER_NAME="$FILTER" - FILTER_BIN="/data/pre-gsoc/cups-filters/$FILTER_NAME" -fi - -CASE_DIR="$(cd "$CASE_DIR" && pwd)" -OUT_DIR="${OUT_DIR:-$ROOT/work/asan-replay/$(basename "$CASE_DIR")-$FILTER_NAME}" -mkdir -p "$OUT_DIR" - -PPD_FILE="$CASE_DIR/candidate.ppd" -if [[ ! -f "$PPD_FILE" ]]; then - echo "missing PPD: $PPD_FILE" >&2 - exit 2 -fi - -DOC_FILE="" -for candidate in \ - "$CASE_DIR"/document.pwg \ - "$CASE_DIR"/document.ras \ - "$CASE_DIR"/document.pdf \ - "$CASE_DIR"/document.ps \ - "$CASE_DIR"/document.txt \ - "$CASE_DIR"/document.ppm \ - "$CASE_DIR"/document.pgm \ - "$CASE_DIR"/document.pbm \ - "$CASE_DIR"/document.png \ - "$CASE_DIR"/document.cmd \ - "$CASE_DIR"/document.bin; do - if [[ -f "$candidate" ]]; then - DOC_FILE="$candidate" - break - fi -done - -if [[ -z "$DOC_FILE" ]]; then - echo "missing document input in $CASE_DIR" >&2 - exit 2 -fi - -if [[ ! -x "$FILTER_BIN" ]]; then - echo "missing executable filter: $FILTER_BIN" >&2 - exit 2 -fi - -LIBPPD_ASAN="${LIBPPD_ASAN:-${SMT_FUZZER_LIBPPD_ASAN:-/data/pre-gsoc/libppd-origin-latest/.libs}}" -LIBCUPSFILTERS_ASAN="${LIBCUPSFILTERS_ASAN:-${SMT_FUZZER_LIBCUPSFILTERS_ASAN:-/data/pre-gsoc/libcupsfilters-master-asan/.libs}}" -if [[ ! -d "$LIBCUPSFILTERS_ASAN" ]]; then - LIBCUPSFILTERS_ASAN="/data/pre-gsoc/libcupsfilters/.libs" -fi -PDFIO_LIB="${PDFIO_LIB:-${SMT_FUZZER_PDFIO_LIB:-/data/pre-gsoc/env/pdfio-install/lib}}" -LD_LIBRARY_PATH_VALUE="$LIBPPD_ASAN:$LIBCUPSFILTERS_ASAN:$PDFIO_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -ASAN_OPTIONS_VALUE="${ASAN_OPTIONS:-abort_on_error=0:detect_leaks=0:symbolize=1:exitcode=86}" -JOB_OPTIONS="${SMT_FUZZER_REPLAY_JOB_OPTIONS:-}" - -if [[ -z "$JOB_OPTIONS" && -f "$CASE_DIR/meta.json" ]]; then - JOB_OPTIONS="$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1], encoding="utf-8")).get("job_options", ""))' "$CASE_DIR/meta.json")" -fi - -if [[ -z "$JOB_OPTIONS" && -f "$CASE_DIR/command.txt" ]]; then - JOB_OPTIONS="$(python3 -c 'import shlex, sys -parts = shlex.split(open(sys.argv[1], encoding="utf-8").read()) -while parts and "=" in parts[0] and parts[0].split("=", 1)[0].replace("_", "").isalnum(): - parts.pop(0) -print(parts[-2] if len(parts) >= 6 else "")' "$CASE_DIR/command.txt")" -fi - -cat > "$OUT_DIR/replay.env" < "$OUT_DIR/stdout.txt" \ - 2> "$OUT_DIR/asan.txt" -status=$? -set -e - -{ - echo "status=$status" - echo "filter=$FILTER_BIN" - echo "ppd=$PPD_FILE" - echo "document=$DOC_FILE" - echo "job_options=$JOB_OPTIONS" - echo "stdout=$OUT_DIR/stdout.txt" - echo "asan=$OUT_DIR/asan.txt" -} > "$OUT_DIR/summary.txt" - -cat "$OUT_DIR/summary.txt" -exit "$status" diff --git a/parser-fuzzers/scripts/report_campaign_result.sh b/parser-fuzzers/scripts/report_campaign_result.sh deleted file mode 100755 index 539beef..0000000 --- a/parser-fuzzers/scripts/report_campaign_result.sh +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -RUN_DIR="${1:-}" -ASAN_ROOT="${2:-}" - -if [[ -z "$RUN_DIR" ]]; then - echo "usage: $0 [asan-root]" >&2 - exit 2 -fi - -python3 - "$RUN_DIR" "$ASAN_ROOT" <<'PY' -import json -import os -import re -import sys -from pathlib import Path - -run_dir = Path(sys.argv[1]) -asan_root = sys.argv[2] or "work/openprinting-asan" -summary_path = run_dir / "summary.concise.json" -dedup_path = run_dir / "crash_dedup.json" -COLOR_ENABLED = ( - os.environ.get("NO_COLOR") is None - and (sys.stdout.isatty() or os.environ.get("FORCE_COLOR") in {"1", "true", "yes"}) -) - -COLORS = { - "bold": "\033[1m", - "dim": "\033[2m", - "red": "\033[31m", - "green": "\033[32m", - "yellow": "\033[33m", - "cyan": "\033[36m", - "reset": "\033[0m", -} - - -def load_json(path): - if not path.exists(): - return {} - return json.loads(path.read_text(encoding="utf-8")) - - -def color(name, text): - if not COLOR_ENABLED: - return text - return f"{COLORS[name]}{text}{COLORS['reset']}" - - -def label(name, text=None): - text = text or name - palette = { - "ok": "green", - "crash": "red", - "asan": "red", - "warn": "yellow", - "triage": "yellow", - "files": "cyan", - "next": "cyan", - } - return color(palette.get(name, "bold"), f"[{text}]") - - -def metric(name, value, bad=False, warn=False): - rendered = f"{name}={value}" - if bad: - return color("red", rendered) - if warn: - return color("yellow", rendered) - return rendered - - -def infer_filter(command): - match = re.search(r"/(?:usr/)?(?:lib|libexec)/cups/filter/([A-Za-z0-9_.+-]+)", command) - if match: - return match.group(1) - parts = command.split() - for part in parts: - name = Path(part).name - if name in { - "rastertopclx", - "rastertoescpx", - "rastertops", - "pwgtoraster", - "pwgtopdf", - "pdftopdf", - "pdftops", - "pdftoraster", - "mupdftopwg", - "imagetoraster", - "imagetopdf", - "imagetops", - "texttopdf", - "texttotext", - "gstoraster", - "gstopdf", - "gstopxl", - "pwgtopclm", - "commandtoescpx", - "commandtopclx", - }: - return name - return "" - - -def read_stderr_for_cluster(cluster, rep): - stderr_path = cluster.get("representative_stderr", "") - candidates = [] - if stderr_path: - candidates.append(Path(stderr_path)) - if rep: - candidates.append(Path(rep) / "stderr.txt") - - for path in candidates: - if path.exists(): - return path, path.read_text(encoding="utf-8", errors="replace") - return None, "" - - -def sanitizer_excerpt(stderr_text, limit_frames=6): - if not stderr_text: - return [] - - lines = stderr_text.splitlines() - interesting = [] - seen = set() - keywords = ( - "ERROR: AddressSanitizer", - "SUMMARY: AddressSanitizer", - "AddressSanitizer:", - "UndefinedBehaviorSanitizer", - "runtime error:", - "Sanitizer CHECK failed", - ) - - for line in lines: - if any(keyword in line for keyword in keywords): - normalized = " ".join(line.strip().split()) - if normalized and normalized not in seen: - interesting.append(normalized) - seen.add(normalized) - - frames = [] - for line in lines: - stripped = line.strip() - if re.match(r"#\d+\s+", stripped): - frames.append(stripped) - if len(frames) >= limit_frames: - break - - excerpt = interesting[:4] + frames - return [line[:240] for line in excerpt] - - -summary = load_json(summary_path) -dedup = load_json(dedup_path) -crashes = int(dedup.get("crash_records", summary.get("crashes", 0) or 0)) -unique = int(dedup.get("unique_crashes", summary.get("unique_crashes", 0) or 0)) - -print() -print(color("bold", "[campaign-result]")) -print(f"run_dir: {run_dir}") -if summary: - print( - "counts: " - f"{metric('cases', summary.get('cases', 0))} " - f"{metric('reached', summary.get('reached', 0))} " - f"{metric('valid_ppds', summary.get('valid_ppds', 0))} " - f"{metric('crashes', summary.get('crashes', 0), bad=int(summary.get('crashes', 0) or 0) > 0)} " - f"{metric('timeouts', summary.get('timeouts', 0), warn=int(summary.get('timeouts', 0) or 0) > 0)} " - f"{metric('unique_runtime', summary.get('unique_crashes', 0), bad=int(summary.get('unique_crashes', 0) or 0) > 0)}" - ) - -if crashes == 0: - print(f"{label('ok')} no crash-classified cases in this run") - print(f"{label('next')} try a longer ASan run after build: scripts/run_asan_cups_filters_campaign.sh {asan_root} 300 4 5") - raise SystemExit(0) - -print(f"{label('crash')} crash-classified cases={crashes}, unique dedup signatures={unique}") -print(f"{label('files')} dedup report: {run_dir / 'crash_dedup.md'}") - -clusters = dedup.get("clusters") or [] -for index, cluster in enumerate(clusters[:3], start=1): - rep = cluster.get("representative_work_dir", "") - cmd = cluster.get("representative_command", "") - filter_name = infer_filter(cmd) - stderr_path, stderr_text = read_stderr_for_cluster(cluster, rep) - asan_lines = sanitizer_excerpt(stderr_text) - print() - print(color("bold", f"[cluster {index}]") + f" target={cluster.get('target_id', '')} count={cluster.get('count', 0)}") - print(f"signature: {color('red', cluster.get('signature', ''))}") - print(f"case: {rep}") - print(f"command: {cmd}") - if asan_lines: - print(f"{label('asan')} sanitizer excerpt from {stderr_path}:") - for line in asan_lines: - print(f" {color('red', line)}") - else: - print(f"{label('warn', 'asan')} no sanitizer report in representative stderr") - if rep: - print(f"ASan replay: scripts/replay_asan_filter.sh {filter_name} {rep}") - print(f"GDB triage: scripts/gdb_crash_filter.sh {filter_name} {rep}") - -print() -print(label("triage")) -print("System-filter crashes are reachability signals, not final issue reports.") -print(f"For issue-quality output, build isolated ASan first: scripts/print_cups_filters_build_plan.sh {asan_root}") -PY diff --git a/parser-fuzzers/scripts/run_afl.sh b/parser-fuzzers/scripts/run_afl.sh deleted file mode 100755 index 48968f3..0000000 --- a/parser-fuzzers/scripts/run_afl.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" - -if [[ "$#" -lt 3 ]]; then - echo "usage: $0 [--execute]" - exit 2 -fi - -target="$1" -config="$2" -binary="$3" -shift 3 - -export PYTHONPATH="$ROOT/src" -python3 -m parser_fuzzers.cli afl-run \ - --root "$ROOT" \ - --configs configs \ - --target "$target" \ - --config "$config" \ - --binary "$binary" \ - "$@" diff --git a/parser-fuzzers/scripts/run_afl_feedback_smt_round.sh b/parser-fuzzers/scripts/run_afl_feedback_smt_round.sh deleted file mode 100755 index 9489776..0000000 --- a/parser-fuzzers/scripts/run_afl_feedback_smt_round.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" - -feedback_profile="${1:-auto}" -duration_sec="${2:-300}" -workers="${3:-6}" -timeout_sec="${4:-5}" -config="${SMT_AFL_SMT_CONFIG:-configs/parser_targets_afl_pwg_feedback.yaml}" -work_root="${SMT_AFL_SMT_WORK_ROOT:-work/afl-smt-feedback}" -max_run_gb="${SMT_FUZZER_MAX_RUN_GB:-10}" -skip_probe_rate="${SMT_FUZZER_SKIP_PROBE_RATE:-0.02}" -expansion_level="${SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL:-2}" -summary_mode="${SMT_FUZZER_SUMMARY_MODE:-concise}" -capture_stdout="${SMT_AFL_SMT_CAPTURE_STDOUT:-${SMT_FUZZER_CAPTURE_STDOUT:-0}}" - -latest_file() { - local root_dir="$1" - local pattern="$2" - if [[ ! -d "$root_dir" ]]; then - return 1 - fi - find "$root_dir" -type f -name "$pattern" -printf '%T@ %p\n' \ - | sort -nr \ - | sed -n '1s/^[^ ]* //p' -} - -latest_run_dir_since() { - local root_dir="$1" - local marker="$2" - if [[ ! -d "$root_dir" ]]; then - return 1 - fi - find "$root_dir" -mindepth 1 -maxdepth 1 -type d -newer "$marker" -printf '%T@ %p\n' \ - | sort -nr \ - | sed -n '1s/^[^ ]* //p' -} - -if [[ "$feedback_profile" == "auto" ]]; then - feedback_profile="$(latest_file work/afl 'afl-feedback.json' || true)" -fi -if [[ -z "$feedback_profile" || ! -f "$feedback_profile" ]]; then - echo "AFL feedback profile not found. Pass a profile path or run scripts/run_afl_pwg_frontier.sh first." >&2 - exit 2 -fi - -echo "feedback_profile=$feedback_profile" >&2 -echo "config=$config" >&2 -echo "work_root=$work_root" >&2 -echo "duration_sec=$duration_sec" >&2 -echo "workers=$workers" >&2 -echo "timeout_sec=$timeout_sec" >&2 -echo "template_expansion_level=$expansion_level" >&2 -echo "capture_stdout=$capture_stdout" >&2 - -mkdir -p "$work_root" work/template-feedback -run_marker="work/.afl-smt-feedback-start-$$.marker" -: > "$run_marker" - -SMT_FUZZER_TEMPLATE_FEEDBACK="$feedback_profile" \ -SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL="$expansion_level" \ -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config "$config" \ - --work-root "$work_root" \ - --workers "$workers" \ - --timeout-sec "$timeout_sec" \ - --duration-sec "$duration_sec" \ - --max-run-gb "$max_run_gb" \ - --discovery-mode coverage \ - --scheduler novelty \ - --runtime-skip \ - --auto-skip-state \ - --auto-skip-root work \ - --skip-probe-rate "$skip_probe_rate" \ - --summary-mode "$summary_mode" \ - --prune-uninteresting \ - $(if [[ "$capture_stdout" != "1" && "$capture_stdout" != "true" ]]; then echo "--discard-stdout"; fi) - -run_dir="$(latest_run_dir_since "$work_root" "$run_marker" || true)" -if [[ -z "$run_dir" ]]; then - echo "could not locate new SMT feedback run directory" >&2 - exit 1 -fi - -next_profile="work/template-feedback/afl-smt-$(basename "$run_dir")-feedback.json" -PYTHONPATH=src python3 -m parser_fuzzers.cli build-template-feedback \ - --run-dir "$run_dir" \ - --output "$next_profile" \ - --max-cases-per-kind 256 - -echo "run_dir=$run_dir" -echo "next_profile=$next_profile" diff --git a/parser-fuzzers/scripts/run_afl_pwg_bundle.sh b/parser-fuzzers/scripts/run_afl_pwg_bundle.sh deleted file mode 100755 index 29deeb8..0000000 --- a/parser-fuzzers/scripts/run_afl_pwg_bundle.sh +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" - -template_run="${1:-work/template-real-afl/pwgtopdf-20260605-120508/template/20260605-120508}" -seconds="${2:-600}" -monitor_interval="${3:-300}" -target_id="${SMT_AFL_BUNDLE_TARGET_ID:-pwg_to_pdf_afl_feedback}" -seed_limit="${SMT_AFL_BUNDLE_SEED_LIMIT:-512}" -dynamic_profile="${SMT_AFL_BUNDLE_DYNAMIC_PROFILE:-}" -dynamic_token_limit="${SMT_AFL_BUNDLE_DYNAMIC_TOKEN_LIMIT:-512}" -dynamic_seed_limit="${SMT_AFL_BUNDLE_DYNAMIC_SEED_LIMIT:-64}" -use_cmplog="${SMT_AFL_BUNDLE_USE_CMPLOG:-auto}" -max_gb="${SMT_AFL_BUNDLE_MAX_GB:-0}" -timestamp="$(date +%Y%m%d-%H%M%S)" -campaign_dir="${SMT_AFL_BUNDLE_CAMPAIGN_DIR:-work/afl/pwg-bundle-${timestamp}}" -seed_dir="$campaign_dir/seeds" -out_dir="$campaign_dir/out" -harness="${SMT_AFL_BUNDLE_HARNESS:-work/afl/bin/pwg_bundle_harness}" -cmplog_harness="${SMT_AFL_BUNDLE_CMPLOG_HARNESS:-${harness}.cmplog}" -fallback_ppd="$campaign_dir/fallback.ppd" - -if [[ ! -x "$harness" ]]; then - scripts/build_afl_pwg_bundle_harness.sh "$harness" >/dev/null -fi -cmplog_args=() -if [[ "$use_cmplog" != "0" ]]; then - if [[ ! -x "$cmplog_harness" && "$use_cmplog" != "no-build" ]]; then - SMT_AFL_BUNDLE_CMPLOG=1 scripts/build_afl_pwg_bundle_harness.sh "$cmplog_harness" >/dev/null - fi - if [[ -x "$cmplog_harness" ]]; then - cmplog_args=(-c "$cmplog_harness") - fi -fi -source work/afl-install/afl-env.sh - -mkdir -p "$campaign_dir" -PYTHONPATH=src python3 -c "from pathlib import Path; from parser_fuzzers.ppd_templates import make_ppd; Path('$fallback_ppd').write_text(make_ppd('pwgtopdf_coverage_options', 0), encoding='utf-8')" - -scripts/export_pwg_bundle_seeds.py \ - --run-dir "$template_run" \ - --target-id "$target_id" \ - --output-dir "$seed_dir" \ - --limit "$seed_limit" >"$campaign_dir/bundle-seed-export.json" -if [[ -f "$seed_dir/bundle_seed_manifest.json" ]]; then - mv "$seed_dir/bundle_seed_manifest.json" "$campaign_dir/bundle_seed_manifest.json" -fi -if [[ -n "$dynamic_profile" ]]; then - if [[ ! -f "$dynamic_profile" ]]; then - echo "dynamic profile not found: $dynamic_profile" >&2 - exit 2 - fi - PYTHONPATH=src scripts/augment_pwg_bundle_seeds_from_dynamic_profile.py \ - --seed-dir "$seed_dir" \ - --profile "$dynamic_profile" \ - --limit "$dynamic_seed_limit" >"$campaign_dir/dynamic-seed-augment.json" -fi - -export AFL_NO_UI=1 -export AFL_SKIP_CPUFREQ=1 -export AFL_CRASH_EXITCODE=86 -export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 -export ASAN_OPTIONS="${ASAN_OPTIONS:-abort_on_error=1:detect_leaks=0:symbolize=0}" -export LD_LIBRARY_PATH="$SMT_AFL_LIBCUPSFILTERS_LIB:$SMT_AFL_LIBPPD_LIB:$SMT_AFL_PDFIO_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -export SMT_AFL_BUNDLE_FALLBACK_PPD="$fallback_ppd" - -dict_args=() -dict_path="dictionaries/pwg_bundle.dict" -if [[ -n "$dynamic_profile" ]]; then - dict_path="$campaign_dir/dynamic-pwg-bundle.dict" - PYTHONPATH=src scripts/dynamic_profile_to_afl_dict.py \ - --profile "$dynamic_profile" \ - --base-dictionary dictionaries/pwg_bundle.dict \ - --output "$dict_path" \ - --max-tokens "$dynamic_token_limit" >"$campaign_dir/dynamic-dict-export.json" -fi -if [[ -f "$dict_path" ]]; then - dict_args=(-x "$dict_path") -fi - -echo "campaign_dir=$campaign_dir" -echo "mode=pwg-bundle-afl" -echo "template_run=$template_run" -echo "seed_dir=$seed_dir" -echo "harness=$harness" -echo "cmplog=$([[ ${#cmplog_args[@]} -gt 0 ]] && echo "$cmplog_harness" || echo none)" -echo "dictionary=$([[ ${#dict_args[@]} -gt 0 ]] && echo "$dict_path" || echo none)" -echo "dynamic_profile=${dynamic_profile:-none}" -echo "duration_sec=$seconds" -echo "monitor_interval_sec=$monitor_interval" -echo "max_gb=$max_gb" -echo "fallback_ppd=$fallback_ppd" - -set +e -timeout --kill-after=10s "$seconds" \ -afl-fuzz \ - "${dict_args[@]}" \ - "${cmplog_args[@]}" \ - -t "${AFL_BUNDLE_TIMEOUT_MS:-5000}" \ - -m none \ - -i "$seed_dir" \ - -o "$out_dir" \ - -T "parser-fuzzers-pwg-bundle" \ - -- "$harness" @@ >"$campaign_dir/afl.log" 2>&1 & -afl_pid="$!" -set -e - -( - while true; do - sleep "$monitor_interval" - scripts/afl_stats_snapshot.py "$out_dir" --label "[bundle-afl]" - size_bytes="$(du -sb "$campaign_dir" 2>/dev/null | cut -f1 || echo 0)" - echo "[bundle-afl:disk] campaign_dir=$campaign_dir bytes=$size_bytes max_gb=$max_gb" - if [[ "$max_gb" != "0" ]]; then - max_bytes="$(python3 -c "print(int(float('$max_gb') * 1024 * 1024 * 1024))")" - if [[ "$size_bytes" =~ ^[0-9]+$ && "$size_bytes" -ge "$max_bytes" ]]; then - echo "[bundle-afl:disk] stopping afl-fuzz: campaign directory reached max_gb=$max_gb" - kill "$afl_pid" 2>/dev/null || true - break - fi - fi - done -) & -monitor_pid="$!" - -set +e -wait "$afl_pid" -status="$?" -set -e -kill "$monitor_pid" 2>/dev/null || true -wait "$monitor_pid" 2>/dev/null || true - -scripts/afl_stats_snapshot.py "$out_dir" --label "[bundle-afl:final]" - -set +e -PYTHONPATH=src scripts/import_afl_frontier_feedback.py \ - --afl-out "$out_dir" \ - --target-id "$target_id" \ - --output-run-dir "$campaign_dir/feedback-import" \ - --extension .pwg-bundle \ - --queue-limit 512 \ - --crash-limit 128 \ - --queue-mode new >"$campaign_dir/afl-import.json" 2>"$campaign_dir/afl-import.stderr" -import_status="$?" -PYTHONPATH=src python3 -m parser_fuzzers.cli summarize-run-metrics \ - --run-dir "$campaign_dir/feedback-import" \ - --afl-output-dir "$out_dir" \ - --output "$campaign_dir/standard-metrics.json" >"$campaign_dir/standard-metrics.stdout.json" 2>"$campaign_dir/standard-metrics.stderr" -metrics_status="$?" -set -e - -cat >"$campaign_dir/manifest.json" <&2 - exit 2 - ;; -esac -filter_binary="${SMT_AFL_DIRECT_FILTER_BINARY:-${AFL_DIRECT_FILTER_BINARY:-$filter_binary}}" - -campaign_dir="${SMT_AFL_CAMPAIGN_DIR:-work/afl/pwg-frontier-${target}-${timestamp}}" -seed_dir="$campaign_dir/seeds" -ppd_path="$campaign_dir/candidate.ppd" -run_log="$campaign_dir/afl.log" -import_run_dir="$campaign_dir/feedback-import" -feedback_profile="$campaign_dir/afl-feedback.json" -mkdir -p "$campaign_dir" - -if [[ -n "$external_seed_dir" ]]; then - if [[ ! -d "$external_seed_dir" ]]; then - echo "external seed dir not found: $external_seed_dir" >&2 - exit 2 - fi - mkdir -p "$seed_dir" - find "$external_seed_dir" -maxdepth 1 -type f -name '*.pwg' -exec cp -a {} "$seed_dir/" \; - seed_count="$(find "$seed_dir" -maxdepth 1 -type f | wc -l)" - if [[ "$seed_count" == "0" ]]; then - echo "external seed dir has no .pwg seeds: $external_seed_dir" >&2 - exit 2 - fi - echo "prepared $seed_count AFL++ seeds in $seed_dir from external_seed_dir=$external_seed_dir" -else - scripts/prepare_afl_frontier_corpus.py \ - --run-dir "$seed_run_dir" \ - --target-id "$target_id" \ - --output-dir "$seed_dir" \ - --extension .pwg \ - --limit "$seed_limit" -fi - -PYTHONPATH=src python3 -c "from pathlib import Path; from parser_fuzzers.ppd_templates import make_ppd; Path('$ppd_path').write_text(make_ppd('$ppd_kind', 0), encoding='utf-8')" - -lib_path="${SMT_AFL_DIRECT_LD_LIBRARY_PATH:-${AFL_DIRECT_LD_LIBRARY_PATH:-}}" -if [[ -z "$lib_path" ]]; then - lib_path="$(PYTHONPATH=src python3 -c "from parser_fuzzers.multitarget_runner import _local_filter_library_path; print(_local_filter_library_path())")" -fi - -export AFL_NO_UI=1 -export AFL_SKIP_CPUFREQ=1 -export AFL_CRASH_EXITCODE=86 -export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 -export ASAN_OPTIONS="${ASAN_OPTIONS:-abort_on_error=1:detect_leaks=0:symbolize=0}" -export SMT_AFL_DIRECT_FILTER_BINARY="$filter_binary" -export SMT_AFL_DIRECT_PPD="$ppd_path" -export SMT_AFL_DIRECT_JOB_OPTIONS="${SMT_AFL_DIRECT_JOB_OPTIONS:-${AFL_DIRECT_JOB_OPTIONS:-PageSize=Letter ColorModel=Gray PrintQuality=Normal MediaType=Plain}}" -export SMT_AFL_DIRECT_LD_LIBRARY_PATH="$lib_path" -export PPD="$ppd_path" -export CONTENT_TYPE="${SMT_AFL_DIRECT_CONTENT_TYPE:-${AFL_DIRECT_CONTENT_TYPE:-application/vnd.cups-pwg}}" -export FINAL_CONTENT_TYPE="${SMT_AFL_DIRECT_FINAL_CONTENT_TYPE:-${AFL_DIRECT_FINAL_CONTENT_TYPE:-application/pdf}}" -export PRINTER="${SMT_AFL_DIRECT_PRINTER:-${AFL_DIRECT_PRINTER:-parser-fuzzers}}" -export DEVICE_URI="${SMT_AFL_DIRECT_DEVICE_URI:-${AFL_DIRECT_DEVICE_URI:-file:/dev/null}}" -if [[ -n "$lib_path" ]]; then - export LD_LIBRARY_PATH="$lib_path${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -fi -direct_instrumented="${SMT_AFL_DIRECT_INSTRUMENTED:-${AFL_DIRECT_INSTRUMENTED:-0}}" -unset AFL_DIRECT_FILTER_BINARY AFL_DIRECT_PPD AFL_DIRECT_JOB_OPTIONS AFL_DIRECT_LD_LIBRARY_PATH AFL_DIRECT_INSTRUMENTED AFL_PWG_SEED_LIMIT AFL_PWG_SEED_RUN_DIR AFL_PWG_SEED_DIR - -mode_args=() -if [[ "$direct_instrumented" != "1" ]]; then - mode_args=(-n) -fi - -target_cmd=(scripts/afl_direct_filter_target.sh @@) -if [[ "$direct_instrumented" == "1" ]]; then - target_cmd=("$filter_binary" 1 afl afl 1 "$SMT_AFL_DIRECT_JOB_OPTIONS" @@) -fi - -dict_args=() -if [[ -f "$dict_path" ]]; then - dict_args=(-x "$dict_path") -fi - -cmplog_args=() -cmplog_binary="${SMT_AFL_DIRECT_CMPLOG_BINARY:-${AFL_DIRECT_CMPLOG_BINARY:-${filter_binary}.cmplog}}" -if [[ "$direct_instrumented" == "1" && "${SMT_AFL_USE_CMPLOG:-${AFL_USE_CMPLOG:-auto}}" != "0" && -x "$cmplog_binary" ]]; then - cmplog_args=(-c "$cmplog_binary") -fi - -echo "campaign_dir=$campaign_dir" -echo "seed_dir=$seed_dir" -echo "external_seed_dir=${external_seed_dir:-none}" -echo "target=$target" -echo "filter_binary=$filter_binary" -echo "mode=$([[ ${#mode_args[@]} -gt 0 ]] && echo dumb || echo instrumented)" -echo "target_cmd=${target_cmd[*]}" -echo "dictionary=${dict_path:-none}" -echo "cmplog=$([[ ${#cmplog_args[@]} -gt 0 ]] && echo "$cmplog_binary" || echo none)" -echo "duration_sec=$seconds" -echo "monitor_interval_sec=$monitor_interval" -echo "queue_import_mode=$queue_import_mode" -echo "log=$run_log" - -set +e -timeout --kill-after=10s "$seconds" \ -afl-fuzz \ - "${mode_args[@]}" \ - "${dict_args[@]}" \ - "${cmplog_args[@]}" \ - -t "${AFL_DIRECT_TIMEOUT_MS:-5000}" \ - -m none \ - -i "$seed_dir" \ - -o "$campaign_dir/out" \ - -T "parser-fuzzers-${target}" \ - -- "${target_cmd[@]}" >"$run_log" 2>&1 & -afl_pid="$!" -set -e - -( - while true; do - sleep "$monitor_interval" - scripts/afl_stats_snapshot.py "$campaign_dir/out" --label "[afl:$target]" - done -) & -monitor_pid="$!" - -set +e -wait "$afl_pid" -status="$?" -set -e -kill "$monitor_pid" 2>/dev/null || true -wait "$monitor_pid" 2>/dev/null || true - -scripts/afl_stats_snapshot.py "$campaign_dir/out" --label "[afl:$target:final]" - -set +e -PYTHONPATH=src scripts/import_afl_frontier_feedback.py \ - --afl-out "$campaign_dir/out" \ - --target-id "$target_id" \ - --output-run-dir "$import_run_dir" \ - --extension .pwg \ - --queue-limit "$queue_import_limit" \ - --crash-limit "$crash_import_limit" \ - --queue-mode "$queue_import_mode" >"$campaign_dir/afl-import.json" 2>"$campaign_dir/afl-import.stderr" -import_status="$?" -if [[ "$import_status" == "0" ]]; then - PYTHONPATH=src python3 -m parser_fuzzers.cli build-template-feedback \ - --run-dir "$import_run_dir" \ - --output "$feedback_profile" \ - --max-cases-per-kind 256 >"$campaign_dir/afl-feedback-build.json" 2>"$campaign_dir/afl-feedback-build.stderr" - feedback_status="$?" -else - feedback_status="$import_status" -fi -set -e - -metrics_status=0 -set +e -PYTHONPATH=src python3 -m parser_fuzzers.cli summarize-run-metrics \ - --run-dir "$import_run_dir" \ - --afl-output-dir "$campaign_dir/out" \ - --output "$campaign_dir/standard-metrics.json" >"$campaign_dir/standard-metrics.stdout.json" 2>"$campaign_dir/standard-metrics.stderr" -metrics_status="$?" -set -e - -echo "campaign_dir=$campaign_dir" -echo "import_run_dir=$import_run_dir" -echo "feedback_profile=$feedback_profile" -echo "afl_import_status=$import_status" -echo "afl_feedback_status=$feedback_status" -echo "metrics_status=$metrics_status" -echo "standard_metrics=$campaign_dir/standard-metrics.json" -echo "afl_exit_status=$status" -if [[ "$status" == "124" || "$status" == "137" ]]; then - echo "[note] outer timeout stopped AFL++ after preserving results" - exit 0 -fi -exit "$status" diff --git a/parser-fuzzers/scripts/run_arithmetic_explore.sh b/parser-fuzzers/scripts/run_arithmetic_explore.sh deleted file mode 100755 index e33b338..0000000 --- a/parser-fuzzers/scripts/run_arithmetic_explore.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -DURATION_SEC="${1:-600}" -WORKERS="${2:-4}" -TIMEOUT_SEC="${3:-5}" - -cd "$ROOT" -PYTHONPATH=src python3 -m parser_fuzzers.cli arithmetic-explore \ - --work-dir work/arithmetic-explore \ - --duration-sec "$DURATION_SEC" \ - --workers "$WORKERS" \ - --timeout-sec "$TIMEOUT_SEC" diff --git a/parser-fuzzers/scripts/run_asan_cups_filters_campaign.sh b/parser-fuzzers/scripts/run_asan_cups_filters_campaign.sh deleted file mode 100755 index 4c88a50..0000000 --- a/parser-fuzzers/scripts/run_asan_cups_filters_campaign.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -ASAN_ROOT="${1:-$ROOT/work/openprinting-asan}" -DURATION_SEC="${2:-60}" -WORKERS="${3:-4}" -TIMEOUT_SEC="${4:-5}" -CONFIG="${5:-configs/parser_targets_general.yaml}" - -PREFIX="$ASAN_ROOT/prefix" -SRC_ROOT="$ASAN_ROOT/src" -FILTER_ROOT="$SRC_ROOT/cups-filters" -LIB_PATHS="$SRC_ROOT/libcupsfilters/.libs:$SRC_ROOT/libppd/.libs:$PREFIX/lib:$PREFIX/lib64" - -case "$ASAN_ROOT" in - /|/usr|/usr/*|/usr/local|/usr/local/*|/opt|/opt/*) - echo "refusing non-isolated ASan root: $ASAN_ROOT" >&2 - exit 2 - ;; -esac - -if [[ ! -d "$FILTER_ROOT" ]]; then - echo "missing local ASan cups-filters tree: $FILTER_ROOT" >&2 - echo "print a build plan with: scripts/print_cups_filters_build_plan.sh $ASAN_ROOT" >&2 - exit 2 -fi - -cd "$ROOT" -export SMT_FUZZER_FILTER_ROOT="$FILTER_ROOT" -export SMT_FUZZER_LD_LIBRARY_PATH="$LIB_PATHS" -export SMT_FUZZER_ASSUME_ASAN=1 -export ASAN_OPTIONS="${ASAN_OPTIONS:-abort_on_error=0:detect_leaks=0:symbolize=1:exitcode=86}" -export PYTHONPATH="$ROOT/src" - -scripts/check_cups_filters_targets.sh "$SMT_FUZZER_FILTER_ROOT" -scripts/run_local_cups_filters_campaign.sh "$SMT_FUZZER_FILTER_ROOT" "$DURATION_SEC" "$WORKERS" "$TIMEOUT_SEC" "$CONFIG" diff --git a/parser-fuzzers/scripts/run_baseline_comparison.sh b/parser-fuzzers/scripts/run_baseline_comparison.sh deleted file mode 100755 index 5ad311a..0000000 --- a/parser-fuzzers/scripts/run_baseline_comparison.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" - -DURATION_SEC="${1:-60}" -WORKERS="${2:-4}" -TIMEOUT_SEC="${3:-5}" -CONFIG="${SMT_FUZZER_COMPARE_CONFIG:-work/parser_targets_cold_semantic_llvm.yaml}" -OSS_FUZZ_DIR="${SMT_FUZZER_OSS_FUZZ_DIR:-/data/pre-gsoc/oss-fuzz}" -WORK_ROOT="${SMT_FUZZER_COMPARE_WORK_ROOT:-work/baseline-comparison}" -MAX_RUN_GB="${SMT_FUZZER_COMPARE_MAX_GB:-10}" -OPTIMIZED_POLICY="${SMT_FUZZER_COMPARE_POLICY:-avoidance}" - -if [[ ! -f "$CONFIG" ]]; then - echo "missing config: $CONFIG" >&2 - echo "build coverage filters first, or set SMT_FUZZER_COMPARE_CONFIG=configs/parser_targets_cold_semantic.yaml" >&2 - exit 2 -fi - -LLVM_ARGS=() -if command -v llvm-profdata-18 >/dev/null 2>&1 || command -v llvm-profdata >/dev/null 2>&1; then - if command -v llvm-cov-18 >/dev/null 2>&1 || command -v llvm-cov >/dev/null 2>&1; then - LLVM_ARGS=(--enable-llvm-profiles --export-llvm-coverage) - fi -fi - -PYTHONPATH=src python3 -m parser_fuzzers.cli compare-baseline-metrics \ - --config "$CONFIG" \ - --work-root "$WORK_ROOT" \ - --oss-fuzz-dir "$OSS_FUZZ_DIR" \ - --duration-sec "$DURATION_SEC" \ - --workers "$WORKERS" \ - --timeout-sec "$TIMEOUT_SEC" \ - --max-run-gb "$MAX_RUN_GB" \ - --optimized-policy "$OPTIMIZED_POLICY" \ - "${LLVM_ARGS[@]}" diff --git a/parser-fuzzers/scripts/run_cold_semantic_campaign.sh b/parser-fuzzers/scripts/run_cold_semantic_campaign.sh deleted file mode 100755 index 137229a..0000000 --- a/parser-fuzzers/scripts/run_cold_semantic_campaign.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" - -DURATION="${1:-${SMT_FUZZER_COLD_DURATION:-1200}}" -WORKERS="${SMT_FUZZER_COLD_WORKERS:-10}" -TIMEOUT_SEC="${SMT_FUZZER_COLD_TIMEOUT_SEC:-5}" -MAX_RUN_GB="${SMT_FUZZER_MAX_RUN_GB:-6}" -MIN_FREE_GB="${SMT_FUZZER_MIN_FREE_GB:-40}" -WORK_ROOT="${SMT_FUZZER_COLD_WORK_ROOT:-work/cold-semantic-campaign}" -CONFIG="${SMT_FUZZER_COLD_CONFIG:-configs/parser_targets_cold_semantic.yaml}" - -free_gb() { - df -Pk /data | awk 'NR == 2 { printf "%d", $4 / 1024 / 1024 }' -} - -latest_file() { - local pattern="$1" - find work/template-feedback -maxdepth 1 -type f -name "$pattern" -printf '%T@ %p\n' 2>/dev/null \ - | sort -nr \ - | awk 'NR == 1 { print $2 }' -} - -preferred_feedback() { - local preferred="$1" - local fallback_pattern="$2" - if [[ -f "$preferred" ]]; then - echo "$preferred" - return 0 - fi - latest_file "$fallback_pattern" -} - -before_free="$(free_gb)" -if (( before_free < MIN_FREE_GB )); then - echo "stop: free space below threshold: ${before_free}G < ${MIN_FREE_GB}G" >&2 - exit 3 -fi - -mkdir -p "$WORK_ROOT" work/template-feedback - -template_feedback="${SMT_FUZZER_TEMPLATE_FEEDBACK:-$(preferred_feedback 'work/template-feedback/cold-semantic-feedback.json' '*-feedback.json')}" -output_feedback="${SMT_FUZZER_OUTPUT_FEEDBACK:-$(preferred_feedback 'work/template-feedback/cold-semantic-output-feedback.json' '*-output-feedback.json')}" - -echo "cold_semantic_campaign" -echo "config=$CONFIG" -echo "work_root=$WORK_ROOT" -echo "duration_sec=$DURATION" -echo "workers=$WORKERS" -echo "timeout_sec=$TIMEOUT_SEC" -echo "max_run_gb=$MAX_RUN_GB" -echo "min_free_gb=$MIN_FREE_GB" -echo "free_gb_before=$before_free" -echo "template_feedback=$template_feedback" -echo "output_feedback=$output_feedback" -echo "avoidance_probe_interval=${SMT_FUZZER_AVOIDANCE_PROBE_INTERVAL:-32}" -echo "avoidance_skip_probe_rate=${SMT_FUZZER_AVOIDANCE_SKIP_PROBE_RATE:-0.06}" -echo "avoidance_scheduler_penalty_cap=${SMT_FUZZER_AVOIDANCE_SCHEDULER_PENALTY_CAP:-2.0}" - -SMT_FUZZER_TEMPLATE_FEEDBACK="$template_feedback" \ -SMT_FUZZER_OUTPUT_FEEDBACK="$output_feedback" \ -SMT_FUZZER_STRUCTURE_MUTATOR=1 \ -SMT_FUZZER_AUTO_DIMENSIONS="${SMT_FUZZER_AUTO_DIMENSIONS:-1}" \ -SMT_FUZZER_AUTO_DIMENSION_BUDGET="${SMT_FUZZER_AUTO_DIMENSION_BUDGET:-64}" \ -SMT_FUZZER_CRASH_AVOIDANCE="${SMT_FUZZER_CRASH_AVOIDANCE:-1}" \ -SMT_FUZZER_CRASH_AVOIDANCE_STATE="${SMT_FUZZER_CRASH_AVOIDANCE_STATE:-auto}" \ -SMT_FUZZER_CRASH_AVOIDANCE_ROOT="${SMT_FUZZER_CRASH_AVOIDANCE_ROOT:-work}" \ -SMT_FUZZER_CRASH_AVOIDANCE_GENERALIZE="${SMT_FUZZER_CRASH_AVOIDANCE_GENERALIZE:-1}" \ -SMT_FUZZER_AVOIDANCE_PROBE_INTERVAL="${SMT_FUZZER_AVOIDANCE_PROBE_INTERVAL:-32}" \ -SMT_FUZZER_AVOIDANCE_SKIP_PROBE_RATE="${SMT_FUZZER_AVOIDANCE_SKIP_PROBE_RATE:-0.06}" \ -SMT_FUZZER_AVOIDANCE_SCHEDULER_PENALTY_CAP="${SMT_FUZZER_AVOIDANCE_SCHEDULER_PENALTY_CAP:-2.0}" \ -SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL="${SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL:-2}" \ -SMT_FUZZER_SKIP_WARM_TEMPLATE_CACHE="${SMT_FUZZER_SKIP_WARM_TEMPLATE_CACHE:-1}" \ -SMT_FUZZER_TEMPLATE_CYCLE_EPOCHS="${SMT_FUZZER_TEMPLATE_CYCLE_EPOCHS:-8}" \ -SMT_FUZZER_HAZARD_SKIP_AFTER="${SMT_FUZZER_HAZARD_SKIP_AFTER:-24}" \ -SMT_FUZZER_SEMANTIC_SKIP_AFTER="${SMT_FUZZER_SEMANTIC_SKIP_AFTER:-3}" \ -SMT_FUZZER_SKIP_SHORT_IMAGE_ABORTS="${SMT_FUZZER_SKIP_SHORT_IMAGE_ABORTS:-1}" \ -SMT_FUZZER_LOAD_LEGACY_SKIP_STATE="${SMT_FUZZER_LOAD_LEGACY_SKIP_STATE:-1}" \ -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config "$CONFIG" \ - --work-root "$WORK_ROOT" \ - --workers "$WORKERS" \ - --timeout-sec "$TIMEOUT_SEC" \ - --duration-sec "$DURATION" \ - --max-run-gb "$MAX_RUN_GB" \ - --discard-stdout \ - --discovery-mode coverage \ - --scheduler novelty \ - --min-target-share "${SMT_FUZZER_MIN_TARGET_SHARE:-0.03}" \ - --max-target-share "${SMT_FUZZER_MAX_TARGET_SHARE:-0.16}" \ - --runtime-skip \ - --auto-skip-state \ - --auto-skip-root work \ - --generalized-skip \ - --family-skip-after "${SMT_FUZZER_FAMILY_SKIP_AFTER:-24}" \ - --skip-probe-rate "${SMT_FUZZER_SKIP_PROBE_RATE:-0.03}" \ - --summary-mode concise \ - --prune-uninteresting - -run_dir="$(find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | awk 'NR == 1 { print $2 }')" -if [[ -n "$run_dir" ]]; then - PYTHONPATH=src python3 -m parser_fuzzers.cli dedup-crashes \ - --run-dir "$run_dir" \ - --output-json "$run_dir/dedup.json" \ - --output-md "$run_dir/dedup.md" - PYTHONPATH=src python3 -m parser_fuzzers.cli build-template-feedback \ - --run-dir "$run_dir" \ - --output "work/template-feedback/cold-semantic-feedback.json" \ - --max-cases-per-kind "${SMT_FUZZER_COLD_FEEDBACK_CASES:-192}" - PYTHONPATH=src python3 -m parser_fuzzers.cli build-output-feedback \ - --run-dir "$run_dir" \ - --output "work/template-feedback/cold-semantic-output-feedback.json" - echo "run_dir=$run_dir" - echo "dedup=$run_dir/dedup.md" - echo "template_feedback=work/template-feedback/cold-semantic-feedback.json" - echo "output_feedback=work/template-feedback/cold-semantic-output-feedback.json" -fi - -echo "free_gb_after=$(free_gb)" diff --git a/parser-fuzzers/scripts/run_coverage_discovery_campaign.sh b/parser-fuzzers/scripts/run_coverage_discovery_campaign.sh deleted file mode 100755 index a8e998a..0000000 --- a/parser-fuzzers/scripts/run_coverage_discovery_campaign.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -DURATION_SEC="${1:-300}" -WORKERS="${2:-4}" -TIMEOUT_SEC="${3:-5}" - -cd "$ROOT" -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config configs/parser_targets_general.yaml \ - --work-root work/coverage-discovery \ - --workers "$WORKERS" \ - --timeout-sec "$TIMEOUT_SEC" \ - --duration-sec "$DURATION_SEC" \ - --discard-stdout \ - --discovery-mode coverage diff --git a/parser-fuzzers/scripts/run_deep_coverage_campaign.sh b/parser-fuzzers/scripts/run_deep_coverage_campaign.sh deleted file mode 100755 index 376aa6d..0000000 --- a/parser-fuzzers/scripts/run_deep_coverage_campaign.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -DURATION_SEC="${1:-300}" -WORKERS="${2:-4}" -TIMEOUT_SEC="${3:-5}" -SEED_SKIP_STATE="${4:-}" -FAMILY_SKIP_AFTER="${5:-}" - -cd "$ROOT" -ARGS=( - --config configs/parser_targets_coverage.yaml - --work-root work/deep-coverage - --workers "$WORKERS" - --timeout-sec "$TIMEOUT_SEC" - --duration-sec "$DURATION_SEC" - --discard-stdout - --discovery-mode coverage - --scheduler novelty - --runtime-skip - --prune-uninteresting -) - -if [[ -n "$SEED_SKIP_STATE" ]]; then - ARGS+=(--seed-skip-state "$SEED_SKIP_STATE") -fi -if [[ -n "$FAMILY_SKIP_AFTER" ]]; then - ARGS+=(--generalized-skip --family-skip-after "$FAMILY_SKIP_AFTER") -fi - -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor "${ARGS[@]}" diff --git a/parser-fuzzers/scripts/run_explore_ppd_fuzz.sh b/parser-fuzzers/scripts/run_explore_ppd_fuzz.sh deleted file mode 100755 index d105acf..0000000 --- a/parser-fuzzers/scripts/run_explore_ppd_fuzz.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -CASES_PER_TARGET="${1:-}" -WORKERS="${2:-4}" - -cd "$ROOT" -cmd=( - python3 -m parser_fuzzers.cli multitarget-monitor - --config configs/parser_targets_explore.yaml - --work-root work/explore - --workers "$WORKERS" -) -if [[ -n "$CASES_PER_TARGET" ]]; then - cmd+=(--cases-per-target "$CASES_PER_TARGET") -fi - -PYTHONPATH=src "${cmd[@]}" diff --git a/parser-fuzzers/scripts/run_feedback_template_campaign.sh b/parser-fuzzers/scripts/run_feedback_template_campaign.sh deleted file mode 100755 index 7d56df4..0000000 --- a/parser-fuzzers/scripts/run_feedback_template_campaign.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -DURATION_SEC="${1:-1200}" -WORKERS="${2:-10}" -TIMEOUT_SEC="${3:-5}" -FEEDBACK_PROFILE="${4:-auto}" -SEED_SKIP_STATE="${5:-}" -FAMILY_SKIP_AFTER="${6:-12}" -MAX_RUN_GB="${SMT_FUZZER_MAX_RUN_GB:-10}" -SKIP_PROBE_RATE="${SMT_FUZZER_SKIP_PROBE_RATE:-0.01}" - -cd "$ROOT" - -latest_file() { - local root_dir="$1" - local pattern="$2" - if [[ ! -d "$root_dir" ]]; then - return 1 - fi - find "$root_dir" -type f -name "$pattern" -printf '%T@ %p\n' \ - | sort -nr \ - | sed -n '1s/^[^ ]* //p' -} - -latest_campaign_dir() { - local paths=() - [[ -d work/feedback-campaign ]] && paths+=(work/feedback-campaign) - [[ -d work/structural-campaign ]] && paths+=(work/structural-campaign) - [[ -d work/deep-coverage ]] && paths+=(work/deep-coverage) - [[ -d work/general-parser ]] && paths+=(work/general-parser) - [[ ${#paths[@]} -gt 0 ]] || return 1 - find "${paths[@]}" -maxdepth 2 -type f -name summary.concise.json -printf '%T@ %h\n' \ - | sort -nr \ - | sed -n '1s/^[^ ]* //p' -} - -if [[ "$FEEDBACK_PROFILE" == "auto" ]]; then - FEEDBACK_PROFILE="$(latest_file work/template-feedback '*-feedback.json' || true)" - if [[ -z "$FEEDBACK_PROFILE" ]]; then - RUN_DIR="$(latest_campaign_dir || true)" - if [[ -z "$RUN_DIR" ]]; then - echo "No feedback profile or previous campaign found. Run a structural/feedback campaign first." >&2 - exit 2 - fi - mkdir -p work/template-feedback - FEEDBACK_PROFILE="work/template-feedback/auto-$(basename "$RUN_DIR")-feedback.json" - PYTHONPATH=src python3 -m parser_fuzzers.cli build-template-feedback \ - --run-dir "$RUN_DIR" \ - --output "$FEEDBACK_PROFILE" \ - --max-cases-per-kind 160 >&2 - fi -fi - -echo "feedback_profile=$FEEDBACK_PROFILE" >&2 -echo "max_run_gb=$MAX_RUN_GB" >&2 -echo "skip_probe_rate=$SKIP_PROBE_RATE" >&2 - -ARGS=( - --config configs/parser_targets_feedback.yaml - --work-root work/feedback-campaign - --workers "$WORKERS" - --timeout-sec "$TIMEOUT_SEC" - --max-run-gb "$MAX_RUN_GB" - --duration-sec "$DURATION_SEC" - --discard-stdout - --discovery-mode coverage - --scheduler novelty - --runtime-skip - --generalized-skip - --family-skip-after "$FAMILY_SKIP_AFTER" - --skip-probe-rate "$SKIP_PROBE_RATE" - --prune-uninteresting -) - -if [[ -n "$SEED_SKIP_STATE" ]]; then - ARGS+=(--seed-skip-state "$SEED_SKIP_STATE") -else - ARGS+=(--auto-skip-state --auto-skip-root work) -fi - -SMT_FUZZER_TEMPLATE_FEEDBACK="$FEEDBACK_PROFILE" \ - PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor "${ARGS[@]}" diff --git a/parser-fuzzers/scripts/run_general_parser_campaign.sh b/parser-fuzzers/scripts/run_general_parser_campaign.sh deleted file mode 100755 index d47502a..0000000 --- a/parser-fuzzers/scripts/run_general_parser_campaign.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -DURATION_SEC="${1:-900}" -WORKERS="${2:-4}" -TIMEOUT_SEC="${3:-5}" -SEED_SKIP_STATE="${4:-}" -FAMILY_SKIP_AFTER="${5:-}" - -cd "$ROOT" -ARGS=( - --config configs/parser_targets_general.yaml - --work-root work/general-campaign - --workers "$WORKERS" - --timeout-sec "$TIMEOUT_SEC" - --duration-sec "$DURATION_SEC" - --discard-stdout - --discovery-mode coverage - --scheduler novelty - --runtime-skip - --prune-uninteresting -) - -if [[ -n "$SEED_SKIP_STATE" ]]; then - ARGS+=(--seed-skip-state "$SEED_SKIP_STATE") -fi -if [[ -n "$FAMILY_SKIP_AFTER" ]]; then - ARGS+=(--generalized-skip --family-skip-after "$FAMILY_SKIP_AFTER") -fi - -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor "${ARGS[@]}" diff --git a/parser-fuzzers/scripts/run_image_cycle_campaign.sh b/parser-fuzzers/scripts/run_image_cycle_campaign.sh deleted file mode 100755 index ab0f94c..0000000 --- a/parser-fuzzers/scripts/run_image_cycle_campaign.sh +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -TOTAL_SEC="${1:-1200}" -WORKERS="${2:-8}" -TIMEOUT_SEC="${3:-5}" -TARGETS_CSV="${4:-imagetops,imagetoraster,imagetopdf}" -IMAGE_PROFILE="${5:-auto}" - -SLICE_SEC="${SMT_FUZZER_IMAGE_CYCLE_SLICE_SEC:-420}" -MIN_SLICE_SEC="${SMT_FUZZER_IMAGE_CYCLE_MIN_SLICE_SEC:-60}" -STAGNATION_STOP_AFTER_SEC="${SMT_FUZZER_STAGNATION_STOP_AFTER_SEC:-180}" -MIN_FREE_GB="${SMT_FUZZER_MIN_FREE_GB:-20}" -CAMPAIGN_MAX_GB="${SMT_FUZZER_CAMPAIGN_MAX_GB:-30}" -SKIP_PROBE_RATE="${SMT_FUZZER_SKIP_PROBE_RATE:-0.01}" -CYCLE_EPOCHS="${SMT_FUZZER_IMAGE_CYCLE_EPOCHS:-${SMT_FUZZER_TEMPLATE_CYCLE_EPOCHS:-8}}" -ENABLE_LLVM_PROFILES="${SMT_FUZZER_ENABLE_LLVM_PROFILES:-0}" -HAZARD_SKIP_AFTER="${SMT_FUZZER_HAZARD_SKIP_AFTER:-24}" -PROFILE_POLICY="${SMT_FUZZER_IMAGE_PROFILE_POLICY:-clean-first}" -SEMANTIC_SKIP_AFTER="${SMT_FUZZER_SEMANTIC_SKIP_AFTER:-0}" -CRASH_SKIP_AFTER="${SMT_FUZZER_CRASH_SKIP_AFTER:-1}" -AUTO_SKIP_STATE="${SMT_FUZZER_AUTO_SKIP_STATE:-1}" -LOAD_LEGACY_SKIP_STATE="${SMT_FUZZER_LOAD_LEGACY_SKIP_STATE:-1}" -if [[ "$AUTO_SKIP_STATE" == "semantic" ]]; then - LOAD_LEGACY_SKIP_STATE="0" -fi - -cd "$ROOT" - -IFS=',' read -r -a TARGETS <<< "$TARGETS_CSV" -if [[ "${#TARGETS[@]}" -eq 0 ]]; then - echo "no targets configured" >&2 - exit 2 -fi - -STAMP="$(date +%Y-%m-%d)-image-cycle-$(date +%H%M%S)" -CAMPAIGN_DIR="findings/$STAMP" -mkdir -p "$CAMPAIGN_DIR" - -START_EPOCH="$(date +%s)" -END_EPOCH="$((START_EPOCH + TOTAL_SEC))" -ROUND=0 -PROFILE="$IMAGE_PROFILE" -OUTPUT_PROFILE="${SMT_FUZZER_OUTPUT_FEEDBACK:-auto}" -STOP_REASON="budget-complete" -CAMPAIGN_RUN_DIRS=() -FINALIZED=0 - -gb_to_kb() { - local gb="$1" - awk -v value="$gb" 'BEGIN { printf "%.0f", value * 1024 * 1024 }' -} - -gb_to_bytes() { - local gb="$1" - awk -v value="$gb" 'BEGIN { printf "%.0f", value * 1024 * 1024 * 1024 }' -} - -available_kb() { - df -Pk "$ROOT" | awk 'NR == 2 { print $4 }' -} - -campaign_bytes() { - local total=0 - local size=0 - local path="" - for path in "$CAMPAIGN_DIR" "${CAMPAIGN_RUN_DIRS[@]}"; do - if [[ -e "$path" ]]; then - size="$(du -sb "$path" 2>/dev/null | awk '{ print $1 }')" - total="$((total + ${size:-0}))" - fi - done - echo "$total" -} - -guard_disk() { - local min_free_kb - min_free_kb="$(gb_to_kb "$MIN_FREE_GB")" - if [[ "$min_free_kb" -gt 0 && "$(available_kb)" -lt "$min_free_kb" ]]; then - STOP_REASON="disk-free-low" - echo "stopping: free disk below ${MIN_FREE_GB}G" >&2 - return 1 - fi - - local max_bytes - max_bytes="$(gb_to_bytes "$CAMPAIGN_MAX_GB")" - if [[ "$max_bytes" -gt 0 && "$(campaign_bytes)" -ge "$max_bytes" ]]; then - STOP_REASON="campaign-size-limit" - echo "stopping: campaign size reached ${CAMPAIGN_MAX_GB}G" >&2 - return 1 - fi - return 0 -} - -finalize_campaign() { - if [[ "$FINALIZED" -eq 1 ]]; then - return - fi - FINALIZED=1 - FINISHED_EPOCH="$(date +%s)" - { - echo - echo "finished_sec: $((FINISHED_EPOCH - START_EPOCH))" - echo "stop_reason: $STOP_REASON" - echo "final_profile: ${PROFILE:-auto}" - echo "final_output_feedback: ${OUTPUT_PROFILE:-auto}" - } >> "$CAMPAIGN_DIR/README.md" - tar -czf "$CAMPAIGN_DIR.tar.gz" -C findings "$STAMP" - echo "campaign_dir=$CAMPAIGN_DIR" - echo "archive=$CAMPAIGN_DIR.tar.gz" - echo "rounds=$ROUND" - echo "stop_reason=$STOP_REASON" - echo "final_profile=${PROFILE:-auto}" - echo "final_output_feedback=${OUTPUT_PROFILE:-auto}" -} - -trap 'STOP_REASON="interrupted"; exit 130' INT TERM -trap finalize_campaign EXIT - -cat > "$CAMPAIGN_DIR/README.md" <&2 -echo "total_sec=$TOTAL_SEC" >&2 -echo "targets=$TARGETS_CSV" >&2 -echo "slice_sec=$SLICE_SEC" >&2 -echo "stagnation_stop_after_sec=$STAGNATION_STOP_AFTER_SEC" >&2 -echo "skip_probe_rate=$SKIP_PROBE_RATE" >&2 -echo "image_cycle_epochs=$CYCLE_EPOCHS" >&2 -echo "llvm_profiles=$ENABLE_LLVM_PROFILES" >&2 -echo "hazard_skip_after=$HAZARD_SKIP_AFTER" >&2 -echo "semantic_skip_after=$SEMANTIC_SKIP_AFTER" >&2 -echo "crash_skip_after=$CRASH_SKIP_AFTER" >&2 -echo "auto_skip_state=$AUTO_SKIP_STATE" >&2 -echo "load_legacy_skip_state=$LOAD_LEGACY_SKIP_STATE" >&2 -echo "profile_policy=$PROFILE_POLICY" >&2 -echo "output_feedback=${OUTPUT_PROFILE:-auto}" >&2 -echo "min_free_gb=$MIN_FREE_GB" >&2 -echo "campaign_max_gb=$CAMPAIGN_MAX_GB" >&2 - -while true; do - if ! guard_disk; then - break - fi - NOW="$(date +%s)" - REMAINING="$((END_EPOCH - NOW))" - if [[ "$REMAINING" -lt "$MIN_SLICE_SEC" ]]; then - STOP_REASON="budget-exhausted" - break - fi - - TARGET_INDEX="$((ROUND % ${#TARGETS[@]}))" - TARGET="${TARGETS[$TARGET_INDEX]}" - RUN_SEC="$SLICE_SEC" - if [[ "$RUN_SEC" -gt "$REMAINING" ]]; then - RUN_SEC="$REMAINING" - fi - - ROUND_ID="$(printf 'round-%02d-%s' "$((ROUND + 1))" "$TARGET")" - ROUND_LOG="$CAMPAIGN_DIR/$ROUND_ID.log" - echo "round=$((ROUND + 1)) target=$TARGET duration_sec=$RUN_SEC profile=${PROFILE:-auto} output_feedback=${OUTPUT_PROFILE:-auto}" >&2 - - set +e - SMT_FUZZER_OUTPUT_FEEDBACK="${OUTPUT_PROFILE:-auto}" \ - SMT_FUZZER_STAGNATION_STOP_AFTER_SEC="$STAGNATION_STOP_AFTER_SEC" \ - SMT_FUZZER_SKIP_PROBE_RATE="$SKIP_PROBE_RATE" \ - SMT_FUZZER_IMAGE_CYCLE_EPOCHS="$CYCLE_EPOCHS" \ - SMT_FUZZER_ENABLE_LLVM_PROFILES="$ENABLE_LLVM_PROFILES" \ - SMT_FUZZER_HAZARD_SKIP_AFTER="$HAZARD_SKIP_AFTER" \ - SMT_FUZZER_SEMANTIC_SKIP_AFTER="$SEMANTIC_SKIP_AFTER" \ - SMT_FUZZER_CRASH_SKIP_AFTER="$CRASH_SKIP_AFTER" \ - SMT_FUZZER_AUTO_SKIP_STATE="$AUTO_SKIP_STATE" \ - SMT_FUZZER_LOAD_LEGACY_SKIP_STATE="$LOAD_LEGACY_SKIP_STATE" \ - SMT_FUZZER_IMAGE_PROFILE_POLICY="$PROFILE_POLICY" \ - scripts/run_image_deep_campaign.sh "$TARGET" "$RUN_SEC" "$WORKERS" "$TIMEOUT_SEC" "$PROFILE" \ - >"$ROUND_LOG" 2>&1 - STATUS="$?" - set -e - - RUN_DIR="$(sed -n 's/^run_dir=//p' "$ROUND_LOG" | tail -n 1)" - ROUND_FINDINGS="$(sed -n 's/^findings_dir=//p' "$ROUND_LOG" | tail -n 1)" - ARCHIVE="$(sed -n 's/^archive=//p' "$ROUND_LOG" | tail -n 1)" - NEXT_PROFILE="$(sed -n 's/^next_profile=//p' "$ROUND_LOG" | tail -n 1)" - NEXT_OUTPUT_FEEDBACK="$(sed -n 's/^next_output_feedback=//p' "$ROUND_LOG" | tail -n 1)" - if [[ -n "$RUN_DIR" && -d "$RUN_DIR" ]]; then - CAMPAIGN_RUN_DIRS+=("$RUN_DIR") - fi - if [[ -n "$NEXT_PROFILE" && -f "$NEXT_PROFILE" ]]; then - PROFILE="$NEXT_PROFILE" - fi - if [[ -n "$NEXT_OUTPUT_FEEDBACK" && -f "$NEXT_OUTPUT_FEEDBACK" ]]; then - OUTPUT_PROFILE="$NEXT_OUTPUT_FEEDBACK" - fi - - { - echo "- $ROUND_ID" - echo " status: $STATUS" - echo " duration_sec: $RUN_SEC" - echo " profile: ${PROFILE:-auto}" - echo " output_feedback: ${OUTPUT_PROFILE:-auto}" - echo " run_dir: ${RUN_DIR:-unknown}" - echo " findings_dir: ${ROUND_FINDINGS:-unknown}" - echo " archive: ${ARCHIVE:-unknown}" - echo " log: $ROUND_LOG" - } >> "$CAMPAIGN_DIR/README.md" - - printf '{"round":%d,"target":"%s","status":%d,"duration_sec":%d,"run_dir":"%s","findings_dir":"%s","archive":"%s","next_profile":"%s","next_output_feedback":"%s","log":"%s"}\n' \ - "$((ROUND + 1))" "$TARGET" "$STATUS" "$RUN_SEC" \ - "${RUN_DIR:-}" "${ROUND_FINDINGS:-}" "${ARCHIVE:-}" "${NEXT_PROFILE:-}" "${NEXT_OUTPUT_FEEDBACK:-}" "$ROUND_LOG" \ - >> "$CAMPAIGN_DIR/rounds.jsonl" - - if [[ "$STATUS" -ne 0 ]]; then - STOP_REASON="round-failed" - echo "round failed: $ROUND_ID, see $ROUND_LOG" >&2 - break - fi - - ROUND="$((ROUND + 1))" -done diff --git a/parser-fuzzers/scripts/run_image_deep_campaign.sh b/parser-fuzzers/scripts/run_image_deep_campaign.sh deleted file mode 100755 index cda348b..0000000 --- a/parser-fuzzers/scripts/run_image_deep_campaign.sh +++ /dev/null @@ -1,310 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -TARGET="${1:-imagetops}" -DURATION_SEC="${2:-1200}" -WORKERS="${3:-6}" -TIMEOUT_SEC="${4:-5}" -IMAGE_PROFILE="${5:-auto}" - -EXPANSION_LEVEL="${SMT_FUZZER_IMAGE_EXPANSION_LEVEL:-${SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL:-2}}" -OUTPUT_FEEDBACK="${SMT_FUZZER_OUTPUT_FEEDBACK:-auto}" -STRUCTURE_MUTATOR="${SMT_FUZZER_STRUCTURE_MUTATOR:-1}" -MAX_RUN_GB="${SMT_FUZZER_MAX_RUN_GB:-10}" -SKIP_PROBE_RATE="${SMT_FUZZER_SKIP_PROBE_RATE:-0.01}" -VALID_BIAS="${SMT_FUZZER_IMAGE_VALID_BIAS:-1}" -SHORT_PAYLOAD_EVERY="${SMT_FUZZER_IMAGE_SHORT_PAYLOAD_EVERY:-0}" -SKIP_SHORT_ABORTS="${SMT_FUZZER_SKIP_SHORT_IMAGE_ABORTS:-1}" -SUMMARY_MODE="${SMT_FUZZER_SUMMARY_MODE:-concise}" -CAPTURE_STDOUT="${SMT_FUZZER_CAPTURE_STDOUT:-0}" -GENERALIZED_SKIP="${SMT_FUZZER_GENERALIZED_SKIP:-0}" -SKIP_ONLY_STOP_AFTER="${SMT_FUZZER_SKIP_ONLY_STOP_AFTER:-20000}" -STAGNATION_STOP_AFTER_SEC="${SMT_FUZZER_STAGNATION_STOP_AFTER_SEC:-300}" -CYCLE_EPOCHS="${SMT_FUZZER_IMAGE_CYCLE_EPOCHS:-${SMT_FUZZER_TEMPLATE_CYCLE_EPOCHS:-8}}" -ENABLE_LLVM_PROFILES="${SMT_FUZZER_ENABLE_LLVM_PROFILES:-0}" -HAZARD_SKIP_AFTER="${SMT_FUZZER_HAZARD_SKIP_AFTER:-24}" -PROFILE_POLICY="${SMT_FUZZER_IMAGE_PROFILE_POLICY:-clean-first}" -SEMANTIC_SKIP_AFTER="${SMT_FUZZER_SEMANTIC_SKIP_AFTER:-0}" -CRASH_SKIP_AFTER="${SMT_FUZZER_CRASH_SKIP_AFTER:-1}" -AUTO_SKIP_STATE="${SMT_FUZZER_AUTO_SKIP_STATE:-1}" -LOAD_LEGACY_SKIP_STATE="${SMT_FUZZER_LOAD_LEGACY_SKIP_STATE:-1}" -if [[ "$AUTO_SKIP_STATE" == "semantic" ]]; then - LOAD_LEGACY_SKIP_STATE="0" -fi - -cd "$ROOT" - -case "$TARGET" in - imagetops) - CONFIG="configs/parser_targets_image_imagetops_feedback.yaml" - WORK_ROOT="work/image-deep-imagetops" - TARGET_ID="image_to_imagetops_feedback" - ;; - imagetoraster) - CONFIG="configs/parser_targets_image_imagetoraster_feedback.yaml" - WORK_ROOT="work/image-deep-imagetoraster" - TARGET_ID="image_to_imagetoraster_feedback" - ;; - imagetopdf) - CONFIG="configs/parser_targets_image_imagetopdf_feedback.yaml" - WORK_ROOT="work/image-deep-imagetopdf" - TARGET_ID="image_to_imagetopdf_feedback" - ;; - all) - CONFIG="configs/parser_targets_image_feedback.yaml" - WORK_ROOT="work/image-deep-all" - TARGET_ID="image_to_imagetoraster_feedback" - ;; - *) - echo "unknown target: $TARGET" >&2 - echo "usage: $0 [imagetops|imagetoraster|imagetopdf|all] [duration_sec] [workers] [timeout_sec] [image_profile|auto]" >&2 - exit 2 - ;; -esac - -latest_file() { - local root_dir="$1" - local pattern="$2" - if [[ ! -d "$root_dir" ]]; then - return 1 - fi - find "$root_dir" -type f -name "$pattern" -printf '%T@ %p\n' \ - | sort -nr \ - | sed -n '1s/^[^ ]* //p' -} - -latest_run_dir() { - local root_dir="$1" - if [[ ! -d "$root_dir" ]]; then - return 1 - fi - find "$root_dir" -maxdepth 2 -type f \( -name summary.concise.json -o -name timeline.jsonl \) -printf '%T@ %h\n' \ - | sort -nr \ - | sed -n '1s/^[^ ]* //p' -} - -latest_run_dir_since() { - local root_dir="$1" - local marker="$2" - if [[ ! -d "$root_dir" ]]; then - return 1 - fi - find "$root_dir" -mindepth 1 -maxdepth 1 -type d -newer "$marker" -printf '%T@ %p\n' \ - | sort -nr \ - | sed -n '1s/^[^ ]* //p' -} - -latest_source_run_dir() { - local paths=() - [[ -d work/image-deep-imagetops ]] && paths+=(work/image-deep-imagetops) - [[ -d work/image-deep-imagetoraster ]] && paths+=(work/image-deep-imagetoraster) - [[ -d work/image-deep-imagetopdf ]] && paths+=(work/image-deep-imagetopdf) - [[ -d work/image-feedback-campaign ]] && paths+=(work/image-feedback-campaign) - [[ ${#paths[@]} -gt 0 ]] || return 1 - find "${paths[@]}" -maxdepth 2 -type f -name summary.concise.json -printf '%T@ %h\n' \ - | sort -nr \ - | sed -n '1s/^[^ ]* //p' -} - -if [[ "$IMAGE_PROFILE" == "auto" ]]; then - if [[ "$PROFILE_POLICY" == "clean-first" ]]; then - IMAGE_PROFILE="$(latest_file work/template-feedback "*${TARGET}*-feedback.json" || true)" - if [[ -z "$IMAGE_PROFILE" ]]; then - IMAGE_PROFILE="$(latest_file work/template-feedback '*imagetopdf*-feedback.json' || true)" - fi - if [[ -z "$IMAGE_PROFILE" ]]; then - IMAGE_PROFILE="$(latest_file work/template-feedback '*imagetoraster*-feedback.json' || true)" - fi - fi - if [[ -z "$IMAGE_PROFILE" ]]; then - IMAGE_PROFILE="$(latest_file work/template-feedback '*image*-feedback.json' || true)" - fi - if [[ -z "$IMAGE_PROFILE" ]]; then - RUN_DIR="$(latest_source_run_dir || true)" - if [[ -n "$RUN_DIR" ]]; then - mkdir -p work/template-feedback - IMAGE_PROFILE="work/template-feedback/auto-image-$(basename "$RUN_DIR")-feedback.json" - PYTHONPATH=src python3 -m parser_fuzzers.cli build-template-feedback \ - --run-dir "$RUN_DIR" \ - --output "$IMAGE_PROFILE" \ - --max-cases-per-kind 256 >&2 - fi - fi -fi - -if [[ "$OUTPUT_FEEDBACK" == "auto" ]]; then - OUTPUT_FEEDBACK="$(latest_file work/template-feedback "*${TARGET}*-output-feedback.json" || true)" - if [[ -z "$OUTPUT_FEEDBACK" ]]; then - OUTPUT_FEEDBACK="$(latest_file work/template-feedback '*image-deep-*-output-feedback.json' || true)" - fi -fi - -echo "target=$TARGET" >&2 -echo "target_id=$TARGET_ID" >&2 -echo "config=$CONFIG" >&2 -echo "work_root=$WORK_ROOT" >&2 -echo "image_profile=${IMAGE_PROFILE:-synthetic}" >&2 -echo "output_feedback=${OUTPUT_FEEDBACK:-none}" >&2 -echo "structure_mutator=$STRUCTURE_MUTATOR" >&2 -echo "image_expansion_level=$EXPANSION_LEVEL" >&2 -echo "image_valid_bias=$VALID_BIAS" >&2 -echo "image_short_payload_every=$SHORT_PAYLOAD_EVERY" >&2 -echo "skip_short_image_aborts=$SKIP_SHORT_ABORTS" >&2 -echo "skip_probe_rate=$SKIP_PROBE_RATE" >&2 -echo "skip_only_stop_after=$SKIP_ONLY_STOP_AFTER" >&2 -echo "stagnation_stop_after_sec=$STAGNATION_STOP_AFTER_SEC" >&2 -echo "image_cycle_epochs=$CYCLE_EPOCHS" >&2 -echo "llvm_profiles=$ENABLE_LLVM_PROFILES" >&2 -echo "hazard_skip_after=$HAZARD_SKIP_AFTER" >&2 -echo "semantic_skip_after=$SEMANTIC_SKIP_AFTER" >&2 -echo "crash_skip_after=$CRASH_SKIP_AFTER" >&2 -echo "auto_skip_state=$AUTO_SKIP_STATE" >&2 -echo "load_legacy_skip_state=$LOAD_LEGACY_SKIP_STATE" >&2 -echo "profile_policy=$PROFILE_POLICY" >&2 -echo "summary_mode=$SUMMARY_MODE" >&2 -echo "capture_stdout=$CAPTURE_STDOUT" >&2 -echo "generalized_skip=$GENERALIZED_SKIP" >&2 - -ARGS=( - --config "$CONFIG" - --work-root "$WORK_ROOT" - --workers "$WORKERS" - --timeout-sec "$TIMEOUT_SEC" - --max-run-gb "$MAX_RUN_GB" - --duration-sec "$DURATION_SEC" - --discovery-mode coverage - --scheduler novelty - --runtime-skip - --crash-skip-after "$CRASH_SKIP_AFTER" - --auto-skip-root work - --skip-probe-rate "$SKIP_PROBE_RATE" - --skip-only-stop-after "$SKIP_ONLY_STOP_AFTER" - --stagnation-stop-after-sec "$STAGNATION_STOP_AFTER_SEC" - --summary-mode "$SUMMARY_MODE" - --prune-uninteresting -) - -if [[ "$CAPTURE_STDOUT" != "1" && "$CAPTURE_STDOUT" != "true" ]]; then - ARGS+=(--discard-stdout) -fi - -if [[ "$AUTO_SKIP_STATE" == "1" || "$AUTO_SKIP_STATE" == "true" || "$AUTO_SKIP_STATE" == "semantic" ]]; then - ARGS+=(--auto-skip-state) -fi - -if [[ "$GENERALIZED_SKIP" == "1" || "$GENERALIZED_SKIP" == "true" ]]; then - ARGS+=(--generalized-skip --family-skip-after "${SMT_FUZZER_FAMILY_SKIP_AFTER:-12}") -fi - -mkdir -p work -RUN_MARKER="work/.image-deep-${TARGET}-start-$$.marker" -: > "$RUN_MARKER" - -if [[ -n "$IMAGE_PROFILE" && -f "$IMAGE_PROFILE" ]]; then - SMT_FUZZER_IMAGE_FEEDBACK="$IMAGE_PROFILE" \ - SMT_FUZZER_OUTPUT_FEEDBACK="${OUTPUT_FEEDBACK:-}" \ - SMT_FUZZER_STRUCTURE_MUTATOR="$STRUCTURE_MUTATOR" \ - SMT_FUZZER_TARGET_ID="$TARGET_ID" \ - SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL="$EXPANSION_LEVEL" \ - SMT_FUZZER_IMAGE_CYCLE_EPOCHS="$CYCLE_EPOCHS" \ - SMT_FUZZER_IMAGE_VALID_BIAS="$VALID_BIAS" \ - SMT_FUZZER_IMAGE_SHORT_PAYLOAD_EVERY="$SHORT_PAYLOAD_EVERY" \ - SMT_FUZZER_SKIP_SHORT_IMAGE_ABORTS="$SKIP_SHORT_ABORTS" \ - SMT_FUZZER_ENABLE_LLVM_PROFILES="$ENABLE_LLVM_PROFILES" \ - SMT_FUZZER_HAZARD_SKIP_AFTER="$HAZARD_SKIP_AFTER" \ - SMT_FUZZER_SEMANTIC_SKIP_AFTER="$SEMANTIC_SKIP_AFTER" \ - SMT_FUZZER_LOAD_LEGACY_SKIP_STATE="$LOAD_LEGACY_SKIP_STATE" \ - PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor "${ARGS[@]}" -else - SMT_FUZZER_OUTPUT_FEEDBACK="${OUTPUT_FEEDBACK:-}" \ - SMT_FUZZER_STRUCTURE_MUTATOR="$STRUCTURE_MUTATOR" \ - SMT_FUZZER_TARGET_ID="$TARGET_ID" \ - SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL="$EXPANSION_LEVEL" \ - SMT_FUZZER_IMAGE_CYCLE_EPOCHS="$CYCLE_EPOCHS" \ - SMT_FUZZER_IMAGE_VALID_BIAS="$VALID_BIAS" \ - SMT_FUZZER_IMAGE_SHORT_PAYLOAD_EVERY="$SHORT_PAYLOAD_EVERY" \ - SMT_FUZZER_SKIP_SHORT_IMAGE_ABORTS="$SKIP_SHORT_ABORTS" \ - SMT_FUZZER_ENABLE_LLVM_PROFILES="$ENABLE_LLVM_PROFILES" \ - SMT_FUZZER_HAZARD_SKIP_AFTER="$HAZARD_SKIP_AFTER" \ - SMT_FUZZER_SEMANTIC_SKIP_AFTER="$SEMANTIC_SKIP_AFTER" \ - SMT_FUZZER_LOAD_LEGACY_SKIP_STATE="$LOAD_LEGACY_SKIP_STATE" \ - PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor "${ARGS[@]}" -fi - -RUN_DIR="$(latest_run_dir_since "$WORK_ROOT" "$RUN_MARKER" || true)" -if [[ -z "$RUN_DIR" ]]; then - RUN_DIR="$(latest_run_dir "$WORK_ROOT")" -fi -if [[ ! -f "$RUN_DIR/summary.concise.json" ]]; then - PYTHONPATH=src python3 -m parser_fuzzers.cli recover-run-summary --run-dir "$RUN_DIR" >&2 -fi -STAMP="$(date +%Y-%m-%d)-image-deep-${TARGET}-$(date +%H%M%S)" -FINDINGS_DIR="findings/$STAMP" -mkdir -p "$FINDINGS_DIR" - -PYTHONPATH=src python3 -m parser_fuzzers.cli dedup-crashes \ - --run-dir "$RUN_DIR" \ - --output-json "$FINDINGS_DIR/dedup.json" \ - --output-md "$FINDINGS_DIR/dedup.md" - -mkdir -p work/template-feedback -NEXT_PROFILE="work/template-feedback/image-deep-${TARGET}-$(basename "$RUN_DIR")-feedback.json" -PYTHONPATH=src python3 -m parser_fuzzers.cli build-template-feedback \ - --run-dir "$RUN_DIR" \ - --output "$NEXT_PROFILE" \ - --max-cases-per-kind 256 -NEXT_OUTPUT_FEEDBACK="work/template-feedback/image-deep-${TARGET}-$(basename "$RUN_DIR")-output-feedback.json" -PYTHONPATH=src python3 -m parser_fuzzers.cli build-output-feedback \ - --run-dir "$RUN_DIR" \ - --output "$NEXT_OUTPUT_FEEDBACK" - -cp "$RUN_DIR/summary.concise.json" "$FINDINGS_DIR/summary.concise.json" -cp "$RUN_DIR/summary.json" "$FINDINGS_DIR/summary.json" -cp "$RUN_DIR/run.log" "$FINDINGS_DIR/run.log" -cp "$RUN_DIR/run_manifest.json" "$FINDINGS_DIR/run_manifest.json" -if [[ -f "$RUN_DIR/discovery_state.json" ]]; then - cp "$RUN_DIR/discovery_state.json" "$FINDINGS_DIR/discovery_state.json" -fi -cp "$CONFIG" "$FINDINGS_DIR/target_config.yaml" -cp "$NEXT_PROFILE" "$FINDINGS_DIR/next-feedback.json" -cp "$NEXT_OUTPUT_FEEDBACK" "$FINDINGS_DIR/next-output-feedback.json" - -cat > "$FINDINGS_DIR/README.md" <&2 - fi - fi -fi - -echo "image_profile=${IMAGE_PROFILE:-synthetic}" >&2 -echo "image_expansion_level=$EXPANSION_LEVEL" >&2 -echo "max_run_gb=$MAX_RUN_GB" >&2 -echo "skip_probe_rate=$SKIP_PROBE_RATE" >&2 -echo "min_target_share=$MIN_TARGET_SHARE" >&2 -echo "max_target_share=$MAX_TARGET_SHARE" >&2 - -ARGS=( - --config configs/parser_targets_image_feedback.yaml - --work-root work/image-feedback-campaign - --workers "$WORKERS" - --timeout-sec "$TIMEOUT_SEC" - --max-run-gb "$MAX_RUN_GB" - --duration-sec "$DURATION_SEC" - --discard-stdout - --discovery-mode coverage - --scheduler novelty - --min-target-share "$MIN_TARGET_SHARE" - --max-target-share "$MAX_TARGET_SHARE" - --runtime-skip - --auto-skip-state - --auto-skip-root work - --generalized-skip - --family-skip-after 12 - --skip-probe-rate "$SKIP_PROBE_RATE" - --prune-uninteresting -) - -if [[ -n "$IMAGE_PROFILE" && -f "$IMAGE_PROFILE" ]]; then - SMT_FUZZER_IMAGE_FEEDBACK="$IMAGE_PROFILE" \ - SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL="$EXPANSION_LEVEL" \ - PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor "${ARGS[@]}" -else - SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL="$EXPANSION_LEVEL" \ - PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor "${ARGS[@]}" -fi diff --git a/parser-fuzzers/scripts/run_local_cups_filters_campaign.sh b/parser-fuzzers/scripts/run_local_cups_filters_campaign.sh deleted file mode 100755 index 5c74447..0000000 --- a/parser-fuzzers/scripts/run_local_cups_filters_campaign.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -FILTER_ROOT="${1:-${SMT_FUZZER_FILTER_ROOT:-/usr/lib/cups/filter}}" -DURATION_SEC="${2:-60}" -WORKERS="${3:-4}" -TIMEOUT_SEC="${4:-5}" -CONFIG="${5:-configs/parser_targets_general.yaml}" - -cd "$ROOT" -export SMT_FUZZER_FILTER_ROOT="$FILTER_ROOT" -export PYTHONPATH="$ROOT/src" - -scripts/check_cups_filters_targets.sh "$FILTER_ROOT" - -python3 -m parser_fuzzers.cli multitarget-monitor \ - --config "$CONFIG" \ - --work-root work/local-cups-filters \ - --filter-root "$FILTER_ROOT" \ - --workers "$WORKERS" \ - --timeout-sec "$TIMEOUT_SEC" \ - --duration-sec "$DURATION_SEC" \ - --discard-stdout \ - --discovery-mode coverage - -latest_run="$(ls -dt work/local-cups-filters/* 2>/dev/null | head -n 1 || true)" -if [[ -n "$latest_run" ]]; then - python3 -m parser_fuzzers.cli dedup-crashes --run-dir "$latest_run" - scripts/report_campaign_result.sh "$latest_run" "${SMT_FUZZER_ASAN_ROOT:-work/openprinting-asan}" -fi diff --git a/parser-fuzzers/scripts/run_multitarget_ppd_fuzz.sh b/parser-fuzzers/scripts/run_multitarget_ppd_fuzz.sh deleted file mode 100755 index 5bc4323..0000000 --- a/parser-fuzzers/scripts/run_multitarget_ppd_fuzz.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -CASES_PER_TARGET="${1:-1}" -WORKERS="${2:-4}" - -cd "$ROOT" -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config configs/parser_targets.yaml \ - --work-root work/multitarget \ - --workers "$WORKERS" \ - --cases-per-target "$CASES_PER_TARGET" diff --git a/parser-fuzzers/scripts/run_smoke.sh b/parser-fuzzers/scripts/run_smoke.sh deleted file mode 100755 index 95facf8..0000000 --- a/parser-fuzzers/scripts/run_smoke.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" - -export PYTHONPATH="$ROOT/src" - -WORK_DIR="work/smoke" -INPUT="$WORK_DIR/input.bin" -EVENT="$WORK_DIR/event.json" -RESULT="$WORK_DIR/result.json" -OUT_DIR="$WORK_DIR/corpus" - -mkdir -p "$WORK_DIR" "$OUT_DIR" - -python3 - "$INPUT" "$EVENT" <<'PY' -import json -import sys -from pathlib import Path - -from parser_fuzzers.hashing import sha256_file - -input_path = Path(sys.argv[1]) -event_path = Path(sys.argv[2]) -input_path.write_bytes(b"\x00SMT-FUZZER-SMOKE\n") -event = { - "target_id": "synthetic_eq_u8", - "input_path": str(input_path), - "input_sha256": sha256_file(input_path), - "offset": 0, - "width": 1, - "endianness": "little", - "signed": False, - "op": "eq", - "rhs": 65, - "description": "first byte must become ASCII A", -} -event_path.write_text(json.dumps(event, indent=2, sort_keys=True) + "\n", encoding="utf-8") -PY - -solve_args=() -if [[ "${SMT_FUZZER_STRICT_Z3:-0}" == "1" ]]; then - echo "[smoke] strict Z3 mode enabled" -else - solve_args+=(--allow-fallback) -fi - -python3 -m parser_fuzzers.cli solve-event --event "$EVENT" --output "$RESULT" "${solve_args[@]}" -patched_path="$(python3 -m parser_fuzzers.cli patch-input --result "$RESULT" --output-dir "$OUT_DIR")" - -python3 - "$EVENT" "$patched_path" "$RESULT" <<'PY' -import json -import sys -from pathlib import Path - -from parser_fuzzers.models import BranchEvent -from parser_fuzzers.solver import condition_holds, read_event_value - -event = BranchEvent.from_dict(json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))) -patched = Path(sys.argv[2]).read_bytes() -result = json.loads(Path(sys.argv[3]).read_text(encoding="utf-8")) -ok = condition_holds(event, read_event_value(event, patched)) -print(json.dumps({ - "ok": ok, - "event": sys.argv[1], - "result": sys.argv[3], - "patched_input": sys.argv[2], - "reason": result["reason"], -}, indent=2, sort_keys=True)) -raise SystemExit(0 if ok else 1) -PY - -python3 -m parser_fuzzers.cli validate --bugs bugs --configs configs --allow-missing-local-artifacts diff --git a/parser-fuzzers/scripts/run_structural_template_campaign.sh b/parser-fuzzers/scripts/run_structural_template_campaign.sh deleted file mode 100755 index 7549296..0000000 --- a/parser-fuzzers/scripts/run_structural_template_campaign.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -DURATION_SEC="${1:-1200}" -WORKERS="${2:-10}" -TIMEOUT_SEC="${3:-5}" -SEED_SKIP_STATE="${4:-}" -FAMILY_SKIP_AFTER="${5:-12}" - -cd "$ROOT" -ARGS=( - --config configs/parser_targets_structural.yaml - --work-root work/structural-campaign - --workers "$WORKERS" - --timeout-sec "$TIMEOUT_SEC" - --duration-sec "$DURATION_SEC" - --discard-stdout - --discovery-mode coverage - --scheduler novelty - --runtime-skip - --generalized-skip - --family-skip-after "$FAMILY_SKIP_AFTER" - --prune-uninteresting -) - -if [[ -n "$SEED_SKIP_STATE" ]]; then - ARGS+=(--seed-skip-state "$SEED_SKIP_STATE") -fi - -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor "${ARGS[@]}" diff --git a/parser-fuzzers/scripts/run_template_afl_loop.sh b/parser-fuzzers/scripts/run_template_afl_loop.sh deleted file mode 100755 index 4059ff1..0000000 --- a/parser-fuzzers/scripts/run_template_afl_loop.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" - -duration_sec="${1:-1200}" -target_id="${2:-pwg_to_pdf_afl_feedback}" -extension="${3:-.pwg}" -binary="${SMT_FUZZER_AFL_BINARY:-work/afl/bin/template_probe}" -config="${SMT_TEMPLATE_AFL_CONFIG:-configs/parser_targets_afl_pwg_feedback.yaml}" -work_root="${SMT_TEMPLATE_AFL_WORK_ROOT:-work/template-afl-loop}" -timeout_sec="${SMT_TEMPLATE_AFL_TIMEOUT_SEC:-5}" -workers="${SMT_TEMPLATE_AFL_WORKERS:-4}" -max_run_gb="${SMT_TEMPLATE_AFL_MAX_RUN_GB:-10}" -afl_target="${SMT_TEMPLATE_AFL_AFL_TARGET:-template_probe_pwg}" -afl_config="${SMT_TEMPLATE_AFL_AFL_CONFIG:-A1}" -filter_root="${SMT_TEMPLATE_FILTER_ROOT:-${SMT_FUZZER_TEMPLATE_FILTER_ROOT:-${SMT_FUZZER_FILTER_ROOT:-/data/pre-gsoc/cups-filters}}}" - -prompt_filter_root() { - local answer - if [[ "${SMT_TEMPLATE_SKIP_FILTER_PROMPT:-0}" == "1" ]]; then - return - fi - if [[ -t 0 ]]; then - printf 'Template runner filter root [%s]: ' "$filter_root" - read -r answer - if [[ -n "$answer" ]]; then - filter_root="$answer" - fi - fi -} - -check_filter_root() { - if [[ ! -d "$filter_root" ]]; then - echo "missing template filter root: $filter_root" >&2 - echo "set SMT_TEMPLATE_FILTER_ROOT or enter a directory containing cups-filters binaries" >&2 - exit 2 - fi -} - -if [[ ! -x "$binary" ]]; then - scripts/build_afl_template_probe.sh "$binary" >/dev/null -fi - -prompt_filter_root -check_filter_root - -stamp="$(date +%Y%m%d-%H%M%S)" -campaign_dir="$work_root/$stamp" -mkdir -p "$campaign_dir" - -template_sec=$((duration_sec / 4)) -afl_sec=$((duration_sec / 2)) -feedback_sec=$((duration_sec - template_sec - afl_sec)) -if (( template_sec < 60 )); then template_sec=60; fi -if (( afl_sec < 60 )); then afl_sec=60; fi -if (( feedback_sec < 60 )); then feedback_sec=60; fi - -echo "campaign_dir=$campaign_dir" -echo "duration_sec=$duration_sec" -echo "template_sec=$template_sec" -echo "afl_sec=$afl_sec" -echo "feedback_sec=$feedback_sec" -echo "target_id=$target_id" -echo "extension=$extension" -echo "binary=$binary" -echo "filter_root=$filter_root" - -template_root="$campaign_dir/template" -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config "$config" \ - --work-root "$template_root" \ - --filter-root "$filter_root" \ - --workers "$workers" \ - --timeout-sec "$timeout_sec" \ - --duration-sec "$template_sec" \ - --max-run-gb "$max_run_gb" \ - --discard-stdout \ - --discovery-mode coverage \ - --scheduler novelty \ - --runtime-skip \ - --summary-mode concise \ - --prune-uninteresting >"$campaign_dir/template.stdout.json" - -template_run="$(find "$template_root" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | sed -n '1s/^[^ ]* //p')" -if [[ -z "$template_run" ]]; then - echo "template run not found" >&2 - exit 1 -fi - -seed_dir="$campaign_dir/seeds" -PYTHONPATH=src python3 -m parser_fuzzers.cli export-template-seeds \ - --run-dir "$template_run" \ - --target-id "$target_id" \ - --extension "$extension" \ - --output-dir "$seed_dir" \ - --limit 512 >"$campaign_dir/seed-export.json" - -afl_out="$campaign_dir/afl-out" -PYTHONPATH=src python3 -m parser_fuzzers.cli afl-run \ - --target "$afl_target" \ - --config "$afl_config" \ - --binary "$binary" \ - --input-dir "$seed_dir" \ - --output-dir "$afl_out" \ - --duration-sec "$afl_sec" \ - --timeout-ms 1000 \ - --memory-mb 1024 \ - --execute >"$campaign_dir/afl-run.log" 2>&1 - -PYTHONPATH=src python3 -m parser_fuzzers.cli summarize-run-metrics \ - --run-dir "$campaign_dir" \ - --afl-output-dir "$afl_out" \ - --output "$campaign_dir/afl-standard-metrics.json" >"$campaign_dir/afl-standard-metrics.stdout.json" - -import_dir="$campaign_dir/afl-feedback-import" -feedback_profile="$campaign_dir/afl-feedback-profile.json" -PYTHONPATH=src scripts/import_afl_frontier_feedback.py \ - --afl-out "$afl_out" \ - --target-id "$target_id" \ - --output-run-dir "$import_dir" \ - --extension "$extension" \ - --queue-limit 512 \ - --crash-limit 128 \ - --queue-mode new >"$campaign_dir/afl-import.json" - -PYTHONPATH=src python3 -m parser_fuzzers.cli build-template-feedback \ - --run-dir "$import_dir" \ - --output "$feedback_profile" \ - --max-cases-per-kind 256 >"$campaign_dir/feedback-profile-build.json" - -feedback_root="$campaign_dir/feedback-template" -SMT_FUZZER_TEMPLATE_FEEDBACK="$feedback_profile" \ -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config "$config" \ - --work-root "$feedback_root" \ - --filter-root "$filter_root" \ - --workers "$workers" \ - --timeout-sec "$timeout_sec" \ - --duration-sec "$feedback_sec" \ - --max-run-gb "$max_run_gb" \ - --discard-stdout \ - --discovery-mode coverage \ - --scheduler novelty \ - --runtime-skip \ - --summary-mode concise \ - --prune-uninteresting >"$campaign_dir/feedback-template.stdout.json" - -feedback_run="$(find "$feedback_root" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | sed -n '1s/^[^ ]* //p')" - -cat >"$campaign_dir/loop_manifest.json" <"$campaign_dir/loop-standard-metrics.stdout.json" - -echo "campaign_dir=$campaign_dir" -echo "template_run=$template_run" -echo "seed_dir=$seed_dir" -echo "afl_out=$afl_out" -echo "feedback_profile=$feedback_profile" -echo "feedback_run=$feedback_run" -echo "loop_metrics=$campaign_dir/loop_standard_metrics.json" diff --git a/parser-fuzzers/scripts/run_template_multi_afl_filters.py b/parser-fuzzers/scripts/run_template_multi_afl_filters.py deleted file mode 100755 index be88d9a..0000000 --- a/parser-fuzzers/scripts/run_template_multi_afl_filters.py +++ /dev/null @@ -1,865 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import shutil -import subprocess -import struct -import sys -import time -import zlib -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import yaml - - -RASTER_DICT = "dictionaries/pwg_raster.dict" - - -@dataclass -class TargetPlan: - target_id: str - filter_binary: Path - input_mime: str - document_kind: str - ppd_path: Path - job_options: str - seed_dir: Path - out_dir: Path - log_path: Path - metrics_path: Path - dictionary: Path | None - seed_count: int - fallback_seed: bool - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Run standard AFL++ for every parser target using template-generated documents as seeds.", - ) - parser.add_argument("--template-run-dir", required=True, help="completed multitarget template/direct-filter run") - parser.add_argument("--config", required=True, help="parser target YAML matching the template run") - parser.add_argument("--output-root", required=True, help="AFL++ campaign output directory") - parser.add_argument("--duration-sec", type=int, default=1800) - parser.add_argument("--max-parallel", type=int, default=20) - parser.add_argument("--seed-limit", type=int, default=768) - parser.add_argument("--timeout-ms", type=int, default=5000) - parser.add_argument("--max-campaign-gb", type=float, default=25.0) - parser.add_argument("--min-free-gb", type=float, default=70.0) - parser.add_argument("--monitor-interval-sec", type=int, default=60) - parser.add_argument("--ld-library-path", default="") - parser.add_argument("--afl-fuzz", default="afl-fuzz") - parser.add_argument("--target-id", action="append", default=[], help="run only this target id; repeatable") - parser.add_argument("--include-crashing-seeds", action="store_true") - parser.add_argument("--no-preflight-seeds", action="store_false", dest="preflight_seeds") - parser.add_argument("--preflight-timeout-sec", type=float, default=3.0) - parser.add_argument("--upgrade-generated-seeds", type=int, default=0) - parser.add_argument("--template-expansion-level", type=int, default=3) - parser.add_argument("--template-cycle-epochs", type=int, default=4) - parser.add_argument("--drop-nonviable-seeds", action="store_true") - parser.set_defaults(preflight_seeds=True) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - root = Path.cwd() - template_run_dir = Path(args.template_run_dir) - config_path = Path(args.config) - output_root = Path(args.output_root) - output_root.mkdir(parents=True, exist_ok=True) - - if shutil.which(args.afl_fuzz) is None: - raise SystemExit(f"AFL++ fuzzer not found in PATH: {args.afl_fuzz}") - if not template_run_dir.is_dir(): - raise SystemExit(f"template run dir does not exist: {template_run_dir}") - - ld_library_path = args.ld_library_path or local_filter_library_path(root) - configure_template_upgrade_env(args) - plans = build_plans( - root=root, - config_path=config_path, - template_run_dir=template_run_dir, - output_root=output_root, - seed_limit=args.seed_limit, - include_crashing=args.include_crashing_seeds, - target_ids=set(args.target_id or []), - ld_library_path=ld_library_path, - preflight_seeds=args.preflight_seeds, - preflight_timeout_sec=args.preflight_timeout_sec, - upgrade_generated_seeds=args.upgrade_generated_seeds, - drop_nonviable_seeds=args.drop_nonviable_seeds or args.upgrade_generated_seeds > 0, - ) - plan_path = output_root / "plans.json" - plan_path.write_text( - json.dumps([plan_to_json(plan) for plan in plans], indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - - manifest = { - "template_run_dir": str(template_run_dir), - "config": str(config_path), - "output_root": str(output_root), - "duration_sec": args.duration_sec, - "max_parallel": args.max_parallel, - "seed_limit": args.seed_limit, - "timeout_ms": args.timeout_ms, - "max_campaign_gb": args.max_campaign_gb, - "min_free_gb": args.min_free_gb, - "ld_library_path": ld_library_path, - "targets": len(plans), - "plans": str(plan_path), - "upgrade_generated_seeds": args.upgrade_generated_seeds, - "template_expansion_level": args.template_expansion_level, - "template_cycle_epochs": args.template_cycle_epochs, - "drop_nonviable_seeds": bool(args.drop_nonviable_seeds or args.upgrade_generated_seeds > 0), - } - (output_root / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - print(json.dumps({"event": "prepared", **manifest}, sort_keys=True), flush=True) - return run_scheduler( - plans=plans, - args=args, - ld_library_path=ld_library_path, - output_root=output_root, - ) - - -def build_plans( - *, - root: Path, - config_path: Path, - template_run_dir: Path, - output_root: Path, - seed_limit: int, - include_crashing: bool, - target_ids: set[str], - ld_library_path: str, - preflight_seeds: bool, - preflight_timeout_sec: float, - upgrade_generated_seeds: int, - drop_nonviable_seeds: bool, -) -> list[TargetPlan]: - data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} - plans: list[TargetPlan] = [] - for target in data.get("targets", []) or []: - target_id = str(target["id"]) - if target_ids and target_id not in target_ids: - continue - target_dir = template_run_dir / target_id - if not target_dir.is_dir(): - print(json.dumps({"event": "skip-missing-template-target", "target_id": target_id}), flush=True) - continue - target_out = output_root / target_id - seed_dir = target_out / "seeds" - seed_dir.mkdir(parents=True, exist_ok=True) - selected = select_seed_cases(target_dir, seed_limit=seed_limit, include_crashing=include_crashing) - first_meta = first_case_meta(target_dir, selected) - ppd_path = target_out / "fixed.ppd" - write_fixed_ppd(ppd_path, first_meta, target) - job_options = str(first_meta.get("job_options") or default_job_options(target)) - seed_count, fallback_seed = export_documents(seed_dir, selected, target) - if seed_count == 0: - seed_count, fallback_seed = write_fallback_seed(seed_dir, target) - if upgrade_generated_seeds > 0: - generated = generate_upgraded_seeds( - root=root, - seed_dir=seed_dir, - target=target, - count=upgrade_generated_seeds, - ) - seed_count += generated - dictionary = dictionary_for_target(root, target) - plan = TargetPlan( - target_id=target_id, - filter_binary=Path(str(target["filter_binary"])), - input_mime=str(target.get("input_mime", "")), - document_kind=str(target.get("document_kind", "")), - ppd_path=ppd_path, - job_options=job_options, - seed_dir=seed_dir, - out_dir=target_out / "out", - log_path=target_out / "afl-run.log", - metrics_path=target_out / "standard-metrics.json", - dictionary=dictionary, - seed_count=seed_count, - fallback_seed=fallback_seed, - ) - if preflight_seeds: - plan.seed_count = ensure_viable_seed( - plan, - target, - ld_library_path=ld_library_path, - timeout_sec=preflight_timeout_sec, - drop_nonviable=drop_nonviable_seeds, - ) - plans.append(plan) - return plans - - -def select_seed_cases(target_dir: Path, *, seed_limit: int, include_crashing: bool) -> list[Path]: - cases = sorted(p for p in target_dir.iterdir() if p.is_dir() and p.name.startswith("case-")) - selected: list[Path] = [] - fallback: list[Path] = [] - for case_dir in cases: - doc = find_document(case_dir) - if doc is None: - continue - meta = read_json(case_dir / "meta.json") - crashed = bool(meta.get("crashed")) - timed_out = bool(meta.get("timed_out")) - if not crashed and not timed_out: - selected.append(case_dir) - elif include_crashing: - fallback.append(case_dir) - if seed_limit and len(selected) >= seed_limit: - break - if selected: - return selected[:seed_limit] if seed_limit else selected - if fallback: - return fallback[:seed_limit] if seed_limit else fallback - return [] - - -def first_case_meta(target_dir: Path, selected: list[Path]) -> dict[str, Any]: - candidates = selected or sorted(p for p in target_dir.iterdir() if p.is_dir() and p.name.startswith("case-")) - for case_dir in candidates: - meta = read_json(case_dir / "meta.json") - if meta: - return meta - return {} - - -def write_fixed_ppd(ppd_path: Path, meta: dict[str, Any], target: dict[str, Any]) -> None: - ppd_source = meta.get("ppd_path") - if ppd_source and Path(str(ppd_source)).exists(): - shutil.copy2(str(ppd_source), ppd_path) - return - candidate = meta.get("env_overrides", {}).get("PPD") if isinstance(meta.get("env_overrides"), dict) else "" - if candidate and Path(str(candidate)).exists(): - shutil.copy2(str(candidate), ppd_path) - return - ppd_path.write_text(minimal_ppd(str(target.get("id", "smt-afl"))), encoding="utf-8") - - -def export_documents(seed_dir: Path, selected: list[Path], target: dict[str, Any]) -> tuple[int, bool]: - count = 0 - for case_dir in selected: - doc = find_document(case_dir) - if doc is None: - continue - suffix = doc.suffix or default_extension(target) - out = seed_dir / f"seed-{count:06d}{suffix}" - shutil.copy2(doc, out) - count += 1 - return count, False - - -def write_fallback_seed(seed_dir: Path, target: dict[str, Any]) -> tuple[int, bool]: - ext = default_extension(target) - path = seed_dir / f"fallback-000000{ext}" - kind = str(target.get("document_kind", "")) - input_mime = str(target.get("input_mime", "")) - if "pdf" in kind or input_mime == "application/pdf": - data = b"%PDF-1.1\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\n" - elif "postscript" in kind or "postscript" in input_mime: - data = b"%!PS\nshowpage\n" - elif "image" in kind or input_mime.startswith("image/"): - data = b"P3\n1 1\n255\n0 0 0\n" - elif "text" in kind or input_mime == "text/plain": - data = b"hello\n" - elif "command" in kind or "cups-command" in input_mime: - data = b"#CUPS-COMMAND\n" - else: - data = b"x\n" - path.write_bytes(data) - return 1, True - - -def configure_template_upgrade_env(args: argparse.Namespace) -> None: - if args.upgrade_generated_seeds <= 0: - return - os.environ.setdefault("SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL", str(args.template_expansion_level)) - os.environ.setdefault("SMT_FUZZER_IMAGE_EXPANSION_LEVEL", str(args.template_expansion_level)) - os.environ.setdefault("SMT_FUZZER_TEMPLATE_CYCLE_EPOCHS", str(args.template_cycle_epochs)) - os.environ.setdefault("SMT_FUZZER_IMAGE_CYCLE_EPOCHS", str(args.template_cycle_epochs)) - os.environ.setdefault("SMT_FUZZER_IMAGE_VALID_BIAS", "1") - os.environ.setdefault("SMT_FUZZER_STRUCTURE_MUTATOR", "1") - - -def generate_upgraded_seeds(*, root: Path, seed_dir: Path, target: dict[str, Any], count: int) -> int: - if count <= 0: - return 0 - _ensure_src_path(root) - from parser_fuzzers.runner.document_harness import make_document - - target_id = str(target["id"]) - kinds = upgraded_document_kinds(str(target.get("document_kind", ""))) - if not kinds: - return 0 - - existing_hashes = { - hashlib.sha256(path.read_bytes()).hexdigest() - for path in seed_dir.iterdir() - if path.is_file() and path.stat().st_size > 0 - } - written = 0 - attempts = 0 - max_attempts = max(count * 12, count + 64) - manifest: list[dict[str, Any]] = [] - while written < count and attempts < max_attempts: - kind = kinds[attempts % len(kinds)] - case_index = attempts // len(kinds) + (attempts % len(kinds)) * 100000 - try: - document = make_document(kind, case_index, target_id=target_id) - except Exception as exc: # pragma: no cover - a broken template should not stop the whole campaign - manifest.append({"kind": kind, "case_index": case_index, "error": f"{type(exc).__name__}: {exc}"}) - attempts += 1 - continue - digest = hashlib.sha256(document.data).hexdigest() - if digest in existing_hashes: - attempts += 1 - continue - existing_hashes.add(digest) - suffix = document.extension or default_extension(target) - out = seed_dir / f"upgrade-{written:06d}-{_safe_name(kind)}{suffix}" - out.write_bytes(document.data) - manifest.append( - { - "path": str(out), - "kind": kind, - "case_index": case_index, - "description": document.description, - "mime": document.mime, - "size": len(document.data), - "sha256": digest, - } - ) - written += 1 - attempts += 1 - (seed_dir.parent / "upgrade-seeds.json").write_text( - json.dumps( - { - "target_id": target_id, - "requested": count, - "generated": written, - "document_kinds": kinds, - "manifest": manifest, - }, - indent=2, - sort_keys=True, - ) - + "\n", - encoding="utf-8", - ) - return written - - -def upgraded_document_kinds(document_kind: str) -> list[str]: - if document_kind == "cups_raster_feedback_sweep": - return [ - "cups_raster_feedback_sweep", - "cups_raster_structural_sweep", - "cups_raster_coverage_sweep", - "cups_raster_general_sweep", - ] - if document_kind == "cups_raster_coverage_sweep": - return ["cups_raster_structural_sweep", "cups_raster_coverage_sweep", "cups_raster_general_sweep"] - if document_kind == "pwg_raster_feedback_sweep": - return [ - "pwg_raster_feedback_sweep", - "pwg_raster_structural_sweep", - "pwg_raster_coverage_sweep", - "pwg_raster_general_sweep", - ] - if document_kind == "pwg_raster_coverage_sweep": - return ["pwg_raster_structural_sweep", "pwg_raster_coverage_sweep", "pwg_raster_general_sweep"] - if document_kind == "image_coverage_sweep": - return ["image_feedback_sweep", "image_coverage_sweep"] - if document_kind == "pdf_coverage_sweep": - return ["pdf_semantic_sweep", "pdf_coverage_sweep"] - if document_kind == "postscript_coverage_sweep": - return ["postscript_semantic_sweep", "postscript_coverage_sweep"] - if document_kind == "text_coverage_sweep": - return ["text_semantic_sweep", "text_coverage_sweep"] - if document_kind == "command_coverage_sweep": - return ["command_semantic_sweep", "command_coverage_sweep"] - return [document_kind] if document_kind else [] - - -def _ensure_src_path(root: Path) -> None: - src = root / "src" - if str(src) not in sys.path: - sys.path.insert(0, str(src)) - - -def _safe_name(value: str) -> str: - return "".join(ch if ch.isalnum() or ch in {"-", "_"} else "-" for ch in value)[:64] - - -def ensure_viable_seed( - plan: TargetPlan, - target: dict[str, Any], - *, - ld_library_path: str, - timeout_sec: float, - drop_nonviable: bool = False, -) -> int: - seeds = sorted(path for path in plan.seed_dir.iterdir() if path.is_file() and path.stat().st_size > 0) - report: list[dict[str, Any]] = [] - viable = [path for path in seeds if seed_is_viable(plan, path, ld_library_path=ld_library_path, timeout_sec=timeout_sec, report=report)] - if not viable: - for index, data in enumerate(fallback_seed_candidates(target)): - ext = default_extension(target) - candidate = plan.seed_dir / f"viable-fallback-{index:06d}{ext}" - candidate.write_bytes(data) - if seed_is_viable(plan, candidate, ld_library_path=ld_library_path, timeout_sec=timeout_sec, report=report): - viable.append(candidate) - break - try: - candidate.unlink() - except OSError: - pass - if viable: - viable_names = {path.name for path in viable} - for path in seeds: - if path.name not in viable_names and (drop_nonviable or path.name.startswith("fallback-")): - try: - path.unlink() - except OSError: - pass - count = len([path for path in plan.seed_dir.iterdir() if path.is_file() and path.stat().st_size > 0]) - else: - count = len(seeds) - (plan.seed_dir.parent / "seed-preflight.json").write_text( - json.dumps( - { - "target_id": plan.target_id, - "viable_seed_count": len(viable), - "seed_count_after_preflight": count, - "checked": report, - }, - indent=2, - sort_keys=True, - ) - + "\n", - encoding="utf-8", - ) - return count - - -def seed_is_viable( - plan: TargetPlan, - seed: Path, - *, - ld_library_path: str, - timeout_sec: float, - report: list[dict[str, Any]], -) -> bool: - if not plan.filter_binary.exists(): - report.append({"seed": str(seed), "status": "missing-binary"}) - return False - env = direct_filter_env(plan, ld_library_path) - cmd = [str(plan.filter_binary), "1", "afl", "afl", "1", plan.job_options, str(seed)] - started = time.monotonic() - try: - result = subprocess.run( - cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - env=env, - timeout=timeout_sec, - check=False, - ) - except subprocess.TimeoutExpired: - report.append({"seed": str(seed), "status": "timeout", "timeout_sec": timeout_sec}) - return False - stderr = result.stderr.decode("utf-8", errors="replace") - crashed = result.returncode < 0 or result.returncode == 86 or "ERROR: AddressSanitizer" in stderr - status = "crash" if crashed else "ok" - report.append( - { - "seed": str(seed), - "status": status, - "returncode": result.returncode, - "elapsed_ms": int((time.monotonic() - started) * 1000), - } - ) - return not crashed - - -def direct_filter_env(plan: TargetPlan, ld_library_path: str) -> dict[str, str]: - env = os.environ.copy() - env.update( - { - "ASAN_OPTIONS": env.get("ASAN_OPTIONS", "abort_on_error=1:detect_leaks=0:symbolize=0:exitcode=86"), - "PPD": str(plan.ppd_path), - "CONTENT_TYPE": plan.input_mime or "application/octet-stream", - "FINAL_CONTENT_TYPE": "application/octet-stream", - "PRINTER": "parser-fuzzers", - "DEVICE_URI": "file:/dev/null", - "SMT_FUZZER_TARGET_ID": plan.target_id, - } - ) - if ld_library_path: - env["LD_LIBRARY_PATH"] = ld_library_path + (":" + env["LD_LIBRARY_PATH"] if env.get("LD_LIBRARY_PATH") else "") - return env - - -def fallback_seed_candidates(target: dict[str, Any]) -> list[bytes]: - kind = str(target.get("document_kind", "")) - input_mime = str(target.get("input_mime", "")) - if "pdf" in kind or input_mime == "application/pdf": - return [ - b"x\n", - b"%PDF-1.1\n%%EOF\n", - b"%PDF-1.1\n1 0 obj\n<< /Type /Catalog >>\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n", - ] - if "postscript" in kind or "postscript" in input_mime: - return [b"x\n", b"%!PS\nshowpage\n", b"%!PS-Adobe-3.0\n%%Pages: 1\nshowpage\n%%EOF\n"] - if "image" in kind or input_mime.startswith("image/"): - return [b"x\n", b"P1\n1 1\n0\n", b"P3\n1 1\n255\n0 0 0\n", minimal_png()] - if "text" in kind or input_mime == "text/plain": - return [b"x\n", b"hello\n"] - if "command" in kind or "cups-command" in input_mime: - return [b"x\n", b"#CUPS-COMMAND\n"] - return [b"x\n"] - - -def minimal_png() -> bytes: - def chunk(tag: bytes, data: bytes) -> bytes: - checksum = zlib.crc32(tag + data) & 0xFFFFFFFF - return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", checksum) - - raw_scanline = b"\x00\x00\x00\x00" - return ( - b"\x89PNG\r\n\x1a\n" - + chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0)) - + chunk(b"IDAT", zlib.compress(raw_scanline)) - + chunk(b"IEND", b"") - ) - - -def find_document(case_dir: Path) -> Path | None: - for path in sorted(case_dir.glob("document.*")): - if path.is_file() and path.stat().st_size > 0: - return path - return None - - -def dictionary_for_target(root: Path, target: dict[str, Any]) -> Path | None: - kind = str(target.get("document_kind", "")) - input_mime = str(target.get("input_mime", "")) - if "raster" in kind or input_mime in {"application/vnd.cups-pwg", "application/vnd.cups-raster"}: - path = root / RASTER_DICT - return path if path.exists() else None - return None - - -def default_extension(target: dict[str, Any]) -> str: - kind = str(target.get("document_kind", "")) - input_mime = str(target.get("input_mime", "")) - if "pwg" in kind or input_mime == "application/vnd.cups-pwg": - return ".pwg" - if "cups_raster" in kind or input_mime == "application/vnd.cups-raster": - return ".ras" - if "pdf" in kind or input_mime == "application/pdf": - return ".pdf" - if "postscript" in kind or input_mime == "application/postscript": - return ".ps" - if "image" in kind: - return ".ppm" - if "text" in kind or input_mime == "text/plain": - return ".txt" - if "command" in kind: - return ".cmd" - return ".bin" - - -def default_job_options(target: dict[str, Any]) -> str: - kind = str(target.get("document_kind", "")) - options = ["PageSize=Letter", "PageRegion=Letter", "ColorModel=Gray", "PrintQuality=Normal", "MediaType=Plain"] - if "raster" not in kind and "pwg" not in kind: - options.extend(["Duplex=None", "Resolution=300x300dpi"]) - return " ".join(options) - - -def minimal_ppd(name: str) -> str: - return f"""*PPD-Adobe: "4.3" -*FormatVersion: "4.3" -*FileVersion: "1.0" -*LanguageVersion: English -*LanguageEncoding: ISOLatin1 -*PCFileName: "SMTAFL.PPD" -*Manufacturer: "parser-fuzzers" -*Product: "({name})" -*ModelName: "{name}" -*NickName: "{name}" -*ShortNickName: "{name}" -*ColorDevice: True -*DefaultPageSize: Letter -*PageSize Letter/Letter: "<>setpagedevice" -*DefaultPageRegion: Letter -*PageRegion Letter/Letter: "<>setpagedevice" -*DefaultImageableArea: Letter -*ImageableArea Letter/Letter: "0 0 612 792" -*DefaultPaperDimension: Letter -*PaperDimension Letter/Letter: "612 792" -*DefaultColorModel: Gray -*ColorModel Gray/Gray: "<>setpagedevice" -*ColorModel RGB/RGB: "<>setpagedevice" -*DefaultResolution: 300dpi -*Resolution 300dpi/300 DPI: "<>setpagedevice" -*cupsFilter2: "application/octet-stream application/octet-stream 0 -" -*% EOF -""" - - -def run_scheduler( - *, - plans: list[TargetPlan], - args: argparse.Namespace, - ld_library_path: str, - output_root: Path, -) -> int: - active: dict[str, subprocess.Popen[bytes]] = {} - pending = list(plans) - completed: dict[str, int] = {} - last_monitor = 0.0 - stop_reason = "complete" - while pending or active: - now = time.monotonic() - while pending and len(active) < max(1, args.max_parallel): - plan = pending.pop(0) - status = launch_target(plan, args, ld_library_path) - if status is None: - completed[plan.target_id] = 2 - else: - active[plan.target_id] = status - for target_id, proc in list(active.items()): - status = proc.poll() - if status is not None: - completed[target_id] = status - active.pop(target_id) - plan = next(item for item in plans if item.target_id == target_id) - write_metrics(plan) - if now - last_monitor >= args.monitor_interval_sec: - last_monitor = now - snapshot = { - "event": "monitor", - "active": sorted(active), - "pending": len(pending), - "completed": len(completed), - "campaign_gb": round(path_bytes(output_root) / (1024.0**3), 3), - "free_gb": free_gb(Path("/data")), - } - print(json.dumps(snapshot, sort_keys=True), flush=True) - if snapshot["free_gb"] < args.min_free_gb: - stop_reason = "disk-free-low" - terminate_all(active) - pending.clear() - elif snapshot["campaign_gb"] > args.max_campaign_gb: - stop_reason = "campaign-growth-limit" - terminate_all(active) - pending.clear() - time.sleep(1.0) - for plan in plans: - if not plan.metrics_path.exists(): - write_metrics(plan) - summary = { - "event": "done", - "stop_reason": stop_reason, - "targets": len(plans), - "completed": completed, - "campaign_gb": round(path_bytes(output_root) / (1024.0**3), 3), - "free_gb": free_gb(Path("/data")), - } - (output_root / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") - print(json.dumps(summary, sort_keys=True), flush=True) - return 0 if stop_reason == "complete" else 90 - - -def launch_target(plan: TargetPlan, args: argparse.Namespace, ld_library_path: str) -> subprocess.Popen[bytes] | None: - plan.out_dir.parent.mkdir(parents=True, exist_ok=True) - if not plan.filter_binary.exists(): - print(json.dumps({"event": "skip-missing-binary", "target_id": plan.target_id, "binary": str(plan.filter_binary)}), flush=True) - return None - env = os.environ.copy() - env.update( - { - "AFL_NO_UI": "1", - "AFL_SKIP_CPUFREQ": "1", - "AFL_CRASH_EXITCODE": "86", - "AFL_SKIP_CRASHES": "1", - "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES": "1", - "ASAN_OPTIONS": env.get("ASAN_OPTIONS", "abort_on_error=1:detect_leaks=0:symbolize=0:exitcode=86"), - "PPD": str(plan.ppd_path), - "CONTENT_TYPE": plan.input_mime or "application/octet-stream", - "FINAL_CONTENT_TYPE": "application/octet-stream", - "PRINTER": "parser-fuzzers", - "DEVICE_URI": "file:/dev/null", - "SMT_FUZZER_TARGET_ID": plan.target_id, - } - ) - if ld_library_path: - env["LD_LIBRARY_PATH"] = ld_library_path + (":" + env["LD_LIBRARY_PATH"] if env.get("LD_LIBRARY_PATH") else "") - cmd = [ - args.afl_fuzz, - "-V", - str(args.duration_sec), - "-t", - str(args.timeout_ms), - "-m", - "none", - "-i", - str(plan.seed_dir), - "-o", - str(plan.out_dir), - "-T", - f"parser-fuzzers-{plan.target_id}", - ] - if plan.dictionary: - cmd.extend(["-x", str(plan.dictionary)]) - cmd.extend(["--", str(plan.filter_binary), "1", "afl", "afl", "1", plan.job_options, "@@"]) - plan_json = plan.out_dir.parent / "run-command.json" - plan_json.write_text( - json.dumps({"argv": cmd, "env": selected_env(env), **plan_to_json(plan)}, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - log = plan.log_path.open("wb") - print(json.dumps({"event": "launch", "target_id": plan.target_id, "seed_count": plan.seed_count, "fallback_seed": plan.fallback_seed}), flush=True) - return subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, env=env) - - -def write_metrics(plan: TargetPlan) -> None: - payload = { - "schema_version": "template-multi-afl-target-v1", - "target_id": plan.target_id, - "seed_count": plan.seed_count, - "fallback_seed": plan.fallback_seed, - "dictionary": str(plan.dictionary) if plan.dictionary else "", - "afl": read_afl_stats(plan.out_dir), - "paths": { - "seed_dir": str(plan.seed_dir), - "out_dir": str(plan.out_dir), - "ppd": str(plan.ppd_path), - "log": str(plan.log_path), - }, - } - plan.metrics_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def read_afl_stats(out_dir: Path) -> dict[str, Any]: - stats_path = find_fuzzer_stats(out_dir) - if not stats_path: - return {"status": "missing-fuzzer-stats", "out_dir": str(out_dir)} - stats: dict[str, Any] = {"status": "ok", "fuzzer_stats": str(stats_path)} - for line in stats_path.read_text(encoding="utf-8", errors="replace").splitlines(): - if ":" not in line: - continue - key, value = line.split(":", 1) - stats[key.strip()] = value.strip() - for key in ("run_time", "execs_done", "corpus_count", "corpus_found", "saved_crashes", "saved_hangs"): - try: - stats[key] = int(str(stats.get(key, "0"))) - except ValueError: - pass - try: - stats["execs_per_sec"] = float(str(stats.get("execs_per_sec", "0"))) - except ValueError: - pass - return stats - - -def find_fuzzer_stats(out_dir: Path) -> Path | None: - candidates = [out_dir / "default" / "fuzzer_stats", out_dir / "fuzzer_stats"] - candidates.extend(sorted(out_dir.glob("*/fuzzer_stats"))) - for path in candidates: - if path.exists(): - return path - return None - - -def terminate_all(active: dict[str, subprocess.Popen[bytes]]) -> None: - for proc in active.values(): - try: - proc.terminate() - except ProcessLookupError: - pass - - -def local_filter_library_path(root: Path) -> str: - pieces = [ - root / "work" / "afl-install" / "libcupsfilters" / "lib", - root / "work" / "afl-install" / "libppd" / "lib", - Path("/data/pre-gsoc/env/pdfio-install/lib"), - ] - return ":".join(str(path) for path in pieces if path.exists()) - - -def path_bytes(root: Path) -> int: - total = 0 - if not root.exists(): - return 0 - for path in root.rglob("*"): - try: - if path.is_file(): - total += path.stat().st_size - except OSError: - continue - return total - - -def free_gb(path: Path) -> float: - usage = shutil.disk_usage(path) - return round(usage.free / (1024.0**3), 3) - - -def read_json(path: Path) -> dict[str, Any]: - try: - return json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return {} - - -def selected_env(env: dict[str, str]) -> dict[str, str]: - keys = [ - "AFL_NO_UI", - "AFL_SKIP_CPUFREQ", - "AFL_CRASH_EXITCODE", - "AFL_SKIP_CRASHES", - "ASAN_OPTIONS", - "PPD", - "CONTENT_TYPE", - "FINAL_CONTENT_TYPE", - "LD_LIBRARY_PATH", - "SMT_FUZZER_TARGET_ID", - ] - return {key: env[key] for key in keys if key in env} - - -def plan_to_json(plan: TargetPlan) -> dict[str, Any]: - return { - "target_id": plan.target_id, - "filter_binary": str(plan.filter_binary), - "input_mime": plan.input_mime, - "document_kind": plan.document_kind, - "ppd_path": str(plan.ppd_path), - "job_options": plan.job_options, - "seed_dir": str(plan.seed_dir), - "out_dir": str(plan.out_dir), - "log_path": str(plan.log_path), - "metrics_path": str(plan.metrics_path), - "dictionary": str(plan.dictionary) if plan.dictionary else "", - "seed_count": plan.seed_count, - "fallback_seed": plan.fallback_seed, - } - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/parser-fuzzers/scripts/run_template_real_afl_loop.sh b/parser-fuzzers/scripts/run_template_real_afl_loop.sh deleted file mode 100755 index b15c4ff..0000000 --- a/parser-fuzzers/scripts/run_template_real_afl_loop.sh +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" - -target="${1:-pwgtopdf}" -duration_sec="${2:-1200}" -monitor_interval="${3:-300}" -workers="${SMT_TEMPLATE_REAL_WORKERS:-4}" -timeout_sec="${SMT_TEMPLATE_REAL_TIMEOUT_SEC:-5}" -max_run_gb="${SMT_TEMPLATE_REAL_MAX_RUN_GB:-10}" -base_config="${SMT_TEMPLATE_REAL_CONFIG:-configs/parser_targets_afl_pwg_feedback.yaml}" -work_root="${SMT_TEMPLATE_REAL_LOOP_WORK_ROOT:-work/template-real-afl-loop}" -seed_limit="${SMT_TEMPLATE_REAL_SEED_LIMIT:-512}" - -case "$target" in - pwgtopdf) - template_target_id="pwg_to_pdf_afl_feedback" - extension=".pwg" - ;; - pwgtopclm) - template_target_id="pwg_to_pclm_afl_feedback" - extension=".pwg" - ;; - *) - echo "usage: $0 [pwgtopdf|pwgtopclm] [duration-seconds] [monitor-interval]" >&2 - exit 2 - ;; -esac - -template_sec="${SMT_TEMPLATE_REAL_LOOP_TEMPLATE_SEC:-$((duration_sec / 4))}" -afl_sec="${SMT_TEMPLATE_REAL_LOOP_AFL_SEC:-$((duration_sec / 2))}" -feedback_sec="${SMT_TEMPLATE_REAL_LOOP_FEEDBACK_SEC:-$((duration_sec - template_sec - afl_sec))}" -if (( template_sec < 60 )); then template_sec=60; fi -if (( afl_sec < 60 )); then afl_sec=60; fi -if (( feedback_sec < 60 )); then feedback_sec=60; fi - -if [[ ! -f work/afl-install/afl-env.sh ]]; then - bash scripts/build_afl_cupsfilters_stack.sh -fi -source work/afl-install/afl-env.sh - -filter_binary="$SMT_AFL_CUPSFILTERS_BIN/$target" -if [[ ! -x "$filter_binary" ]]; then - echo "missing AFL++ filter binary: $filter_binary" >&2 - exit 2 -fi - -stamp="$(date +%Y%m%d-%H%M%S)" -campaign_dir="$work_root/${target}-${stamp}" -template_root="$campaign_dir/template" -seed_dir="$campaign_dir/seeds" -afl_campaign_dir="$campaign_dir/afl-standard" -feedback_root="$campaign_dir/feedback-template" -filtered_config="$campaign_dir/template-target.yaml" -mkdir -p "$campaign_dir" - -python3 - "$base_config" "$template_target_id" "$filter_binary" "$filtered_config" <<'PY' -from pathlib import Path -import sys -import yaml - -base_config, target_id, filter_binary, output = sys.argv[1:] -data = yaml.safe_load(Path(base_config).read_text(encoding="utf-8")) -targets = [item for item in data.get("targets", []) if item.get("id") == target_id] -if not targets: - raise SystemExit(f"target {target_id!r} not found in {base_config}") -targets[0]["filter_binary"] = filter_binary -Path(output).write_text(yaml.safe_dump({"targets": targets}, sort_keys=False), encoding="utf-8") -PY - -export LD_LIBRARY_PATH="$SMT_AFL_LIBCUPSFILTERS_LIB:$SMT_AFL_LIBPPD_LIB:$SMT_AFL_PDFIO_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -export ASAN_OPTIONS="${ASAN_OPTIONS:-abort_on_error=0:detect_leaks=0:symbolize=1:exitcode=86}" -export SMT_FUZZER_LIBPPD_ASAN="$SMT_AFL_LIBPPD_LIB" -export SMT_FUZZER_LIBCUPSFILTERS_ASAN="$SMT_AFL_LIBCUPSFILTERS_LIB" -export SMT_FUZZER_PDFIO_LIB="$SMT_AFL_PDFIO_LIB" - -echo "campaign_dir=$campaign_dir" -echo "mode=template-real-afl-loop" -echo "target=$target" -echo "template_target_id=$template_target_id" -echo "filter_binary=$filter_binary" -echo "duration_sec=$duration_sec" -echo "template_sec=$template_sec" -echo "afl_sec=$afl_sec" -echo "feedback_sec=$feedback_sec" -echo "monitor_interval=$monitor_interval" - -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config "$filtered_config" \ - --work-root "$template_root" \ - --workers "$workers" \ - --timeout-sec "$timeout_sec" \ - --duration-sec "$template_sec" \ - --max-run-gb "$max_run_gb" \ - --discard-stdout \ - --discovery-mode coverage \ - --scheduler novelty \ - --runtime-skip \ - --summary-mode concise \ - --prune-uninteresting >"$campaign_dir/template.stdout.json" - -template_run="$(find "$template_root" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | sed -n '1s/^[^ ]* //p')" -if [[ -z "$template_run" ]]; then - echo "template run not found" >&2 - exit 1 -fi - -set +e -PYTHONPATH=src python3 -m parser_fuzzers.cli dedup-crashes \ - --run-dir "$template_run" \ - --output-json "$template_run/dedup.json" \ - --output-md "$template_run/dedup.md" >/dev/null 2>"$campaign_dir/template-dedup.stderr" -set -e - -PYTHONPATH=src python3 -m parser_fuzzers.cli summarize-run-metrics \ - --run-dir "$template_run" \ - --output "$template_run/standard_metrics.json" >"$campaign_dir/template-standard-metrics.stdout.json" - -PYTHONPATH=src python3 -m parser_fuzzers.cli export-template-seeds \ - --run-dir "$template_run" \ - --target-id "$template_target_id" \ - --extension "$extension" \ - --output-dir "$seed_dir" \ - --limit "$seed_limit" >"$campaign_dir/seed-export.json" - -SMT_AFL_CAMPAIGN_DIR="$afl_campaign_dir" \ -SMT_AFL_PWG_SEED_DIR="$seed_dir" \ -SMT_AFL_DIRECT_INSTRUMENTED=1 \ -SMT_AFL_DIRECT_FILTER_BINARY="$filter_binary" \ -SMT_AFL_DIRECT_LD_LIBRARY_PATH="$LD_LIBRARY_PATH" \ -SMT_AFL_IMPORT_QUEUE_MODE=new \ -scripts/run_afl_pwg_frontier.sh "$target" "$afl_sec" "$monitor_interval" >"$campaign_dir/afl-run.stdout" 2>"$campaign_dir/afl-run.stderr" - -cp "$afl_campaign_dir/standard-metrics.json" "$campaign_dir/afl-standard-metrics.json" -cp "$afl_campaign_dir/afl-import.json" "$campaign_dir/afl-import.json" -cp "$afl_campaign_dir/afl-feedback-build.json" "$campaign_dir/feedback-profile-build.json" -feedback_profile="$afl_campaign_dir/afl-feedback.json" - -SMT_FUZZER_TEMPLATE_FEEDBACK="$feedback_profile" \ -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config "$filtered_config" \ - --work-root "$feedback_root" \ - --workers "$workers" \ - --timeout-sec "$timeout_sec" \ - --duration-sec "$feedback_sec" \ - --max-run-gb "$max_run_gb" \ - --discard-stdout \ - --discovery-mode coverage \ - --scheduler novelty \ - --runtime-skip \ - --summary-mode concise \ - --prune-uninteresting >"$campaign_dir/feedback-template.stdout.json" - -feedback_run="$(find "$feedback_root" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | sed -n '1s/^[^ ]* //p')" -if [[ -z "$feedback_run" ]]; then - echo "feedback template run not found" >&2 - exit 1 -fi - -set +e -PYTHONPATH=src python3 -m parser_fuzzers.cli dedup-crashes \ - --run-dir "$feedback_run" \ - --output-json "$feedback_run/dedup.json" \ - --output-md "$feedback_run/dedup.md" >/dev/null 2>"$campaign_dir/feedback-dedup.stderr" -set -e - -PYTHONPATH=src python3 -m parser_fuzzers.cli summarize-run-metrics \ - --run-dir "$feedback_run" \ - --output "$feedback_run/standard_metrics.json" >"$campaign_dir/feedback-template-standard-metrics.stdout.json" - -python3 - "$campaign_dir" "$template_run" "$seed_dir" "$afl_campaign_dir" "$feedback_profile" "$feedback_run" "$duration_sec" "$template_sec" "$afl_sec" "$feedback_sec" <<'PY' -from pathlib import Path -import json -import sys - -( - campaign_dir, - template_run, - seed_dir, - afl_campaign_dir, - feedback_profile, - feedback_run, - duration_sec, - template_sec, - afl_sec, - feedback_sec, -) = sys.argv[1:] -manifest = { - "campaign_dir": campaign_dir, - "mode": "template-real-afl-loop", - "template_run": template_run, - "seed_dir": seed_dir, - "afl_out": str(Path(afl_campaign_dir) / "out"), - "afl_campaign_dir": afl_campaign_dir, - "import_dir": str(Path(afl_campaign_dir) / "feedback-import"), - "feedback_profile": feedback_profile, - "feedback_run": feedback_run, - "duration_sec": int(duration_sec), - "template_sec": int(template_sec), - "afl_sec": int(afl_sec), - "feedback_sec": int(feedback_sec), -} -Path(campaign_dir, "loop_manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") -PY - -PYTHONPATH=src python3 -m parser_fuzzers.cli summarize-loop-metrics \ - --campaign-dir "$campaign_dir" \ - --output "$campaign_dir/loop_standard_metrics.json" >"$campaign_dir/loop-standard-metrics.stdout.json" - -echo "campaign_dir=$campaign_dir" -echo "template_run=$template_run" -echo "template_metrics=$template_run/standard_metrics.json" -echo "seed_dir=$seed_dir" -echo "seed_export=$campaign_dir/seed-export.json" -echo "afl_campaign_dir=$afl_campaign_dir" -echo "afl_metrics=$campaign_dir/afl-standard-metrics.json" -echo "feedback_profile=$feedback_profile" -echo "feedback_run=$feedback_run" -echo "feedback_metrics=$feedback_run/standard_metrics.json" -echo "loop_metrics=$campaign_dir/loop_standard_metrics.json" diff --git a/parser-fuzzers/scripts/run_template_runner_campaign.sh b/parser-fuzzers/scripts/run_template_runner_campaign.sh deleted file mode 100755 index 5a4313b..0000000 --- a/parser-fuzzers/scripts/run_template_runner_campaign.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" - -DURATION_SEC="${1:-${SMT_TEMPLATE_DURATION:-60}}" -WORKERS="${2:-${SMT_TEMPLATE_WORKERS:-2}}" -TIMEOUT_SEC="${3:-${SMT_TEMPLATE_TIMEOUT_SEC:-5}}" -CONFIG="${4:-${SMT_TEMPLATE_CONFIG:-configs/parser_targets_auto_hybrid.yaml}}" -WORK_ROOT="${SMT_TEMPLATE_WORK_ROOT:-work/template-runner}" -MAX_RUN_GB="${SMT_TEMPLATE_MAX_RUN_GB:-1}" -FILTER_ROOT="${SMT_TEMPLATE_FILTER_ROOT:-${SMT_FUZZER_TEMPLATE_FILTER_ROOT:-${SMT_FUZZER_FILTER_ROOT:-/data/pre-gsoc/cups-filters}}}" - -prompt_filter_root() { - local answer - if [[ "${SMT_TEMPLATE_SKIP_FILTER_PROMPT:-0}" == "1" ]]; then - return - fi - if [[ -t 0 ]]; then - printf 'Template runner filter root [%s]: ' "$FILTER_ROOT" - read -r answer - if [[ -n "$answer" ]]; then - FILTER_ROOT="$answer" - fi - fi -} - -prompt_filter_root - -if [[ ! -d "$FILTER_ROOT" ]]; then - echo "missing template filter root: $FILTER_ROOT" >&2 - echo "set SMT_TEMPLATE_FILTER_ROOT or enter a directory containing cups-filters binaries" >&2 - exit 2 -fi - -if [[ "${SMT_TEMPLATE_CHECK_FILTERS:-1}" != "0" ]]; then - scripts/check_cups_filters_targets.sh --coverage "$FILTER_ROOT" -fi - -export PYTHONPATH="$ROOT/src" -python3 -m parser_fuzzers.cli multitarget-monitor \ - --config "$CONFIG" \ - --work-root "$WORK_ROOT" \ - --filter-root "$FILTER_ROOT" \ - --workers "$WORKERS" \ - --timeout-sec "$TIMEOUT_SEC" \ - --duration-sec "$DURATION_SEC" \ - --max-run-gb "$MAX_RUN_GB" \ - --discard-stdout \ - --discovery-mode coverage \ - --scheduler novelty \ - --runtime-skip \ - --summary-mode concise \ - --prune-uninteresting diff --git a/parser-fuzzers/scripts/run_template_to_real_afl.sh b/parser-fuzzers/scripts/run_template_to_real_afl.sh deleted file mode 100755 index b9e4484..0000000 --- a/parser-fuzzers/scripts/run_template_to_real_afl.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" - -target="${1:-pwgtopdf}" -afl_seconds="${2:-1200}" -template_seconds="${3:-1200}" -monitor_interval="${4:-300}" -workers="${SMT_TEMPLATE_REAL_WORKERS:-4}" -timeout_sec="${SMT_TEMPLATE_REAL_TIMEOUT_SEC:-5}" -max_run_gb="${SMT_TEMPLATE_REAL_MAX_RUN_GB:-10}" -base_config="${SMT_TEMPLATE_REAL_CONFIG:-configs/parser_targets_afl_pwg_feedback.yaml}" -work_root="${SMT_TEMPLATE_REAL_WORK_ROOT:-work/template-real-afl}" -seed_limit="${SMT_TEMPLATE_REAL_SEED_LIMIT:-512}" - -case "$target" in - pwgtopdf) - template_target_id="pwg_to_pdf_afl_feedback" - extension=".pwg" - ;; - pwgtopclm) - template_target_id="pwg_to_pclm_afl_feedback" - extension=".pwg" - ;; - *) - echo "usage: $0 [pwgtopdf|pwgtopclm] [afl-seconds] [template-seconds] [monitor-interval]" >&2 - exit 2 - ;; -esac - -if [[ ! -f work/afl-install/afl-env.sh ]]; then - bash scripts/build_afl_cupsfilters_stack.sh -fi -source work/afl-install/afl-env.sh - -filter_binary="$SMT_AFL_CUPSFILTERS_BIN/$target" -if [[ ! -x "$filter_binary" ]]; then - echo "missing AFL++ filter binary: $filter_binary" >&2 - exit 2 -fi - -stamp="$(date +%Y%m%d-%H%M%S)" -campaign_dir="$work_root/${target}-${stamp}" -template_root="$campaign_dir/template" -seed_dir="$campaign_dir/seeds" -afl_campaign_dir="$campaign_dir/afl-standard" -filtered_config="$campaign_dir/template-target.yaml" -mkdir -p "$campaign_dir" - -python3 - "$base_config" "$template_target_id" "$filter_binary" "$filtered_config" <<'PY' -from pathlib import Path -import sys -import yaml - -base_config, target_id, filter_binary, output = sys.argv[1:] -data = yaml.safe_load(Path(base_config).read_text(encoding="utf-8")) -targets = [item for item in data.get("targets", []) if item.get("id") == target_id] -if not targets: - raise SystemExit(f"target {target_id!r} not found in {base_config}") -targets[0]["filter_binary"] = filter_binary -Path(output).write_text(yaml.safe_dump({"targets": targets}, sort_keys=False), encoding="utf-8") -PY - -export LD_LIBRARY_PATH="$SMT_AFL_LIBCUPSFILTERS_LIB:$SMT_AFL_LIBPPD_LIB:$SMT_AFL_PDFIO_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -export ASAN_OPTIONS="${ASAN_OPTIONS:-abort_on_error=0:detect_leaks=0:symbolize=1:exitcode=86}" -export SMT_FUZZER_LIBPPD_ASAN="$SMT_AFL_LIBPPD_LIB" -export SMT_FUZZER_LIBCUPSFILTERS_ASAN="$SMT_AFL_LIBCUPSFILTERS_LIB" -export SMT_FUZZER_PDFIO_LIB="$SMT_AFL_PDFIO_LIB" - -echo "campaign_dir=$campaign_dir" -echo "mode=template-to-real-afl" -echo "target=$target" -echo "template_target_id=$template_target_id" -echo "filter_binary=$filter_binary" -echo "template_seconds=$template_seconds" -echo "afl_seconds=$afl_seconds" -echo "monitor_interval=$monitor_interval" - -PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor \ - --config "$filtered_config" \ - --work-root "$template_root" \ - --workers "$workers" \ - --timeout-sec "$timeout_sec" \ - --duration-sec "$template_seconds" \ - --max-run-gb "$max_run_gb" \ - --discard-stdout \ - --discovery-mode coverage \ - --scheduler novelty \ - --runtime-skip \ - --summary-mode concise \ - --prune-uninteresting >"$campaign_dir/template.stdout.json" - -template_run="$(find "$template_root" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | sed -n '1s/^[^ ]* //p')" -if [[ -z "$template_run" ]]; then - echo "template run not found" >&2 - exit 1 -fi - -set +e -PYTHONPATH=src python3 -m parser_fuzzers.cli dedup-crashes \ - --run-dir "$template_run" \ - --output-json "$template_run/dedup.json" \ - --output-md "$template_run/dedup.md" >/dev/null 2>"$campaign_dir/template-dedup.stderr" -set -e - -PYTHONPATH=src python3 -m parser_fuzzers.cli summarize-run-metrics \ - --run-dir "$template_run" \ - --output "$template_run/standard_metrics.json" >"$campaign_dir/template-standard-metrics.stdout.json" - -PYTHONPATH=src python3 -m parser_fuzzers.cli export-template-seeds \ - --run-dir "$template_run" \ - --target-id "$template_target_id" \ - --extension "$extension" \ - --output-dir "$seed_dir" \ - --limit "$seed_limit" >"$campaign_dir/seed-export.json" - -SMT_AFL_CAMPAIGN_DIR="$afl_campaign_dir" \ -SMT_AFL_PWG_SEED_DIR="$seed_dir" \ -SMT_AFL_DIRECT_INSTRUMENTED=1 \ -SMT_AFL_DIRECT_FILTER_BINARY="$filter_binary" \ -SMT_AFL_DIRECT_LD_LIBRARY_PATH="$LD_LIBRARY_PATH" \ -SMT_AFL_IMPORT_QUEUE_MODE=new \ -scripts/run_afl_pwg_frontier.sh "$target" "$afl_seconds" "$monitor_interval" >"$campaign_dir/afl-run.stdout" 2>"$campaign_dir/afl-run.stderr" - -python3 - "$campaign_dir" "$template_run" "$seed_dir" "$afl_campaign_dir" "$template_seconds" "$afl_seconds" <<'PY' -from pathlib import Path -import json -import sys - -campaign_dir, template_run, seed_dir, afl_campaign_dir, template_seconds, afl_seconds = sys.argv[1:] -manifest = { - "campaign_dir": campaign_dir, - "mode": "template-to-real-afl", - "template_run": template_run, - "seed_dir": seed_dir, - "afl_campaign_dir": afl_campaign_dir, - "template_seconds": int(template_seconds), - "afl_seconds": int(afl_seconds), -} -Path(campaign_dir, "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") -PY - -echo "campaign_dir=$campaign_dir" -echo "template_run=$template_run" -echo "template_metrics=$template_run/standard_metrics.json" -echo "seed_dir=$seed_dir" -echo "seed_export=$campaign_dir/seed-export.json" -echo "afl_campaign_dir=$afl_campaign_dir" -echo "afl_metrics=$afl_campaign_dir/standard-metrics.json" diff --git a/parser-fuzzers/scripts/setup_tui.sh b/parser-fuzzers/scripts/setup_tui.sh deleted file mode 100755 index 6296293..0000000 --- a/parser-fuzzers/scripts/setup_tui.sh +++ /dev/null @@ -1,532 +0,0 @@ -#!/usr/bin/env bash -set -u - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -CONFIG_FILE="$ROOT/work/setup.env" - -FILTER_ROOT="${SMT_FUZZER_FILTER_ROOT:-/usr/lib/cups/filter}" -TEMPLATE_FILTER_ROOT="${SMT_TEMPLATE_FILTER_ROOT:-${SMT_FUZZER_TEMPLATE_FILTER_ROOT:-/data/pre-gsoc/cups-filters}}" -ASAN_ROOT="${SMT_FUZZER_ASAN_ROOT:-$ROOT/work/openprinting-asan}" -VENV_DIR="${SMT_FUZZER_VENV_DIR:-.venv}" -TARGET_CONFIG="${SMT_FUZZER_TARGET_CONFIG:-configs/parser_targets.yaml}" -TEMPLATE_CONFIG="${SMT_TEMPLATE_CONFIG:-configs/parser_targets_auto_hybrid.yaml}" -QUICK_DURATION="${SMT_FUZZER_QUICK_DURATION:-1}" -TEMPLATE_DURATION="${SMT_TEMPLATE_DURATION:-60}" -ASAN_DURATION="${SMT_FUZZER_ASAN_DURATION:-60}" -WORKERS="${SMT_FUZZER_WORKERS:-2}" -TEMPLATE_WORKERS="${SMT_TEMPLATE_WORKERS:-2}" -TIMEOUT_SEC="${SMT_FUZZER_TIMEOUT_SEC:-5}" -TEMPLATE_TIMEOUT_SEC="${SMT_TEMPLATE_TIMEOUT_SEC:-5}" -TEMPLATE_MAX_RUN_GB="${SMT_TEMPLATE_MAX_RUN_GB:-1}" - -if [[ -r "$CONFIG_FILE" ]]; then - # shellcheck disable=SC1090 - . "$CONFIG_FILE" -fi - -if [[ -t 1 && "${TERM:-dumb}" != "dumb" ]]; then - BOLD="$(printf '\033[1m')" - DIM="$(printf '\033[2m')" - RED="$(printf '\033[31m')" - GREEN="$(printf '\033[32m')" - YELLOW="$(printf '\033[33m')" - RESET="$(printf '\033[0m')" -else - BOLD="" - DIM="" - RED="" - GREEN="" - YELLOW="" - RESET="" -fi - -usage() { - cat <<'EOF' -usage: scripts/setup_tui.sh [--status|--commands|--help] - -Interactive setup menu for local parser-fuzzers initialization. - -Options: - --status print current configuration and dependency status, then exit - --commands print commands matching the current configuration, then exit - --help show this help -EOF -} - -venv_path() { - case "$VENV_DIR" in - /*) printf '%s\n' "$VENV_DIR" ;; - *) printf '%s\n' "$ROOT/$VENV_DIR" ;; - esac -} - -python_bin() { - local venv - venv="$(venv_path)" - if [[ -x "$venv/bin/python" ]]; then - printf '%s\n' "$venv/bin/python" - elif command -v python3 >/dev/null 2>&1; then - command -v python3 - else - printf '%s\n' "python3" - fi -} - -status_word() { - if "$@" >/dev/null 2>&1; then - printf '%sok%s' "$GREEN" "$RESET" - else - printf '%smissing%s' "$RED" "$RESET" - fi -} - -shell_quote() { - printf '%q' "$1" -} - -save_config() { - mkdir -p "$ROOT/work" - { - echo "# Generated by scripts/setup_tui.sh" - echo "SMT_FUZZER_FILTER_ROOT=$(shell_quote "$FILTER_ROOT")" - echo "SMT_TEMPLATE_FILTER_ROOT=$(shell_quote "$TEMPLATE_FILTER_ROOT")" - echo "SMT_FUZZER_ASAN_ROOT=$(shell_quote "$ASAN_ROOT")" - echo "SMT_FUZZER_VENV_DIR=$(shell_quote "$VENV_DIR")" - echo "SMT_FUZZER_TARGET_CONFIG=$(shell_quote "$TARGET_CONFIG")" - echo "SMT_TEMPLATE_CONFIG=$(shell_quote "$TEMPLATE_CONFIG")" - echo "SMT_FUZZER_QUICK_DURATION=$(shell_quote "$QUICK_DURATION")" - echo "SMT_TEMPLATE_DURATION=$(shell_quote "$TEMPLATE_DURATION")" - echo "SMT_FUZZER_ASAN_DURATION=$(shell_quote "$ASAN_DURATION")" - echo "SMT_FUZZER_WORKERS=$(shell_quote "$WORKERS")" - echo "SMT_TEMPLATE_WORKERS=$(shell_quote "$TEMPLATE_WORKERS")" - echo "SMT_FUZZER_TIMEOUT_SEC=$(shell_quote "$TIMEOUT_SEC")" - echo "SMT_TEMPLATE_TIMEOUT_SEC=$(shell_quote "$TEMPLATE_TIMEOUT_SEC")" - echo "SMT_TEMPLATE_MAX_RUN_GB=$(shell_quote "$TEMPLATE_MAX_RUN_GB")" - } >"$CONFIG_FILE" - echo "[ok] saved $CONFIG_FILE" -} - -pause() { - if [[ -t 0 ]]; then - printf '\nPress Enter to continue... ' - read -r _ - fi -} - -clear_screen() { - if [[ -t 1 && "${TERM:-dumb}" != "dumb" ]]; then - printf '\033c' - fi -} - -read_default() { - local prompt="$1" - local default_value="$2" - local value - printf '%s [%s]: ' "$prompt" "$default_value" - read -r value - if [[ -z "$value" ]]; then - printf '%s\n' "$default_value" - else - printf '%s\n' "$value" - fi -} - -yes_no() { - local prompt="$1" - local default_value="${2:-n}" - local suffix="[y/N]" - local answer - if [[ "$default_value" == "y" ]]; then - suffix="[Y/n]" - fi - printf '%s %s ' "$prompt" "$suffix" - read -r answer - if [[ -z "$answer" ]]; then - answer="$default_value" - fi - [[ "$answer" == "y" || "$answer" == "Y" || "$answer" == "yes" || "$answer" == "YES" ]] -} - -run_project_cmd() { - local status - local venv - venv="$(venv_path)" - printf '\n%s+%s' "$DIM" "$RESET" - printf ' %q' "$@" - printf '\n\n' - ( - cd "$ROOT" || exit 2 - export PATH="$venv/bin:$PATH" - export PYTHONPATH="$ROOT/src" - export SMT_FUZZER_FILTER_ROOT="$FILTER_ROOT" - "$@" - ) - status=$? - printf '\n[exit] %s\n' "$status" - return "$status" -} - -run_python_cmd() { - local py - py="$(python_bin)" - run_project_cmd "$py" "$@" -} - -count_filters() { - local filters=(rastertopclx rastertoescpx pwgtoraster) - local count=0 - local filter - for filter in "${filters[@]}"; do - if [[ -x "$FILTER_ROOT/$filter" ]]; then - count=$((count + 1)) - fi - done - printf '%s/%s' "$count" "${#filters[@]}" -} - -print_status() { - local py - py="$(python_bin)" - - cat <"$plan" - chmod +x "$plan" - echo "[ok] wrote $plan" - echo "[next] inspect it, then run: bash $(shell_quote "$plan")" - if yes_no "Run the ASan build plan now? This may clone and build OpenPrinting projects." "n"; then - run_project_cmd bash "$plan" || true - fi - pause -} - -asan_campaign() { - run_project_cmd scripts/run_asan_cups_filters_campaign.sh "$ASAN_ROOT" "$ASAN_DURATION" "$WORKERS" "$TIMEOUT_SEC" "$TARGET_CONFIG" || true - pause -} - -template_runner_campaign() { - print_status - TEMPLATE_FILTER_ROOT="$(read_default "Template runner filter root" "$TEMPLATE_FILTER_ROOT")" - TEMPLATE_CONFIG="$(read_default "Template runner config" "$TEMPLATE_CONFIG")" - TEMPLATE_DURATION="$(read_default "Template runner duration seconds" "$TEMPLATE_DURATION")" - TEMPLATE_WORKERS="$(read_default "Template runner workers" "$TEMPLATE_WORKERS")" - TEMPLATE_TIMEOUT_SEC="$(read_default "Template runner timeout seconds" "$TEMPLATE_TIMEOUT_SEC")" - TEMPLATE_MAX_RUN_GB="$(read_default "Template runner max run GB" "$TEMPLATE_MAX_RUN_GB")" - save_config - - if [[ ! -d "$TEMPLATE_FILTER_ROOT" ]]; then - echo "[warn] template filter root does not exist: $TEMPLATE_FILTER_ROOT" - if ! yes_no "Continue anyway?" "n"; then - pause - return - fi - else - run_project_cmd scripts/check_cups_filters_targets.sh --coverage "$TEMPLATE_FILTER_ROOT" || true - if ! yes_no "Run template runner with this path?" "y"; then - pause - return - fi - fi - - SMT_TEMPLATE_FILTER_ROOT="$TEMPLATE_FILTER_ROOT" \ - SMT_TEMPLATE_CONFIG="$TEMPLATE_CONFIG" \ - SMT_TEMPLATE_MAX_RUN_GB="$TEMPLATE_MAX_RUN_GB" \ - SMT_TEMPLATE_SKIP_FILTER_PROMPT=1 \ - run_project_cmd scripts/run_template_runner_campaign.sh \ - "$TEMPLATE_DURATION" "$TEMPLATE_WORKERS" "$TEMPLATE_TIMEOUT_SEC" "$TEMPLATE_CONFIG" || true - pause -} - -afl_menu() { - local choice - local target - local config - local binary - - cat <<'EOF' -AFL++: - 1) install/check package group - 2) print AFL++ compiler environment - 3) print sample afl-fuzz command -EOF - choice="$(read_default "Select AFL++ action" "2")" - case "$choice" in - 1) - run_project_cmd scripts/install_ubuntu_deps.sh --afl --dry-run - ;; - 2) - run_project_cmd scripts/afl_build_env.sh - ;; - 3) - target="$(read_default "Target id" "ppd_ipp_parser")" - config="$(read_default "AFL config A0-A4" "A1")" - binary="$(read_default "Instrumented binary" "harnesses/bin/ppd_ipp_parser")" - run_project_cmd scripts/run_afl.sh "$target" "$config" "$binary" - ;; - *) - echo "[warn] unknown AFL++ action" - ;; - esac - pause -} - -safe_init() { - setup_python - validate_smoke_tests -} - -main_menu() { - local choice - while true; do - clear_screen - print_status - cat <<'EOF' - -Menu: - 0) safe clone-only init: venv + pip install + validate/smoke/tests - 1) install Ubuntu dependencies - 2) configure paths and run sizes - 3) validate Python smoke and tests - 4) check local CUPS filter targets - 5) run recommended quick start target campaign - 6) write isolated ASan build script - 7) run isolated ASan campaign - 8) AFL++ setup and command helper - 9) run template runner campaign - 10) print commands for current config - q) quit -EOF - printf '\nSelect action: ' - read -r choice - case "$choice" in - 0) safe_init ;; - 1) install_deps_menu ;; - 2) configure_menu ;; - 3) validate_smoke_tests ;; - 4) check_filters ;; - 5) quick_campaign ;; - 6) write_asan_plan ;; - 7) asan_campaign ;; - 8) afl_menu ;; - 9) template_runner_campaign ;; - 10) print_commands; pause ;; - q|Q) save_config; exit 0 ;; - *) echo "[warn] unknown action"; pause ;; - esac - done -} - -case "${1:-}" in - --help|-h) - usage - exit 0 - ;; - --status) - print_status - exit 0 - ;; - --commands) - print_commands - exit 0 - ;; - "") - main_menu - ;; - *) - echo "unknown argument: $1" >&2 - usage >&2 - exit 2 - ;; -esac diff --git a/parser-fuzzers/seeds/public/README.md b/parser-fuzzers/seeds/public/README.md deleted file mode 100644 index 794465b..0000000 --- a/parser-fuzzers/seeds/public/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Public Seed Policy - -This directory is for weak public seeds only. - -Private reproducers, minimized crash inputs, and local issue artifacts must not -be copied here. Local metadata, when used, should keep -`known_poc_allowed_in_seed: false`. diff --git a/parser-fuzzers/seeds/public/weak-seed.txt b/parser-fuzzers/seeds/public/weak-seed.txt deleted file mode 100644 index 63f9dd6..0000000 --- a/parser-fuzzers/seeds/public/weak-seed.txt +++ /dev/null @@ -1 +0,0 @@ -SMT-FUZZER-WEAK-SEED diff --git a/parser-fuzzers/src/parser_fuzzers/README.md b/parser-fuzzers/src/parser_fuzzers/README.md deleted file mode 100644 index af5e7ad..0000000 --- a/parser-fuzzers/src/parser_fuzzers/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# parser_fuzzers Module Map - -The package is split into layer subpackages. Historical flat imports such as -`parser_fuzzers.solver` are kept as compatibility wrappers. - -## Core - -- `core/models.py`: branch events and solver result models. -- `core/validation.py`: bug/config validation. -- `core/hashing.py`: small hashing helpers. -- `core/experiment.py`: A0-A4 experiment matrix helpers. -- `core/format_specs.py`: format-specific constants and specs. - -## Generator And SMT - -- `generator/solver.py`: Z3-backed branch-event solver. -- `generator/z3_guard.py`: Z3 availability and guarded solving. -- `generator/patcher.py`: apply solver patches to input bytes. -- `generator/template_synth.py`: typed template slot filling. -- `generator/ppd_templates.py`: PPD templates. -- `generator/image_templates.py`: image input templates. -- `generator/structured_templates.py`: structured PPD/Raster/PWG/template builders. -- `generator/structure_mutator.py`: structure-aware mutations. -- `generator/dimension_expander.py`: automatic template dimension expansion. -- `generator/auto_expand.py`: automatic seed/template expansion driver. -- `generator/constraint_repair.py`: repair related fields after mutation. -- `generator/arithmetic_explorer.py`: cross-input arithmetic/boundary exploration. - -## Runner - -- `runner/cli.py`: command-line entrypoint. -- `runner/document_harness.py`: materialize PPD/document inputs and commands. -- `runner/multitarget_runner.py`: scheduling, execution, retention, skip policy. -- `runner/cupsfilter.py`: cupsfilter-specific smoke target helpers. -- `runner/ppd_pipeline.py`: PPD template pipeline helpers. - -## AFL++ Boundary - -- `afl_integration/afl.py`: AFL++ command/corpus/dictionary/CmpLog plan builder. -- `afl_integration/afl_feedback.py`: import AFL++ queue/crash outputs into runner feedback. -- `afl_integration/seed_export.py`: export retained template documents into AFL++ seed directories. - -## Feedback, Triage, And Metrics - -- `feedback/template_feedback.py`: build feedback profiles from retained cases. -- `feedback/output_feedback.py`: output-derived feedback extraction. -- `feedback/semantic_shapes.py`: semantic shape extraction. -- `feedback/crash_avoidance.py`: known crash-shape suppression helpers. -- `feedback/crash_dedup.py`: crash signature normalization and representative choice. -- `metrics/run_metrics.py`: standard run metrics. -- `metrics/loop_metrics.py`: template/AFL++/feedback loop metrics aggregation. -- `metrics/run_set_metrics.py`: multi-run metric summaries. -- `metrics/run_recovery.py`: recover/summarize interrupted runs. -- `metrics/baseline_compare.py`: baseline and LLVM coverage comparison helpers. - -## Import Boundary Rule - -Prefer importing across layers in this direction: - -```text -core -> generator -> runner -> feedback/metrics -core -> afl boundary -> feedback/metrics -``` - -Avoid making generator code depend on AFL++ output formats. Import AFL++ output -through `afl_feedback.py` first, then feed the normalized cases into feedback -or template-generation code. diff --git a/parser-fuzzers/src/parser_fuzzers/__init__.py b/parser-fuzzers/src/parser_fuzzers/__init__.py deleted file mode 100644 index d7ba04f..0000000 --- a/parser-fuzzers/src/parser_fuzzers/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""SMT-assisted fuzzing evaluation helpers for CUPS/cups-filters.""" - -__version__ = "0.1.0" diff --git a/parser-fuzzers/src/parser_fuzzers/afl.py b/parser-fuzzers/src/parser_fuzzers/afl.py deleted file mode 100644 index 18b1cbc..0000000 --- a/parser-fuzzers/src/parser_fuzzers/afl.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.afl_integration.afl`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.afl_integration.afl") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/afl_dynamic_bridge.py b/parser-fuzzers/src/parser_fuzzers/afl_dynamic_bridge.py deleted file mode 100644 index ea53f20..0000000 --- a/parser-fuzzers/src/parser_fuzzers/afl_dynamic_bridge.py +++ /dev/null @@ -1,385 +0,0 @@ -from __future__ import annotations - -import json -import re -import shutil -from pathlib import Path -from typing import Any - - -MAGIC = b"SMT_PWG_BUNDLE_V1\n" -PPD_MARK = b"--SMT-PPD--\n" -OPTIONS_MARK = b"--SMT-OPTIONS--\n" -DOCUMENT_MARK = b"--SMT-DOCUMENT--\n" - -PPD_OPTION_VALUES = { - "PageSize": "Letter", - "PageRegion": "Letter", - "ColorModel": "RGB", - "Duplex": "None", - "MediaType": "Plain", - "PrintQuality": "Normal", - "Resolution": "300dpi", - "HWResolution": "300dpi", - "cupsBitsPerPixel": "24", - "cupsBytesPerLine": "24", - "cupsColorOrder": "0", - "cupsColorSpace": "19", - "cupsCompression": "0", - "cupsManualCopies": "True", - "cupsFilter": "application/vnd.cups-pwg application/pdf 0 -", -} - -STATIC_BUNDLE_TOKENS = [ - "SMT_PWG_BUNDLE_V1", - "--SMT-PPD--", - "--SMT-OPTIONS--", - "--SMT-DOCUMENT--", - "*PPD-Adobe:", - "*cupsFilter2:", - "*OpenUI", - "*CloseUI", - "PageSize=", - "PageRegion=", - "ColorModel=", - "PrintQuality=", - "MediaType=", - "Duplex=", - "Resolution=", - "application/vnd.cups-pwg", - "application/pdf", - "2SaR", - "RaS2", -] - -TARGET_RELEVANT_PATTERNS = [ - re.compile(r"^(?:application|image|text|printer)/[A-Za-z0-9_.+/-]+$"), - re.compile(r"^(?:cups|media|print|page|color|duplex|resolution|orientation|output)-[A-Za-z0-9_.-]+$"), - re.compile(r"^(?:cups[A-Z]|Page|Color|Duplex|Media|Print|Resolution|HWResolution)[A-Za-z0-9]*$"), -] - - -def load_dynamic_profile(path: str | Path) -> dict[str, Any]: - return json.loads(Path(path).read_text(encoding="utf-8")) - - -def extract_dynamic_tokens(profile: dict[str, Any], *, max_tokens: int = 512) -> list[str]: - weighted: dict[str, int] = {} - - def add_token(raw: Any, weight: int = 1) -> None: - token = _normalize_token(raw) - if not token or not _target_relevant_token(token): - return - weighted[token] = weighted.get(token, 0) + max(1, weight) - for variant in _token_variants(token): - weighted[variant] = weighted.get(variant, 0) + max(1, weight // 2) - - for section in ("tokens", "ppd_options", "magic_tokens"): - values = profile.get(section) - if isinstance(values, dict): - for token, count in values.items(): - add_token(token, _safe_int(count, 1)) - - records = profile.get("records") - if isinstance(records, list): - for record in records[: max(0, max_tokens * 4)]: - if not isinstance(record, dict): - continue - for token in record.get("tokens") or []: - add_token(token, 1) - for key in ("a_ascii", "b_ascii"): - value = str(record.get(key) or "") - if _target_relevant_token(value): - add_token(value, 1) - - for token in STATIC_BUNDLE_TOKENS: - add_token(token, 16) - - return [ - token - for token, _ in sorted( - weighted.items(), - key=lambda item: (-item[1], len(item[0]), item[0]), - )[:max_tokens] - ] - - -def write_dynamic_afl_dictionary( - profile_path: str | Path, - output_path: str | Path, - *, - base_dictionary: str | Path | None = None, - max_tokens: int = 512, -) -> dict[str, Any]: - profile = load_dynamic_profile(profile_path) - tokens = extract_dynamic_tokens(profile, max_tokens=max_tokens) - output = Path(output_path) - output.parent.mkdir(parents=True, exist_ok=True) - - lines: list[str] = [] - seen_values: set[str] = set() - if base_dictionary: - base_path = Path(base_dictionary) - if base_path.exists(): - for line in base_path.read_text(encoding="utf-8").splitlines(): - value = _parse_afl_dict_value(line) - if value: - seen_values.add(value) - lines.append(line) - - lines.append("# dynamic compare tokens") - added = 0 - for token in tokens: - if token in seen_values: - continue - lines.append(_afl_quote(token)) - seen_values.add(token) - added += 1 - - output.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") - return { - "profile_path": str(profile_path), - "output_path": str(output), - "base_dictionary": str(base_dictionary) if base_dictionary else "", - "tokens_extracted": len(tokens), - "tokens_added": added, - "dictionary_entries": len([line for line in lines if _parse_afl_dict_value(line)]), - } - - -def augment_pwg_bundle_seed_dir( - seed_dir: str | Path, - profile_path: str | Path, - *, - output_dir: str | Path | None = None, - limit: int = 64, -) -> dict[str, Any]: - seed_root = Path(seed_dir) - destination = Path(output_dir) if output_dir else seed_root - destination.mkdir(parents=True, exist_ok=True) - if destination != seed_root: - for seed in sorted(seed_root.glob("*.pwg-bundle")): - shutil.copy2(seed, destination / seed.name) - - profile = load_dynamic_profile(profile_path) - tokens = extract_dynamic_tokens(profile, max_tokens=max(16, limit * 2)) - option_names = _dynamic_option_names(profile, tokens) - base_seeds = sorted(destination.glob("*.pwg-bundle")) - if not base_seeds: - raise FileNotFoundError(f"no .pwg-bundle seeds found in {destination}") - - created: list[dict[str, str]] = [] - for idx, option in enumerate(option_names): - if len(created) >= limit: - break - base = base_seeds[idx % len(base_seeds)] - bundle = parse_pwg_bundle(base.read_bytes()) - if bundle is None: - continue - ppd, options, document = bundle - value = PPD_OPTION_VALUES.get(option, "1") - out = destination / f"dynamic-option-{idx:04d}-{_safe_name(option)}.pwg-bundle" - out.write_bytes( - compose_pwg_bundle( - ppd + _ppd_dynamic_block(option, value), - _append_job_option(options, option, value), - document, - ) - ) - created.append({"path": str(out), "source": str(base), "option": option, "value": value}) - - if len(created) < limit and tokens: - base = base_seeds[0] - bundle = parse_pwg_bundle(base.read_bytes()) - if bundle is not None: - ppd, options, document = bundle - rich_options = option_names[:16] - out = destination / "dynamic-rich-compare-profile.pwg-bundle" - for option in rich_options: - ppd += _ppd_dynamic_block(option, PPD_OPTION_VALUES.get(option, "1")) - options = _append_job_option(options, option, PPD_OPTION_VALUES.get(option, "1")) - for token in tokens[:32]: - if token.startswith(("application/", "image/", "text/")): - ppd += f'\n*cupsFilter2: "{token} application/pdf 0 -"\n'.encode() - out.write_bytes(compose_pwg_bundle(ppd, options, document)) - created.append({"path": str(out), "source": str(base), "option": "rich", "value": "dynamic-profile"}) - - manifest = { - "profile_path": str(profile_path), - "seed_dir": str(seed_root), - "output_dir": str(destination), - "requested_limit": limit, - "created": len(created), - "variants": created, - } - (destination / "dynamic_seed_manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - return manifest - - -def parse_pwg_bundle(data: bytes) -> tuple[bytes, bytes, bytes] | None: - if not data.startswith(MAGIC): - return None - ppd_mark = data.find(PPD_MARK) - options_mark = data.find(OPTIONS_MARK) - document_mark = data.find(DOCUMENT_MARK) - if ppd_mark < 0 or options_mark < 0 or document_mark < 0: - return None - if not (ppd_mark < options_mark < document_mark): - return None - ppd_start = ppd_mark + len(PPD_MARK) - options_start = options_mark + len(OPTIONS_MARK) - document_start = document_mark + len(DOCUMENT_MARK) - return data[ppd_start:options_mark], data[options_start:document_mark], data[document_start:] - - -def compose_pwg_bundle(ppd: bytes, options: bytes, document: bytes) -> bytes: - return MAGIC + PPD_MARK + ppd + b"\n" + OPTIONS_MARK + options + b"\n" + DOCUMENT_MARK + document - - -def _dynamic_option_names(profile: dict[str, Any], tokens: list[str]) -> list[str]: - names: list[str] = [] - ppd_options = profile.get("ppd_options") - if isinstance(ppd_options, dict): - names.extend(str(name) for name in ppd_options) - for token in tokens: - bare = token.strip("*:=").split("=", 1)[0] - if bare in PPD_OPTION_VALUES: - names.append(bare) - for fallback in PPD_OPTION_VALUES: - names.append(fallback) - deduped: list[str] = [] - seen: set[str] = set() - for name in names: - if name in seen or name not in PPD_OPTION_VALUES: - continue - deduped.append(name) - seen.add(name) - return deduped - - -def _ppd_dynamic_block(option: str, value: str) -> bytes: - if option == "PageSize": - text = """ -*DefaultPageSize: Letter -*PageSize Letter/Letter: "<>setpagedevice" -""" - elif option == "PageRegion": - text = """ -*DefaultPageRegion: Letter -*PageRegion Letter/Letter: "<>setpagedevice" -""" - elif option == "Resolution": - text = """ -*DefaultResolution: 300dpi -*Resolution 300dpi/300 dpi: "" -""" - elif option == "HWResolution": - text = """ -*DefaultHWResolution: 300dpi -*HWResolution 300dpi/300 dpi: "" -""" - elif option == "ColorModel": - text = """ -*DefaultColorModel: RGB -*ColorModel RGB/RGB: "" -""" - elif option == "Duplex": - text = """ -*DefaultDuplex: None -*Duplex None/Off: "" -""" - elif option == "MediaType": - text = """ -*DefaultMediaType: Plain -*MediaType Plain/Plain: "" -""" - elif option == "PrintQuality": - text = """ -*DefaultPrintQuality: Normal -*PrintQuality Normal/Normal: "" -""" - elif option == "cupsFilter": - text = f'\n*cupsFilter2: "{value}"\n' - else: - text = f"\n*{option}: {value}\n" - return text.encode("utf-8") - - -def _append_job_option(options: bytes, option: str, value: str) -> bytes: - if option == "cupsFilter": - return options - clean = options.strip() - fragment = f"{option}={value}".encode("utf-8") - return fragment if not clean else clean + b" " + fragment - - -def _token_variants(token: str) -> list[str]: - variants: list[str] = [] - if token in PPD_OPTION_VALUES: - variants.extend([f"{token}=", f"*{token}:", f"*Default{token}:"]) - elif token.startswith("cups") and re.match(r"^[A-Za-z][A-Za-z0-9]+$", token): - variants.extend([f"{token}=", f"*{token}:"]) - return variants - - -def _target_relevant_token(token: str) -> bool: - if token in STATIC_BUNDLE_TOKENS or token in PPD_OPTION_VALUES: - return True - if not 2 <= len(token) <= 96: - return False - if any(ord(ch) < 32 or ord(ch) > 126 for ch in token): - return False - if any(pattern.match(token) for pattern in TARGET_RELEVANT_PATTERNS): - return True - return token in {"2SaR", "RaS2", "RGB", "CMYK", "Gray", "Black", "Letter", "A4"} - - -def _normalize_token(raw: Any) -> str: - token = str(raw or "").strip().strip("\x00") - token = token.replace("\x00", "") - token = re.sub(r"[.]{2,}$", "", token) - token = token.strip() - return token - - -def _afl_quote(token: str) -> str: - escaped = [] - for byte in token.encode("utf-8", errors="replace"): - ch = chr(byte) - if ch == "\\": - escaped.append("\\\\") - elif ch == '"': - escaped.append('\\"') - elif 32 <= byte <= 126: - escaped.append(ch) - else: - escaped.append(f"\\x{byte:02x}") - return '"' + "".join(escaped) + '"' - - -def _parse_afl_dict_value(line: str) -> str: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - return "" - if stripped.startswith('"') and stripped.endswith('"') and len(stripped) >= 2: - return stripped[1:-1] - if "=" in stripped: - _, value = stripped.split("=", 1) - value = value.strip() - if value.startswith('"') and value.endswith('"') and len(value) >= 2: - return value[1:-1] - return "" - - -def _safe_int(value: Any, default: int) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default - - -def _safe_name(value: str) -> str: - return re.sub(r"[^A-Za-z0-9_.-]+", "_", value)[:80] diff --git a/parser-fuzzers/src/parser_fuzzers/afl_feedback.py b/parser-fuzzers/src/parser_fuzzers/afl_feedback.py deleted file mode 100644 index abed806..0000000 --- a/parser-fuzzers/src/parser_fuzzers/afl_feedback.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.afl_integration.afl_feedback`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.afl_integration.afl_feedback") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/afl_integration/__init__.py b/parser-fuzzers/src/parser_fuzzers/afl_integration/__init__.py deleted file mode 100644 index b05b17c..0000000 --- a/parser-fuzzers/src/parser_fuzzers/afl_integration/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Layer package for parser-fuzzers.""" diff --git a/parser-fuzzers/src/parser_fuzzers/afl_integration/afl.py b/parser-fuzzers/src/parser_fuzzers/afl_integration/afl.py deleted file mode 100644 index 718357f..0000000 --- a/parser-fuzzers/src/parser_fuzzers/afl_integration/afl.py +++ /dev/null @@ -1,297 +0,0 @@ -from __future__ import annotations - -import json -import os -import shlex -import shutil -import subprocess -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any - -from parser_fuzzers.experiment import ExperimentRow, load_experiment_rows -from parser_fuzzers.validation import load_yaml - - -@dataclass(frozen=True) -class AFLSettings: - fuzzer: str - compiler_cc: str - compiler_cxx: str - input_seed_dir: str - smt_corpus_dir: str - work_dir: str - output_dir: str - timeout_ms: int - memory_mb: int - cmplog_binary_suffix: str - custom_mutator_library: str - env: dict[str, str] - - -@dataclass(frozen=True) -class TargetConfig: - id: str - name: str - dictionaries: list[str] - - -@dataclass(frozen=True) -class AFLPlan: - target_id: str - config_id: str - config_name: str - argv: list[str] - env: dict[str, str] - input_dir: str - output_dir: str - dictionary: str | None - cmplog_binary: str | None - duration_sec: int | None - warnings: list[str] - - def to_dict(self) -> dict[str, Any]: - data = asdict(self) - data["command"] = shlex.join(self.argv) - return data - - -def load_afl_settings(configs_dir: str | Path) -> AFLSettings: - path = Path(configs_dir) / "afl.yaml" - data = load_yaml(path) or {} - afl = data.get("afl", {}) - env = {str(key): str(value) for key, value in (afl.get("env") or {}).items()} - return AFLSettings( - fuzzer=str(afl.get("fuzzer", "afl-fuzz")), - compiler_cc=str(afl.get("compiler_cc", "afl-clang-fast")), - compiler_cxx=str(afl.get("compiler_cxx", "afl-clang-fast++")), - input_seed_dir=str(afl.get("input_seed_dir", "seeds/public")), - smt_corpus_dir=str(afl.get("smt_corpus_dir", "work/corpus/smt")), - work_dir=str(afl.get("work_dir", "work/afl")), - output_dir=str(afl.get("output_dir", "work/afl/out")), - timeout_ms=int(afl.get("timeout_ms", 3000)), - memory_mb=int(afl.get("memory_mb", 1024)), - cmplog_binary_suffix=str(afl.get("cmplog_binary_suffix", ".cmplog")), - custom_mutator_library=str(afl.get("custom_mutator_library", "")), - env=env, - ) - - -def load_target_configs(configs_dir: str | Path) -> dict[str, TargetConfig]: - data = load_yaml(Path(configs_dir) / "targets.yaml") or {} - targets: dict[str, TargetConfig] = {} - for item in data.get("targets", []): - target = TargetConfig( - id=str(item["id"]), - name=str(item.get("name", item["id"])), - dictionaries=[str(path) for path in item.get("dictionaries", [])], - ) - targets[target.id] = target - return targets - - -def resolve_experiment(configs_dir: str | Path, config_ref: str) -> ExperimentRow: - rows = load_experiment_rows(Path(configs_dir) / "experiment.yaml") - for row in rows: - if row.id == config_ref or row.name == config_ref: - return row - known = ", ".join(f"{row.id}/{row.name}" for row in rows) - raise ValueError(f"unknown fuzz config {config_ref!r}; known configs: {known}") - - -def build_afl_plan( - root: str | Path, - configs_dir: str | Path, - target_id: str, - config_ref: str, - binary: str | Path, - *, - input_dir: str | Path | None = None, - output_dir: str | Path | None = None, - timeout_ms: int | None = None, - memory_mb: int | str | None = None, - duration_sec: int | None = None, -) -> AFLPlan: - root_path = Path(root) - configs_path = root_path / configs_dir - settings = load_afl_settings(configs_path) - targets = load_target_configs(configs_path) - if target_id not in targets: - known = ", ".join(sorted(targets)) - raise ValueError(f"unknown target {target_id!r}; known targets: {known}") - - row = resolve_experiment(configs_path, config_ref) - target = targets[target_id] - warnings: list[str] = [] - - if input_dir is None: - input_path, smt_count = prepare_input_corpus(root_path, settings, target.id, row) - else: - input_path = Path(input_dir) - if not input_path.is_absolute(): - input_path = root_path / input_path - smt_count = 0 - if not input_path.exists(): - raise FileNotFoundError(f"AFL++ input directory does not exist: {input_path}") - output_path = Path(output_dir) if output_dir is not None else root_path / settings.output_dir / target.id / row.id - if not output_path.is_absolute(): - output_path = root_path / output_path - output_path.mkdir(parents=True, exist_ok=True) - - dictionary_path: Path | None = None - if row.dictionary: - dictionary_path = prepare_dictionary(root_path, settings, target) - - cmplog_binary: Path | None = None - binary_path = Path(binary) - if row.cmplog: - cmplog_binary = Path(str(binary_path) + settings.cmplog_binary_suffix) - if not cmplog_binary.exists(): - warnings.append(f"CmpLog config requested, but {cmplog_binary} does not exist yet") - - env = dict(settings.env) - if row.grammar: - if settings.custom_mutator_library: - env["AFL_CUSTOM_MUTATOR_LIBRARY"] = settings.custom_mutator_library - else: - warnings.append("grammar config requested, but afl.custom_mutator_library is not configured") - if row.smt and smt_count == 0: - warnings.append(f"SMT config requested, but no files were found in {root_path / settings.smt_corpus_dir}") - - argv = [ - settings.fuzzer, - "-i", - str(input_path), - "-o", - str(output_path), - "-m", - str(memory_mb if memory_mb is not None else settings.memory_mb), - "-t", - str(timeout_ms if timeout_ms is not None else settings.timeout_ms), - ] - if duration_sec is not None and duration_sec > 0: - argv.extend(["-V", str(duration_sec)]) - if dictionary_path is not None: - argv.extend(["-x", str(dictionary_path)]) - if cmplog_binary is not None: - argv.extend(["-c", str(cmplog_binary)]) - argv.extend(["--", str(binary_path), "@@"]) - - return AFLPlan( - target_id=target.id, - config_id=row.id, - config_name=row.name, - argv=argv, - env=env, - input_dir=str(input_path), - output_dir=str(output_path), - dictionary=str(dictionary_path) if dictionary_path is not None else None, - cmplog_binary=str(cmplog_binary) if cmplog_binary is not None else None, - duration_sec=duration_sec, - warnings=warnings, - ) - - -def prepare_dictionary(root: Path, settings: AFLSettings, target: TargetConfig) -> Path: - output_dir = root / settings.work_dir / "dicts" - output_dir.mkdir(parents=True, exist_ok=True) - output_path = output_dir / f"{target.id}.dict" - seen: set[str] = set() - lines: list[str] = [] - for relative_path in target.dictionaries: - dictionary_path = root / relative_path - if not dictionary_path.exists(): - raise FileNotFoundError(f"dictionary not found: {dictionary_path}") - for raw_line in dictionary_path.read_text(encoding="utf-8").splitlines(): - line = raw_line.strip() - if not line or line.startswith("#") or line in seen: - continue - seen.add(line) - lines.append(line) - output_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - return output_path - - -def prepare_input_corpus( - root: Path, - settings: AFLSettings, - target_id: str, - row: ExperimentRow, -) -> tuple[Path, int]: - input_dir = root / settings.work_dir / "input" / target_id / row.id - input_dir.mkdir(parents=True, exist_ok=True) - - seed_dir = root / settings.input_seed_dir - if not seed_dir.exists(): - raise FileNotFoundError(f"seed directory not found: {seed_dir}") - _copy_inputs(seed_dir, input_dir, prefix="seed") - - smt_count = 0 - if row.smt: - smt_dir = root / settings.smt_corpus_dir - if smt_dir.exists(): - smt_count = _copy_inputs(smt_dir, input_dir, prefix="smt") - return input_dir, smt_count - - -def run_afl_plan(plan: AFLPlan, *, execute: bool, require_instrumented: bool = True) -> int: - if not execute: - print_afl_plan(plan, as_json=False) - return 0 - if shutil.which(plan.argv[0]) is None: - print(f"AFL++ fuzzer not found in PATH: {plan.argv[0]}") - return 2 - binary = Path(plan.argv[-2]) - if not binary.exists(): - print(f"target binary does not exist: {binary}") - return 2 - if require_instrumented and not binary_looks_afl_instrumented(binary): - print(f"target binary does not look AFL++ instrumented: {binary}") - print("build it with afl-clang-fast/afl-clang-fast++ or pass --allow-non-instrumented explicitly") - return 2 - env = os.environ.copy() - env.update(plan.env) - return subprocess.run(plan.argv, env=env, check=False).returncode - - -def binary_looks_afl_instrumented(path: str | Path) -> bool: - try: - data = Path(path).read_bytes() - except OSError: - return False - return b"__afl" in data or b"AFL++" in data or b"afl-compiler-rt" in data - - -def print_afl_plan(plan: AFLPlan, *, as_json: bool) -> None: - if as_json: - print(json.dumps(plan.to_dict(), indent=2, sort_keys=True)) - return - for warning in plan.warnings: - print(f"WARNING: {warning}") - if plan.env: - env_prefix = " ".join(f"{key}={shlex.quote(value)}" for key, value in sorted(plan.env.items())) - print(f"{env_prefix} {shlex.join(plan.argv)}") - else: - print(shlex.join(plan.argv)) - - -def afl_build_env(configs_dir: str | Path) -> dict[str, str]: - settings = load_afl_settings(configs_dir) - return { - "CC": settings.compiler_cc, - "CXX": settings.compiler_cxx, - "CFLAGS": "-g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer", - "CXXFLAGS": "-g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer", - "LDFLAGS": "-fsanitize=address,undefined", - } - - -def _copy_inputs(source_dir: Path, destination_dir: Path, *, prefix: str) -> int: - count = 0 - for source in sorted(source_dir.iterdir()): - if not source.is_file() or source.name.startswith("."): - continue - destination = destination_dir / f"{prefix}-{source.name}" - shutil.copy2(source, destination) - count += 1 - return count diff --git a/parser-fuzzers/src/parser_fuzzers/afl_integration/afl_feedback.py b/parser-fuzzers/src/parser_fuzzers/afl_integration/afl_feedback.py deleted file mode 100644 index eaa2c71..0000000 --- a/parser-fuzzers/src/parser_fuzzers/afl_integration/afl_feedback.py +++ /dev/null @@ -1,251 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import shutil -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Iterable - - -@dataclass(frozen=True) -class AFLImportedCase: - source: str - source_path: str - target_id: str - case_id: int - document_path: str - crashed: bool - sha256: str - - -@dataclass(frozen=True) -class AFLImportSummary: - afl_instance_dir: str - output_run_dir: str - target_id: str - queue_mode: str - queue_imported: int - crashes_imported: int - duplicates_skipped: int - imported: list[AFLImportedCase] - - def to_dict(self) -> dict[str, object]: - return asdict(self) - - -def import_afl_artifacts( - *, - afl_out: str | Path, - target_id: str, - output_run_dir: str | Path, - extension: str = ".pwg", - queue_limit: int = 512, - crash_limit: int = 128, - queue_mode: str = "new", -) -> AFLImportSummary: - instance_dir = resolve_afl_instance_dir(afl_out) - output_root = Path(output_run_dir) - output_root.mkdir(parents=True, exist_ok=True) - normalized_queue_mode = _normalize_queue_mode(queue_mode) - - imported: list[AFLImportedCase] = [] - seen_hashes: set[str] = set() - duplicates_skipped = 0 - next_case_id = 0 - - queue_imported, next_case_id, duplicates = _import_group( - source_files=_queue_files(instance_dir / "queue", normalized_queue_mode), - output_root=output_root, - target_id=target_id, - extension=extension, - source_label="afl-queue", - crashed=False, - limit=max(0, queue_limit), - start_case_id=next_case_id, - seen_hashes=seen_hashes, - imported=imported, - ) - duplicates_skipped += duplicates - - crashes_imported, next_case_id, duplicates = _import_group( - source_files=_iter_afl_files(instance_dir / "crashes"), - output_root=output_root, - target_id=target_id, - extension=extension, - source_label="afl-crash", - crashed=True, - limit=max(0, crash_limit), - start_case_id=next_case_id, - seen_hashes=seen_hashes, - imported=imported, - ) - duplicates_skipped += duplicates - - summary = AFLImportSummary( - afl_instance_dir=str(instance_dir), - output_run_dir=str(output_root), - target_id=target_id, - queue_mode=normalized_queue_mode, - queue_imported=queue_imported, - crashes_imported=crashes_imported, - duplicates_skipped=duplicates_skipped, - imported=imported, - ) - (output_root / "afl_import_manifest.json").write_text( - json.dumps(summary.to_dict(), indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - _write_timeline(output_root, imported) - return summary - - -def resolve_afl_instance_dir(path: str | Path) -> Path: - root = Path(path) - candidates = [ - root, - root / "default", - root / "out" / "default", - ] - for candidate in candidates: - if (candidate / "queue").exists() or (candidate / "crashes").exists(): - return candidate - raise FileNotFoundError(f"could not locate AFL++ queue/crashes under {root}") - - -def _import_group( - *, - source_files: Iterable[Path], - output_root: Path, - target_id: str, - extension: str, - source_label: str, - crashed: bool, - limit: int, - start_case_id: int, - seen_hashes: set[str], - imported: list[AFLImportedCase], -) -> tuple[int, int, int]: - count = 0 - duplicates = 0 - case_id = start_case_id - if limit <= 0: - return 0, case_id, 0 - for source_path in source_files: - digest = _sha256_file(source_path) - if digest in seen_hashes: - duplicates += 1 - continue - seen_hashes.add(digest) - case_dir = _case_dir(output_root, target_id, case_id, crashed) - case_dir.mkdir(parents=True, exist_ok=True) - document_path = case_dir / f"document{extension}" - shutil.copy2(source_path, document_path) - record = AFLImportedCase( - source=source_label, - source_path=str(source_path), - target_id=target_id, - case_id=case_id, - document_path=str(document_path), - crashed=crashed, - sha256=digest, - ) - _write_meta(case_dir, record) - imported.append(record) - count += 1 - case_id += 1 - if count >= limit: - break - return count, case_id, duplicates - - -def _case_dir(output_root: Path, target_id: str, case_id: int, crashed: bool) -> Path: - if crashed: - return output_root / "quarantine" / "unique" / f"{target_id}-afl-case-{case_id:06d}" - return output_root / "corpus" / "interesting" / target_id / f"case-{case_id:06d}" - - -def _write_meta(case_dir: Path, record: AFLImportedCase) -> None: - meta = { - "case_id": record.case_id, - "crashed": record.crashed, - "document_kind": "afl_import", - "document_path": str(Path(record.document_path).resolve()), - "oracle": "afl-crash" if record.crashed else "", - "source": record.source, - "source_path": record.source_path, - "target_id": record.target_id, - "timed_out": False, - "work_dir": str(case_dir.resolve()), - } - (case_dir / "meta.json").write_text( - json.dumps(meta, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - - -def _write_timeline(output_root: Path, imported: list[AFLImportedCase]) -> None: - timeline = output_root / "timeline.jsonl" - with timeline.open("w", encoding="utf-8") as handle: - for item in imported: - handle.write( - json.dumps( - { - "case_id": item.case_id, - "crashed": item.crashed, - "document_path": item.document_path, - "oracle": "afl-crash" if item.crashed else "", - "source": item.source, - "target_id": item.target_id, - "timed_out": False, - }, - sort_keys=True, - ) - + "\n" - ) - - -def _iter_afl_files(root: Path) -> list[Path]: - if not root.exists(): - return [] - files = [] - for path in sorted(root.iterdir()): - if not path.is_file(): - continue - if path.name.startswith(".") or path.name == "README.txt": - continue - files.append(path) - return files - - -def _queue_files(root: Path, mode: str) -> list[Path]: - if mode == "none": - return [] - files = _iter_afl_files(root) - if mode == "all": - return files - return [path for path in files if _is_afl_discovered_queue_entry(path)] - - -def _is_afl_discovered_queue_entry(path: Path) -> bool: - name = path.name - return "src:" in name or "sync:" in name or "splice" in name - - -def _normalize_queue_mode(mode: str) -> str: - normalized = mode.strip().lower() - if normalized in {"new", "discovered", "mutations", "mutated"}: - return "new" - if normalized in {"all", "everything"}: - return "all" - if normalized in {"none", "off", "0"}: - return "none" - raise ValueError(f"unknown AFL queue import mode: {mode}") - - -def _sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() diff --git a/parser-fuzzers/src/parser_fuzzers/afl_integration/seed_export.py b/parser-fuzzers/src/parser_fuzzers/afl_integration/seed_export.py deleted file mode 100644 index 7e079a2..0000000 --- a/parser-fuzzers/src/parser_fuzzers/afl_integration/seed_export.py +++ /dev/null @@ -1,155 +0,0 @@ -from __future__ import annotations - -import json -import shutil -from dataclasses import asdict, dataclass, field -from pathlib import Path -from typing import Iterable - - -@dataclass(frozen=True) -class ExportedSeed: - source_case_dir: str - source_path: str - target_id: str - output_path: str - - -@dataclass(frozen=True) -class SeedExportSummary: - run_dir: str - output_dir: str - targets: list[str] - extensions: list[str] - exported: int - exported_by_target: dict[str, int] = field(default_factory=dict) - seeds: list[ExportedSeed] = field(default_factory=list) - manifest_path: str = "" - - def to_dict(self) -> dict[str, object]: - return asdict(self) - - -def export_template_seeds( - *, - run_dir: str | Path, - output_dir: str | Path, - target_ids: Iterable[str] | None = None, - extensions: Iterable[str] | None = None, - limit: int = 0, - include_crashes: bool = False, - include_ppd: bool = False, - manifest_name: str = "seed_export_manifest.json", -) -> SeedExportSummary: - root = Path(run_dir) - out = Path(output_dir) - out.mkdir(parents=True, exist_ok=True) - - target_filter = {item for item in (target_ids or []) if item} - extension_filter = {_normalize_extension(item) for item in (extensions or []) if item} - exported: list[ExportedSeed] = [] - by_target: dict[str, int] = {} - - for target_id, case_dir in _iter_case_dirs(root, target_filter, include_crashes=include_crashes): - for source in _iter_seed_files(case_dir, extension_filter, include_ppd=include_ppd): - if limit > 0 and len(exported) >= limit: - break - suffix = source.suffix or ".bin" - destination = out / f"{target_id}-{case_dir.name}-{len(exported):06d}{suffix}" - shutil.copy2(source, destination) - exported.append( - ExportedSeed( - source_case_dir=str(case_dir), - source_path=str(source), - target_id=target_id, - output_path=str(destination), - ) - ) - by_target[target_id] = by_target.get(target_id, 0) + 1 - if limit > 0 and len(exported) >= limit: - break - - manifest_path = _manifest_path_for_seed_dir(out, manifest_name) - summary = SeedExportSummary( - run_dir=str(root), - output_dir=str(out), - targets=sorted(target_filter), - extensions=sorted(extension_filter), - exported=len(exported), - exported_by_target=dict(sorted(by_target.items())), - seeds=exported, - manifest_path=str(manifest_path), - ) - manifest_path.write_text(json.dumps(summary.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") - return summary - - -def _iter_case_dirs( - root: Path, - target_filter: set[str], - *, - include_crashes: bool, -) -> Iterable[tuple[str, Path]]: - interesting_root = root / "corpus" / "interesting" - if interesting_root.exists(): - for target_dir in sorted(interesting_root.iterdir()): - if not target_dir.is_dir(): - continue - target_id = target_dir.name - if target_filter and target_id not in target_filter: - continue - for case_dir in sorted(target_dir.glob("case-*")): - if case_dir.is_dir(): - yield target_id, case_dir - - if include_crashes: - quarantine_root = root / "quarantine" / "unique" - if quarantine_root.exists(): - for case_dir in sorted(quarantine_root.iterdir()): - if not case_dir.is_dir(): - continue - target_id = _target_from_quarantine_case(case_dir.name) - if target_filter and target_id not in target_filter: - continue - yield target_id, case_dir - - -def _iter_seed_files(case_dir: Path, extension_filter: set[str], *, include_ppd: bool) -> Iterable[Path]: - patterns = ["document*"] - if include_ppd: - patterns.append("candidate.ppd") - seen: set[Path] = set() - for pattern in patterns: - for path in sorted(case_dir.glob(pattern)): - if not path.is_file() or path in seen: - continue - seen.add(path) - if extension_filter and _normalize_extension(path.suffix or ".bin") not in extension_filter: - continue - yield path - - -def _normalize_extension(value: str) -> str: - value = value.strip() - if not value: - return "" - return value if value.startswith(".") else f".{value}" - - -def _manifest_path_for_seed_dir(seed_dir: Path, manifest_name: str) -> Path: - requested = Path(manifest_name) - if requested.is_absolute(): - return requested - if requested.parent != Path("."): - return seed_dir / requested - return seed_dir.parent / f"{seed_dir.name}-{manifest_name}" - - -def _target_from_quarantine_case(name: str) -> str: - marker = "-case-" - if marker in name: - return name.split(marker, 1)[0] - marker = "-afl-case-" - if marker in name: - return name.split(marker, 1)[0] - return name diff --git a/parser-fuzzers/src/parser_fuzzers/afl_integration/template_seed_generation.py b/parser-fuzzers/src/parser_fuzzers/afl_integration/template_seed_generation.py deleted file mode 100644 index fb42d26..0000000 --- a/parser-fuzzers/src/parser_fuzzers/afl_integration/template_seed_generation.py +++ /dev/null @@ -1,106 +0,0 @@ -from __future__ import annotations - -import json -from dataclasses import asdict, dataclass, field -from pathlib import Path - -from parser_fuzzers.document_harness import make_document - - -@dataclass(frozen=True) -class GeneratedTemplateSeed: - case_index: int - document_kind: str - extension: str - mime: str - description: str - output_path: str - size: int - - -@dataclass(frozen=True) -class TemplateSeedGenerationSummary: - document_kind: str - target_id: str - output_dir: str - count: int - start_index: int - extension_filter: list[str] - generated: int - seeds: list[GeneratedTemplateSeed] = field(default_factory=list) - manifest_path: str = "" - - def to_dict(self) -> dict[str, object]: - return asdict(self) - - -def generate_template_seeds( - *, - document_kind: str, - output_dir: str | Path, - count: int, - target_id: str = "", - start_index: int = 0, - extensions: list[str] | None = None, - manifest_name: str = "template_seed_manifest.json", -) -> TemplateSeedGenerationSummary: - if count <= 0: - raise ValueError("count must be positive") - out = Path(output_dir) - out.mkdir(parents=True, exist_ok=True) - extension_filter = sorted({_normalize_extension(item) for item in (extensions or []) if item}) - generated: list[GeneratedTemplateSeed] = [] - case_index = max(0, start_index) - max_attempts = count * 32 - attempts = 0 - - while len(generated) < count and attempts < max_attempts: - attempts += 1 - document = make_document(document_kind, case_index, target_id=target_id) - extension = _normalize_extension(document.extension or ".bin") - if not extension_filter or extension in extension_filter: - output_path = out / f"{document_kind}-{case_index:06d}{extension}" - output_path.write_bytes(document.data) - generated.append( - GeneratedTemplateSeed( - case_index=case_index, - document_kind=document.kind, - extension=extension, - mime=document.mime, - description=document.description, - output_path=str(output_path), - size=len(document.data), - ) - ) - case_index += 1 - - manifest_path = _manifest_path_for_seed_dir(out, manifest_name) - summary = TemplateSeedGenerationSummary( - document_kind=document_kind, - target_id=target_id, - output_dir=str(out), - count=count, - start_index=max(0, start_index), - extension_filter=extension_filter, - generated=len(generated), - seeds=generated, - manifest_path=str(manifest_path), - ) - manifest_path.write_text(json.dumps(summary.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") - return summary - - -def _normalize_extension(value: str) -> str: - value = value.strip() - if not value: - return "" - return value if value.startswith(".") else f".{value}" - - -def _manifest_path_for_seed_dir(seed_dir: Path, manifest_name: str) -> Path: - requested = Path(manifest_name) - if requested.is_absolute(): - return requested - if requested.parent != Path("."): - return seed_dir / requested - return seed_dir.parent / f"{seed_dir.name}-{manifest_name}" diff --git a/parser-fuzzers/src/parser_fuzzers/arithmetic_explorer.py b/parser-fuzzers/src/parser_fuzzers/arithmetic_explorer.py deleted file mode 100644 index 709ef83..0000000 --- a/parser-fuzzers/src/parser_fuzzers/arithmetic_explorer.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.arithmetic_explorer`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.arithmetic_explorer") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/auto_expand.py b/parser-fuzzers/src/parser_fuzzers/auto_expand.py deleted file mode 100644 index b0d2491..0000000 --- a/parser-fuzzers/src/parser_fuzzers/auto_expand.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.auto_expand`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.auto_expand") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/baseline_compare.py b/parser-fuzzers/src/parser_fuzzers/baseline_compare.py deleted file mode 100644 index 40375b5..0000000 --- a/parser-fuzzers/src/parser_fuzzers/baseline_compare.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.metrics.baseline_compare`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.metrics.baseline_compare") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/cli.py b/parser-fuzzers/src/parser_fuzzers/cli.py deleted file mode 100644 index e383734..0000000 --- a/parser-fuzzers/src/parser_fuzzers/cli.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.runner.cli`.""" - -from __future__ import annotations - -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.runner.cli") -globals().update( - { - key: value - for key, value in _impl.__dict__.items() - if key not in {"__name__", "__package__", "__loader__", "__spec__"} - } -) - -if __name__ == "__main__": - raise SystemExit(_impl.main()) diff --git a/parser-fuzzers/src/parser_fuzzers/constraint_repair.py b/parser-fuzzers/src/parser_fuzzers/constraint_repair.py deleted file mode 100644 index dd51471..0000000 --- a/parser-fuzzers/src/parser_fuzzers/constraint_repair.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.constraint_repair`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.constraint_repair") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/core/__init__.py b/parser-fuzzers/src/parser_fuzzers/core/__init__.py deleted file mode 100644 index b05b17c..0000000 --- a/parser-fuzzers/src/parser_fuzzers/core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Layer package for parser-fuzzers.""" diff --git a/parser-fuzzers/src/parser_fuzzers/core/experiment.py b/parser-fuzzers/src/parser_fuzzers/core/experiment.py deleted file mode 100644 index 03b2eb7..0000000 --- a/parser-fuzzers/src/parser_fuzzers/core/experiment.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -from parser_fuzzers.validation import load_yaml - - -@dataclass(frozen=True) -class ExperimentRow: - id: str - name: str - description: str - dictionary: bool - cmplog: bool - grammar: bool - smt: bool - - -def load_experiment_rows(config_path: str | Path) -> list[ExperimentRow]: - data = load_yaml(config_path) or {} - rows = [] - for item in data.get("fuzz_configs", []): - rows.append( - ExperimentRow( - id=str(item["id"]), - name=str(item["name"]), - description=str(item.get("description", "")), - dictionary=bool(item.get("dictionary", False)), - cmplog=bool(item.get("cmplog", False)), - grammar=bool(item.get("grammar", False)), - smt=bool(item.get("smt", False)), - ) - ) - return rows - - -def estimate_cpu_hours(config_count: int, target_count: int, repetitions: int, hours: int) -> int: - return config_count * target_count * repetitions * hours diff --git a/parser-fuzzers/src/parser_fuzzers/core/format_specs.py b/parser-fuzzers/src/parser_fuzzers/core/format_specs.py deleted file mode 100644 index a74164c..0000000 --- a/parser-fuzzers/src/parser_fuzzers/core/format_specs.py +++ /dev/null @@ -1,336 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -IMAGE_FORMATS = ("png_gray", "png_rgb", "png_rgba", "ppm", "pgm", "pbm") - - -@dataclass(frozen=True) -class ImageGoal: - name: str - target_family: str - output_format: str - allowed_formats: tuple[str, ...] - min_width: int = 1 - min_height: int = 1 - min_area: int = 1 - max_width: int = 1024 - max_height: int = 256 - aspect: str = "any" - payload_policy: str = "exact" - maxval_class: str = "byte" - png_interlace: int = 0 - comment_style: int | None = None - - -IMAGE_GOALS: tuple[ImageGoal, ...] = ( - ImageGoal( - name="pdf-single-image", - target_family="imagetopdf", - output_format="pdf", - allowed_formats=("png_rgb", "png_rgba", "ppm"), - min_width=96, - min_height=16, - min_area=3072, - ), - ImageGoal( - name="pdf-wide-image", - target_family="imagetopdf", - output_format="pdf", - allowed_formats=("png_rgb", "ppm"), - min_width=256, - min_height=8, - aspect="wide", - ), - ImageGoal( - name="pdf-tall-image", - target_family="imagetopdf", - output_format="pdf", - allowed_formats=("png_gray", "pgm"), - min_width=32, - min_height=96, - aspect="tall", - ), - ImageGoal( - name="pdf-alpha-image", - target_family="imagetopdf", - output_format="pdf", - allowed_formats=("png_rgba",), - min_width=64, - min_height=32, - ), - ImageGoal( - name="pdf-gray-image", - target_family="imagetopdf", - output_format="pdf", - allowed_formats=("png_gray", "pgm"), - min_width=128, - min_height=16, - ), - ImageGoal( - name="pdf-bitmap-image", - target_family="imagetopdf", - output_format="pdf", - allowed_formats=("pbm",), - min_width=63, - min_height=31, - comment_style=1, - ), - ImageGoal( - name="pdf-interlaced-png", - target_family="imagetopdf", - output_format="pdf", - allowed_formats=("png_rgb", "png_rgba"), - min_width=96, - min_height=32, - png_interlace=1, - ), - ImageGoal( - name="pdf-wide-maxval", - target_family="imagetopdf", - output_format="pdf", - allowed_formats=("ppm", "pgm"), - min_width=96, - min_height=16, - maxval_class="wide", - ), - ImageGoal( - name="pdf-low-maxval", - target_family="imagetopdf", - output_format="pdf", - allowed_formats=("ppm", "pgm"), - min_width=127, - min_height=17, - maxval_class="low", - ), - ImageGoal( - name="pdf-large-rgb", - target_family="imagetopdf", - output_format="pdf", - allowed_formats=("png_rgb", "ppm"), - min_width=512, - min_height=64, - min_area=32768, - ), - ImageGoal( - name="ps-showpage-image", - target_family="imagetops", - output_format="postscript", - allowed_formats=("png_rgb", "png_rgba", "ppm"), - min_width=96, - min_height=16, - min_area=3072, - ), - ImageGoal( - name="ps-wide-image", - target_family="imagetops", - output_format="postscript", - allowed_formats=("png_rgb", "ppm"), - min_width=256, - min_height=8, - aspect="wide", - ), - ImageGoal( - name="ps-tall-image", - target_family="imagetops", - output_format="postscript", - allowed_formats=("png_gray", "pgm"), - min_width=32, - min_height=96, - aspect="tall", - ), - ImageGoal( - name="ps-commented-pnm", - target_family="imagetops", - output_format="postscript", - allowed_formats=("ppm", "pgm", "pbm"), - min_width=64, - min_height=32, - comment_style=1, - ), - ImageGoal( - name="ps-alpha-image", - target_family="imagetops", - output_format="postscript", - allowed_formats=("png_rgba",), - min_width=64, - min_height=32, - ), - ImageGoal( - name="ps-gray-image", - target_family="imagetops", - output_format="postscript", - allowed_formats=("png_gray", "pgm"), - min_width=128, - min_height=16, - ), - ImageGoal( - name="ps-bitmap-image", - target_family="imagetops", - output_format="postscript", - allowed_formats=("pbm",), - min_width=63, - min_height=31, - comment_style=1, - ), - ImageGoal( - name="ps-interlaced-png", - target_family="imagetops", - output_format="postscript", - allowed_formats=("png_rgb", "png_rgba"), - min_width=96, - min_height=32, - png_interlace=1, - ), - ImageGoal( - name="ps-wide-maxval", - target_family="imagetops", - output_format="postscript", - allowed_formats=("ppm", "pgm"), - min_width=127, - min_height=17, - maxval_class="wide", - ), - ImageGoal( - name="ps-large-rgb", - target_family="imagetops", - output_format="postscript", - allowed_formats=("png_rgb", "ppm"), - min_width=512, - min_height=64, - min_area=32768, - ), - ImageGoal( - name="raster-rgb24", - target_family="imagetoraster", - output_format="cups-raster", - allowed_formats=("png_rgb", "ppm"), - min_width=96, - min_height=16, - min_area=3072, - ), - ImageGoal( - name="raster-alpha32", - target_family="imagetoraster", - output_format="cups-raster", - allowed_formats=("png_rgba",), - min_width=64, - min_height=32, - ), - ImageGoal( - name="raster-gray8", - target_family="imagetoraster", - output_format="cups-raster", - allowed_formats=("png_gray", "pgm"), - min_width=96, - min_height=16, - ), - ImageGoal( - name="raster-wide-rows", - target_family="imagetoraster", - output_format="cups-raster", - allowed_formats=("png_rgb", "ppm"), - min_width=256, - min_height=8, - aspect="wide", - ), - ImageGoal( - name="raster-boundary-pnm", - target_family="imagetoraster", - output_format="cups-raster", - allowed_formats=("ppm", "pgm", "pbm"), - min_width=31, - min_height=31, - comment_style=1, - ), - ImageGoal( - name="raster-bitmap-rows", - target_family="imagetoraster", - output_format="cups-raster", - allowed_formats=("pbm",), - min_width=63, - min_height=31, - comment_style=1, - ), - ImageGoal( - name="raster-interlaced-png", - target_family="imagetoraster", - output_format="cups-raster", - allowed_formats=("png_rgb", "png_rgba"), - min_width=96, - min_height=32, - png_interlace=1, - ), - ImageGoal( - name="raster-wide-maxval", - target_family="imagetoraster", - output_format="cups-raster", - allowed_formats=("ppm", "pgm"), - min_width=127, - min_height=17, - maxval_class="wide", - ), - ImageGoal( - name="raster-low-maxval", - target_family="imagetoraster", - output_format="cups-raster", - allowed_formats=("ppm", "pgm"), - min_width=127, - min_height=17, - maxval_class="low", - ), - ImageGoal( - name="raster-large-rgb", - target_family="imagetoraster", - output_format="cups-raster", - allowed_formats=("png_rgb", "ppm"), - min_width=512, - min_height=64, - min_area=32768, - ), -) - - -def image_goals_for_target(target_id: str) -> tuple[ImageGoal, ...]: - family = _target_family(target_id) - goals = tuple(goal for goal in IMAGE_GOALS if goal.target_family == family) - return goals or IMAGE_GOALS - - -def image_format_id(image_format: str) -> int: - try: - return IMAGE_FORMATS.index(image_format) - except ValueError: - return 0 - - -def image_channels(image_format: str) -> int: - if image_format in {"png_rgb", "ppm"}: - return 3 - if image_format == "png_rgba": - return 4 - return 1 - - -def maxval_for_class(image_format: str, maxval_class: str, salt: int) -> int: - if image_format == "pbm": - return 1 - if maxval_class == "wide": - return 65535 - if maxval_class == "low": - return [1, 2, 15, 31][salt % 4] - return [127, 255][salt % 2] - - -def _target_family(target_id: str) -> str: - if "imagetopdf" in target_id: - return "imagetopdf" - if "imagetops" in target_id: - return "imagetops" - if "imagetoraster" in target_id: - return "imagetoraster" - for suffix in ("_coverage", "_general", "_explore", "_structural", "_feedback"): - if target_id.endswith(suffix): - return target_id.removesuffix(suffix) - return target_id diff --git a/parser-fuzzers/src/parser_fuzzers/core/hashing.py b/parser-fuzzers/src/parser_fuzzers/core/hashing.py deleted file mode 100644 index 1b4c7d3..0000000 --- a/parser-fuzzers/src/parser_fuzzers/core/hashing.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import annotations - -import hashlib -from pathlib import Path - - -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def sha256_file(path: str | Path) -> str: - digest = hashlib.sha256() - with Path(path).open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() diff --git a/parser-fuzzers/src/parser_fuzzers/core/models.py b/parser-fuzzers/src/parser_fuzzers/core/models.py deleted file mode 100644 index 9296548..0000000 --- a/parser-fuzzers/src/parser_fuzzers/core/models.py +++ /dev/null @@ -1,152 +0,0 @@ -from __future__ import annotations - -from dataclasses import asdict, dataclass -from typing import Any - - -SUPPORTED_OPS = {"eq", "ne", "ult", "ule", "ugt", "uge", "slt", "sle", "sgt", "sge"} -SUPPORTED_WIDTHS = {1, 2, 4, 8} - - -def coerce_int(value: Any, field_name: str) -> int: - if isinstance(value, bool): - raise ValueError(f"{field_name} must be an integer, not a boolean") - if isinstance(value, int): - return value - if isinstance(value, str): - return int(value, 0) - raise ValueError(f"{field_name} must be an integer or integer string") - - -@dataclass(frozen=True) -class BranchEvent: - target_id: str - input_path: str - input_sha256: str - offset: int - width: int - endianness: str - signed: bool - op: str - rhs: int - description: str - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> "BranchEvent": - required = { - "target_id", - "input_path", - "input_sha256", - "offset", - "width", - "endianness", - "signed", - "op", - "rhs", - "description", - } - missing = sorted(required - data.keys()) - if missing: - raise ValueError(f"branch event missing required fields: {', '.join(missing)}") - - event = cls( - target_id=str(data["target_id"]), - input_path=str(data["input_path"]), - input_sha256=str(data["input_sha256"]), - offset=coerce_int(data["offset"], "offset"), - width=coerce_int(data["width"], "width"), - endianness=str(data["endianness"]), - signed=bool(data["signed"]), - op=str(data["op"]), - rhs=coerce_int(data["rhs"], "rhs"), - description=str(data["description"]), - ) - event.validate() - return event - - def validate(self) -> None: - if not self.target_id: - raise ValueError("target_id must not be empty") - if self.offset < 0: - raise ValueError("offset must be non-negative") - if self.width not in SUPPORTED_WIDTHS: - raise ValueError(f"width must be one of {sorted(SUPPORTED_WIDTHS)}") - if self.endianness not in {"little", "big"}: - raise ValueError("endianness must be 'little' or 'big'") - if self.op not in SUPPORTED_OPS: - raise ValueError(f"op must be one of {sorted(SUPPORTED_OPS)}") - if len(self.input_sha256) != 64: - raise ValueError("input_sha256 must be a 64-character hex digest") - int(self.input_sha256, 16) - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - -@dataclass(frozen=True) -class Patch: - offset: int - old_hex: str - new_hex: str - width: int - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> "Patch": - required = {"offset", "old_hex", "new_hex", "width"} - missing = sorted(required - data.keys()) - if missing: - raise ValueError(f"patch missing required fields: {', '.join(missing)}") - patch = cls( - offset=coerce_int(data["offset"], "patch.offset"), - old_hex=str(data["old_hex"]), - new_hex=str(data["new_hex"]), - width=coerce_int(data["width"], "patch.width"), - ) - patch.validate() - return patch - - def validate(self) -> None: - if self.offset < 0: - raise ValueError("patch offset must be non-negative") - if self.width not in SUPPORTED_WIDTHS: - raise ValueError(f"patch width must be one of {sorted(SUPPORTED_WIDTHS)}") - expected_hex_len = self.width * 2 - if len(self.old_hex) != expected_hex_len or len(self.new_hex) != expected_hex_len: - raise ValueError("patch hex strings must match width") - bytes.fromhex(self.old_hex) - bytes.fromhex(self.new_hex) - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - -@dataclass(frozen=True) -class SolverResult: - status: str - solver_ms: float - patches: list[Patch] - reason: str - event: BranchEvent - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> "SolverResult": - required = {"status", "solver_ms", "patches", "reason", "event"} - missing = sorted(required - data.keys()) - if missing: - raise ValueError(f"solver result missing required fields: {', '.join(missing)}") - return cls( - status=str(data["status"]), - solver_ms=float(data["solver_ms"]), - patches=[Patch.from_dict(item) for item in data["patches"]], - reason=str(data["reason"]), - event=BranchEvent.from_dict(data["event"]), - ) - - def to_dict(self) -> dict[str, Any]: - return { - "status": self.status, - "solver_ms": self.solver_ms, - "patches": [patch.to_dict() for patch in self.patches], - "reason": self.reason, - "event": self.event.to_dict(), - } diff --git a/parser-fuzzers/src/parser_fuzzers/core/validation.py b/parser-fuzzers/src/parser_fuzzers/core/validation.py deleted file mode 100644 index c9ea4c4..0000000 --- a/parser-fuzzers/src/parser_fuzzers/core/validation.py +++ /dev/null @@ -1,204 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import yaml - - -BUG_REQUIRED_FIELDS = { - "id", - "title", - "component", - "bug_type", - "target_component", - "oracle", - "poc_path", - "known_poc_allowed_in_seed", - "timeout_sec", - "memory_mb", - "report_path", -} -ORACLE_REQUIRED_FIELDS = {"reached", "triggered", "detected"} -EXPECTED_EXPERIMENT_NAMES = { - "vanilla", - "dictionary", - "dictionary+cmplog", - "dictionary+grammar", - "dictionary+grammar+smt", -} - - -@dataclass(frozen=True) -class ValidationIssue: - level: str - path: str - message: str - - -def load_yaml(path: str | Path) -> Any: - with Path(path).open("r", encoding="utf-8") as handle: - return yaml.safe_load(handle) - - -def validate_all( - bugs_dir: str | Path, - configs_dir: str | Path, - *, - require_local_artifacts: bool = True, -) -> list[ValidationIssue]: - issues: list[ValidationIssue] = [] - issues.extend(validate_bug_suite(bugs_dir, require_local_artifacts=require_local_artifacts)) - issues.extend(validate_config_dir(configs_dir)) - return issues - - -def validate_bug_suite( - bugs_dir: str | Path, - *, - require_local_artifacts: bool = True, -) -> list[ValidationIssue]: - root = Path(bugs_dir) - issues: list[ValidationIssue] = [] - if not root.exists(): - return [ValidationIssue("error", str(root), "bug directory does not exist")] - - metadata_files = sorted(root.glob("*/meta.yaml")) - if not metadata_files: - return issues - - seen_ids: set[str] = set() - for path in metadata_files: - issues.extend(validate_bug_metadata(path, seen_ids, require_local_artifacts=require_local_artifacts)) - return issues - - -def validate_bug_metadata( - path: str | Path, - seen_ids: set[str] | None = None, - *, - require_local_artifacts: bool = True, -) -> list[ValidationIssue]: - meta_path = Path(path) - issues: list[ValidationIssue] = [] - try: - data = load_yaml(meta_path) or {} - except Exception as exc: # pragma: no cover - defensive parsing path - return [ValidationIssue("error", str(meta_path), f"failed to parse YAML: {exc}")] - - missing = sorted(BUG_REQUIRED_FIELDS - data.keys()) - for field in missing: - issues.append(ValidationIssue("error", str(meta_path), f"missing field: {field}")) - - bug_id = str(data.get("id", "")) - if seen_ids is not None and bug_id: - if bug_id in seen_ids: - issues.append(ValidationIssue("error", str(meta_path), f"duplicate bug id: {bug_id}")) - seen_ids.add(bug_id) - - oracle = data.get("oracle", {}) - if not isinstance(oracle, dict): - issues.append(ValidationIssue("error", str(meta_path), "oracle must be a mapping")) - else: - for field in sorted(ORACLE_REQUIRED_FIELDS - oracle.keys()): - issues.append(ValidationIssue("error", str(meta_path), f"oracle missing field: {field}")) - - if data.get("known_poc_allowed_in_seed") is not False: - issues.append( - ValidationIssue( - "error", - str(meta_path), - "known_poc_allowed_in_seed must be false for ground-truth evaluation", - ) - ) - - for field in ("poc_path", "report_path"): - value = data.get(field) - if value and not Path(str(value)).exists(): - level = "error" if require_local_artifacts else "warning" - issues.append(ValidationIssue(level, str(meta_path), f"{field} does not exist: {value}")) - - for field in ("timeout_sec", "memory_mb"): - value = data.get(field) - if not isinstance(value, int) or value <= 0: - issues.append(ValidationIssue("error", str(meta_path), f"{field} must be a positive integer")) - return issues - - -def validate_config_dir(configs_dir: str | Path) -> list[ValidationIssue]: - root = Path(configs_dir) - issues: list[ValidationIssue] = [] - if not root.exists(): - return [ValidationIssue("error", str(root), "config directory does not exist")] - - experiment_path = root / "experiment.yaml" - targets_path = root / "targets.yaml" - afl_path = root / "afl.yaml" - if not experiment_path.exists(): - issues.append(ValidationIssue("error", str(experiment_path), "missing experiment.yaml")) - else: - issues.extend(validate_experiment_config(experiment_path)) - if not targets_path.exists(): - issues.append(ValidationIssue("error", str(targets_path), "missing targets.yaml")) - else: - issues.extend(validate_targets_config(targets_path)) - if not afl_path.exists(): - issues.append(ValidationIssue("error", str(afl_path), "missing afl.yaml")) - else: - issues.extend(validate_afl_config(afl_path)) - return issues - - -def validate_experiment_config(path: str | Path) -> list[ValidationIssue]: - config_path = Path(path) - data = load_yaml(config_path) or {} - issues: list[ValidationIssue] = [] - configs = data.get("fuzz_configs") - if not isinstance(configs, list) or not configs: - return [ValidationIssue("error", str(config_path), "fuzz_configs must be a non-empty list")] - names = {str(item.get("name")) for item in configs if isinstance(item, dict)} - missing = sorted(EXPECTED_EXPERIMENT_NAMES - names) - extra = sorted(names - EXPECTED_EXPERIMENT_NAMES) - for name in missing: - issues.append(ValidationIssue("error", str(config_path), f"missing experiment config: {name}")) - for name in extra: - issues.append(ValidationIssue("warning", str(config_path), f"unexpected experiment config: {name}")) - return issues - - -def validate_targets_config(path: str | Path) -> list[ValidationIssue]: - config_path = Path(path) - data = load_yaml(config_path) or {} - targets = data.get("targets") - if not isinstance(targets, list) or not targets: - return [ValidationIssue("error", str(config_path), "targets must be a non-empty list")] - issues: list[ValidationIssue] = [] - for index, target in enumerate(targets): - if not isinstance(target, dict): - issues.append(ValidationIssue("error", str(config_path), f"target {index} must be a mapping")) - continue - for field in ("id", "name", "components"): - if field not in target: - issues.append(ValidationIssue("error", str(config_path), f"target {index} missing {field}")) - return issues - - -def validate_afl_config(path: str | Path) -> list[ValidationIssue]: - config_path = Path(path) - data = load_yaml(config_path) or {} - afl = data.get("afl") - if not isinstance(afl, dict): - return [ValidationIssue("error", str(config_path), "afl must be a mapping")] - issues: list[ValidationIssue] = [] - for field in ("fuzzer", "compiler_cc", "compiler_cxx", "input_seed_dir", "work_dir", "output_dir"): - if not afl.get(field): - issues.append(ValidationIssue("error", str(config_path), f"afl.{field} must be set")) - for field in ("timeout_ms", "memory_mb"): - value = afl.get(field) - if not isinstance(value, int) or value <= 0: - issues.append(ValidationIssue("error", str(config_path), f"afl.{field} must be a positive integer")) - env = afl.get("env", {}) - if env is not None and not isinstance(env, dict): - issues.append(ValidationIssue("error", str(config_path), "afl.env must be a mapping")) - return issues diff --git a/parser-fuzzers/src/parser_fuzzers/crash_avoidance.py b/parser-fuzzers/src/parser_fuzzers/crash_avoidance.py deleted file mode 100644 index 10ce381..0000000 --- a/parser-fuzzers/src/parser_fuzzers/crash_avoidance.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.feedback.crash_avoidance`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.feedback.crash_avoidance") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/crash_dedup.py b/parser-fuzzers/src/parser_fuzzers/crash_dedup.py deleted file mode 100644 index 91bec9c..0000000 --- a/parser-fuzzers/src/parser_fuzzers/crash_dedup.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.feedback.crash_dedup`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.feedback.crash_dedup") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/dimension_expander.py b/parser-fuzzers/src/parser_fuzzers/dimension_expander.py deleted file mode 100644 index 98bec2c..0000000 --- a/parser-fuzzers/src/parser_fuzzers/dimension_expander.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.dimension_expander`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.dimension_expander") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/document_harness.py b/parser-fuzzers/src/parser_fuzzers/document_harness.py deleted file mode 100644 index c24da98..0000000 --- a/parser-fuzzers/src/parser_fuzzers/document_harness.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.runner.document_harness`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.runner.document_harness") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/dynamic_constraints.py b/parser-fuzzers/src/parser_fuzzers/dynamic_constraints.py deleted file mode 100644 index a865f32..0000000 --- a/parser-fuzzers/src/parser_fuzzers/dynamic_constraints.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.dynamic_constraints`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.dynamic_constraints") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/experiment.py b/parser-fuzzers/src/parser_fuzzers/experiment.py deleted file mode 100644 index 52ec8c4..0000000 --- a/parser-fuzzers/src/parser_fuzzers/experiment.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.core.experiment`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.core.experiment") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/feedback/__init__.py b/parser-fuzzers/src/parser_fuzzers/feedback/__init__.py deleted file mode 100644 index b05b17c..0000000 --- a/parser-fuzzers/src/parser_fuzzers/feedback/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Layer package for parser-fuzzers.""" diff --git a/parser-fuzzers/src/parser_fuzzers/feedback/crash_avoidance.py b/parser-fuzzers/src/parser_fuzzers/feedback/crash_avoidance.py deleted file mode 100644 index 1d8f72b..0000000 --- a/parser-fuzzers/src/parser_fuzzers/feedback/crash_avoidance.py +++ /dev/null @@ -1,243 +0,0 @@ -from __future__ import annotations - -import json -import os -from dataclasses import dataclass -from functools import lru_cache -from pathlib import Path -from typing import Any - -from parser_fuzzers.format_specs import ImageGoal - - -@dataclass(frozen=True) -class CrashHazard: - target_id: str - ppd_kind: str - document_kind: str - image_format: str - objective: str - payload: str - interlace: int | None - signature: str - raw: str - - def matches_target(self, target_id: str) -> bool: - return _same_or_derived_target(self.target_id, target_id) - - def matches_goal(self, target_id: str, goal: ImageGoal) -> bool: - if not self.matches_target(target_id): - return False - if self.image_format and self.image_format not in goal.allowed_formats: - return False - goal_objectives = {goal.name, f"{goal.output_format}:{goal.name}"} - if self.objective in goal_objectives: - return True - if ":" in self.objective and self.objective.split(":", 1)[1] == goal.name: - return True - return False - - -@dataclass(frozen=True) -class CrashAvoidanceProfile: - source_path: str - hazards: tuple[CrashHazard, ...] - - def hazards_for_goal(self, target_id: str, goal: ImageGoal) -> tuple[CrashHazard, ...]: - return tuple(hazard for hazard in self.hazards if hazard.matches_goal(target_id, goal)) - - def generalized_hazards_for_goal(self, target_id: str, goal: ImageGoal) -> tuple[CrashHazard, ...]: - return tuple( - hazard - for hazard in self.hazards - if hazard.matches_target(target_id) and hazard.image_format in goal.allowed_formats - ) - - def goal_penalty(self, target_id: str, goal: ImageGoal, *, generalized: bool = False) -> int: - exact = self.hazards_for_goal(target_id, goal) - if not generalized: - return len(exact) * 100 - seen = {hazard.raw for hazard in exact} - generalized_count = sum( - 1 for hazard in self.generalized_hazards_for_goal(target_id, goal) if hazard.raw not in seen - ) - return len(exact) * 100 + generalized_count * 10 - - def target_hazard_count(self, target_id: str) -> int: - return sum(1 for hazard in self.hazards if hazard.matches_target(target_id)) - - def blocks_exact( - self, - *, - target_id: str, - objective: str, - image_format: str, - payload: str, - interlace: int, - ) -> bool: - objective_suffix = objective.split(":", 1)[1] if ":" in objective else objective - for hazard in self.hazards: - if not hazard.matches_target(target_id): - continue - hazard_suffix = hazard.objective.split(":", 1)[1] if ":" in hazard.objective else hazard.objective - if hazard_suffix != objective_suffix and hazard.objective != objective: - continue - if hazard.image_format != image_format: - continue - if hazard.payload != payload: - continue - if hazard.interlace is not None and hazard.interlace != interlace: - continue - return True - return False - - def blocks_generalized( - self, - *, - target_id: str, - image_format: str, - payload: str, - interlace: int, - ) -> bool: - for hazard in self.hazards: - if not hazard.matches_target(target_id): - continue - if hazard.image_format != image_format: - continue - if hazard.payload != payload: - continue - if hazard.interlace is not None and hazard.interlace != interlace: - continue - return True - return False - - -EMPTY_PROFILE = CrashAvoidanceProfile(source_path="", hazards=()) - - -def preferred_crash_avoidance_profile() -> CrashAvoidanceProfile: - if not _enabled(): - return EMPTY_PROFILE - state = os.environ.get("SMT_FUZZER_CRASH_AVOIDANCE_STATE", "auto").strip() - root = os.environ.get("SMT_FUZZER_CRASH_AVOIDANCE_ROOT", "work").strip() or "work" - return _load_cached(state, root) - - -def generalized_crash_avoidance_enabled() -> bool: - return os.environ.get("SMT_FUZZER_CRASH_AVOIDANCE_GENERALIZE", "").strip().lower() in { - "1", - "true", - "yes", - "on", - } - - -@lru_cache(maxsize=16) -def _load_cached(state: str, root: str) -> CrashAvoidanceProfile: - state_path = _resolve_state_path(state, root) - if not state_path: - return EMPTY_PROFILE - try: - payload = json.loads(state_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return EMPTY_PROFILE - hazards = tuple(_iter_hazards(payload)) - return CrashAvoidanceProfile(source_path=str(state_path), hazards=hazards) - - -def _iter_hazards(payload: dict[str, Any]) -> list[CrashHazard]: - hazards: list[CrashHazard] = [] - for item in payload.get("suppressed_case_hazards", []): - if not isinstance(item, dict): - continue - raw = str(item.get("hazard") or "") - signature = str(item.get("signature") or "") - fields = _parse_hazard_fields(raw) - target_id = fields.get("target", "") - image_format = fields.get("fmt", "") - objective = fields.get("objective", "") - if not target_id or not image_format or not objective: - continue - hazards.append( - CrashHazard( - target_id=target_id, - ppd_kind=fields.get("ppd", ""), - document_kind=fields.get("doc", ""), - image_format=image_format, - objective=objective, - payload=fields.get("payload", ""), - interlace=_safe_int(fields.get("interlace")), - signature=signature, - raw=raw, - ) - ) - return hazards - - -def _parse_hazard_fields(raw: str) -> dict[str, str]: - fields: dict[str, str] = {} - for part in raw.split("|"): - if ":" not in part: - continue - key, value = part.split(":", 1) - fields[key] = value - return fields - - -def _resolve_state_path(state: str, root: str) -> Path | None: - if state and state != "auto": - path = Path(state) - return path if path.exists() else None - return _find_latest_state(Path(root)) - - -def _find_latest_state(root: Path) -> Path | None: - if not root.exists(): - return None - candidates: list[tuple[float, Path]] = [] - stack: list[tuple[Path, int]] = [(root, 0)] - while stack: - path, depth = stack.pop() - state = path / "discovery_state.json" - if state.exists() and _state_has_hazards(state): - try: - candidates.append((state.stat().st_mtime, state)) - except OSError: - pass - if depth >= 4: - continue - try: - children = [child for child in path.iterdir() if child.is_dir()] - except OSError: - continue - for child in children: - stack.append((child, depth + 1)) - if not candidates: - return None - candidates.sort(key=lambda item: (item[0], str(item[1])), reverse=True) - return candidates[0][1] - - -def _state_has_hazards(path: Path) -> bool: - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return False - return bool(payload.get("suppressed_case_hazards")) - - -def _enabled() -> bool: - return os.environ.get("SMT_FUZZER_CRASH_AVOIDANCE", "").strip().lower() in {"1", "true", "yes", "on"} - - -def _same_or_derived_target(left: str, right: str) -> bool: - if left == right: - return True - return left.startswith(f"{right}_") or right.startswith(f"{left}_") - - -def _safe_int(value: str | None) -> int | None: - try: - return int(value) if value is not None else None - except ValueError: - return None diff --git a/parser-fuzzers/src/parser_fuzzers/feedback/crash_dedup.py b/parser-fuzzers/src/parser_fuzzers/feedback/crash_dedup.py deleted file mode 100644 index 0a536cb..0000000 --- a/parser-fuzzers/src/parser_fuzzers/feedback/crash_dedup.py +++ /dev/null @@ -1,300 +0,0 @@ -from __future__ import annotations - -import json -import re -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any - - -ADDRESS_RE = re.compile(r"0x[0-9a-fA-F]+") -PID_RE = re.compile(r"==\d+==") -BUILD_ID_RE = re.compile(r"\(BuildId: [^)]+\)") - - -@dataclass(frozen=True) -class CrashCluster: - signature: str - target_id: str - oracle: str - count: int - representative_work_dir: str - representative_command: str - representative_stderr: str - sample_work_dirs: list[str] - - -@dataclass(frozen=True) -class CrashDedupSummary: - run_dir: str - total_records: int - crash_records: int - timeout_records: int - infra_excluded_records: int - unique_crashes: int - clusters: list[CrashCluster] - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - -def dedup_run( - run_dir: str | Path, - *, - output_json: str | Path | None = None, - output_md: str | Path | None = None, - include_timeouts: bool = False, - exclude_infra: bool = True, -) -> CrashDedupSummary: - root = Path(run_dir) - total_records = 0 - crash_records = 0 - timeout_records = 0 - clusters: dict[tuple[str, str, str], dict[str, Any]] = {} - infra_excluded = 0 - - for record in _iter_timeline(root / "timeline.jsonl"): - total_records += 1 - if record.get("timed_out"): - timeout_records += 1 - if not record.get("crashed"): - continue - crash_records += 1 - if record.get("timed_out") and not include_timeouts: - continue - - stderr_text = _read_stderr(record) - if exclude_infra and _is_infra_noise(record, stderr_text): - infra_excluded += 1 - continue - - target_id = str(record.get("target_id", "unknown")) - oracle = str(record.get("oracle") or "none") - signature = _signature(record, stderr_text) - _add_cluster_record(clusters, (target_id, oracle, signature), record) - - cluster_rows = [ - _make_cluster_from_accumulator(target_id, oracle, signature, accumulator) - for (target_id, oracle, signature), accumulator in clusters.items() - ] - cluster_rows.sort(key=lambda row: (-row.count, row.target_id, row.signature)) - summary = CrashDedupSummary( - run_dir=str(root), - total_records=total_records, - crash_records=crash_records, - timeout_records=timeout_records, - infra_excluded_records=infra_excluded, - unique_crashes=len(cluster_rows), - clusters=cluster_rows, - ) - - json_path = Path(output_json) if output_json else root / "crash_dedup.json" - md_path = Path(output_md) if output_md else root / "crash_dedup.md" - json_path.write_text(json.dumps(summary.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") - md_path.write_text(_format_markdown(summary), encoding="utf-8") - return summary - - -def _iter_timeline(path: Path): - with path.open("r", encoding="utf-8") as handle: - for line in handle: - stripped = line.strip() - if stripped: - yield json.loads(stripped) - - -def _read_timeline(path: Path) -> list[dict[str, Any]]: - records = [] - for record in _iter_timeline(path): - records.append(record) - return records - - -def _read_stderr(record: dict[str, Any]) -> str: - stderr_path = record.get("stderr_path") - if stderr_path: - path = Path(str(stderr_path)) - if path.exists(): - return path.read_text(encoding="utf-8", errors="replace") - return "\n".join(str(line) for line in record.get("stderr_tail") or []) - - -def _is_infra_noise(record: dict[str, Any], stderr_text: str) -> bool: - text = stderr_text.lower() - oracle = str(record.get("oracle") or "").lower() - return "asan runtime does not come first" in text or oracle == "infra-asan-runtime-order" - - -def _signature(record: dict[str, Any], stderr_text: str) -> str: - return compute_crash_signature(record, stderr_text) - - -def compute_crash_signature(record: dict[str, Any], stderr_text: str) -> str: - lines = stderr_text.splitlines() - for line in lines: - if line.startswith("SUMMARY: AddressSanitizer:"): - summary = _normalize_line(line) - project_frame = _first_actionable_frame(lines) - if project_frame and _summary_needs_frame_context(summary): - return f"{summary} | frame:{project_frame}" - return summary - for line in lines: - stripped = line.strip() - if stripped.startswith("#0"): - return _normalize_line(stripped) - for line in lines: - lowered = line.lower() - if "crashed on signal" in lowered: - return _normalize_line(line.strip()) - return f"returncode={record.get('returncode')} oracle={record.get('oracle') or 'none'}" - - -def _summary_needs_frame_context(summary: str) -> bool: - lowered = summary.lower() - generic_markers = ( - "/usr/include/", - "../sysdeps/", - "/sysdeps/", - " in __mem", - " in memset", - " in memcpy", - " in memmove", - " in malloc", - " in free", - ) - return any(marker in lowered for marker in generic_markers) - - -def _first_actionable_frame(lines: list[str]) -> str: - for raw_line in lines: - line = raw_line.strip() - if not line.startswith("#"): - continue - normalized = _normalize_line(line) - lowered = normalized.lower() - if _is_runtime_frame(lowered): - continue - return normalized - return "" - - -def _is_runtime_frame(lowered_frame: str) -> bool: - runtime_markers = ( - "/usr/include/", - "../sysdeps/", - "/sysdeps/", - "libsanitizer", - "asan_", - " in __libc_", - " in __mem", - " in memset ", - " in memcpy ", - " in memmove ", - " in malloc ", - " in free ", - ) - return any(marker in lowered_frame for marker in runtime_markers) - - -def _normalize_line(line: str) -> str: - line = ADDRESS_RE.sub("0xADDR", line) - line = PID_RE.sub("==PID==", line) - line = BUILD_ID_RE.sub("(BuildId: BUILDID)", line) - return " ".join(line.split()) - - -def _add_cluster_record( - clusters: dict[tuple[str, str, str], dict[str, Any]], - key: tuple[str, str, str], - record: dict[str, Any], -) -> None: - accumulator = clusters.setdefault( - key, - { - "count": 0, - "representative": None, - "sample_work_dirs": [], - }, - ) - accumulator["count"] += 1 - work_dir = str(record.get("work_dir", "")) - if work_dir and len(accumulator["sample_work_dirs"]) < 5: - accumulator["sample_work_dirs"].append(work_dir) - representative = accumulator["representative"] - if representative is None or _record_sort_key(record) < _record_sort_key(representative): - accumulator["representative"] = dict(record) - - -def _record_sort_key(record: dict[str, Any]) -> tuple[str, int]: - return (str(record.get("target_id", "")), _safe_int(record.get("case_id", 0))) - - -def _safe_int(value: Any) -> int: - try: - return int(value) - except (TypeError, ValueError): - return 0 - - -def _make_cluster_from_accumulator( - target_id: str, - oracle: str, - signature: str, - accumulator: dict[str, Any], -) -> CrashCluster: - representative = accumulator.get("representative") or {} - return CrashCluster( - signature=signature, - target_id=target_id, - oracle=oracle, - count=int(accumulator.get("count", 0)), - representative_work_dir=str(representative.get("work_dir", "")), - representative_command=str(representative.get("command_line", "")), - representative_stderr=str(representative.get("stderr_path", "")), - sample_work_dirs=[str(item) for item in accumulator.get("sample_work_dirs", [])], - ) - - -def _make_cluster(target_id: str, oracle: str, signature: str, items: list[dict[str, Any]]) -> CrashCluster: - items.sort(key=lambda item: (str(item.get("target_id", "")), int(item.get("case_id", 0)))) - representative = items[0] - return CrashCluster( - signature=signature, - target_id=target_id, - oracle=oracle, - count=len(items), - representative_work_dir=str(representative.get("work_dir", "")), - representative_command=str(representative.get("command_line", "")), - representative_stderr=str(representative.get("stderr_path", "")), - sample_work_dirs=[str(item.get("work_dir", "")) for item in items[:5]], - ) - - -def _format_markdown(summary: CrashDedupSummary) -> str: - lines = [ - "# Crash Dedup", - "", - f"Run dir: `{summary.run_dir}`", - "", - "## Counts", - "", - f"- Total records: {summary.total_records}", - f"- Crash-classified records: {summary.crash_records}", - f"- Timeout records: {summary.timeout_records}", - f"- Infra records excluded: {summary.infra_excluded_records}", - f"- Unique crash signatures: {summary.unique_crashes}", - "", - "## Clusters", - "", - ] - for cluster in summary.clusters: - lines.extend( - [ - f"- count={cluster.count} target=`{cluster.target_id}` oracle=`{cluster.oracle}`", - f" signature: `{cluster.signature}`", - f" representative: `{cluster.representative_work_dir}`", - f" stderr: `{cluster.representative_stderr}`", - f" cmd: `{cluster.representative_command}`", - ] - ) - return "\n".join(lines) + "\n" diff --git a/parser-fuzzers/src/parser_fuzzers/feedback/output_feedback.py b/parser-fuzzers/src/parser_fuzzers/feedback/output_feedback.py deleted file mode 100644 index 99f2e14..0000000 --- a/parser-fuzzers/src/parser_fuzzers/feedback/output_feedback.py +++ /dev/null @@ -1,152 +0,0 @@ -from __future__ import annotations - -import json -from collections import Counter -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Iterable - -from parser_fuzzers.crash_avoidance import generalized_crash_avoidance_enabled, preferred_crash_avoidance_profile -from parser_fuzzers.dimension_expander import expand_image_goals -from parser_fuzzers.format_specs import ImageGoal, image_goals_for_target - - -@dataclass(frozen=True) -class OutputFeedbackProfile: - source_run_dirs: tuple[str, ...] - format_counts: dict[str, int] - structure_counts: dict[str, int] - objective_counts: dict[str, int] - objective_output_counts: dict[str, int] - - def to_dict(self) -> dict[str, Any]: - return { - "source_run_dirs": list(self.source_run_dirs), - "format_counts": self.format_counts, - "structure_counts": self.structure_counts, - "objective_counts": self.objective_counts, - "objective_output_counts": self.objective_output_counts, - } - - -def build_output_feedback_profile(run_dir: str | Path | Iterable[str | Path]) -> OutputFeedbackProfile: - roots = _normalize_run_dirs(run_dir) - formats: Counter[str] = Counter() - structures: Counter[str] = Counter() - objectives: Counter[str] = Counter() - objective_outputs: Counter[str] = Counter() - for root in roots: - for record in _iter_timeline(root): - target_id = str(record.get("target_id") or "") - objective = _record_objective(record) - if objective: - objectives[f"{target_id}|{objective}"] += 1 - output = ((record.get("semantic_shape") or {}).get("output") or {}) - output_format = str(output.get("format") or "") - structure = str(output.get("structure") or "") - if output_format: - formats[f"{target_id}|{output_format}"] += 1 - if structure: - structures[f"{target_id}|{structure}"] += 1 - if objective and output_format and output_format != "empty": - objective_outputs[f"{target_id}|{objective}|{output_format}"] += 1 - return OutputFeedbackProfile( - source_run_dirs=tuple(str(root) for root in roots), - format_counts=dict(sorted(formats.items())), - structure_counts=dict(sorted(structures.items())), - objective_counts=dict(sorted(objectives.items())), - objective_output_counts=dict(sorted(objective_outputs.items())), - ) - - -def write_output_feedback_profile(profile: OutputFeedbackProfile, output_path: str | Path) -> None: - path = Path(output_path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(profile.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def load_output_feedback_profile(path: str | Path) -> OutputFeedbackProfile: - data = json.loads(Path(path).read_text(encoding="utf-8")) - return OutputFeedbackProfile( - source_run_dirs=tuple(str(item) for item in data.get("source_run_dirs", [])), - format_counts={str(k): int(v) for k, v in dict(data.get("format_counts", {})).items()}, - structure_counts={str(k): int(v) for k, v in dict(data.get("structure_counts", {})).items()}, - objective_counts={str(k): int(v) for k, v in dict(data.get("objective_counts", {})).items()}, - objective_output_counts={ - str(k): int(v) for k, v in dict(data.get("objective_output_counts", {})).items() - }, - ) - - -def choose_image_goal( - *, - target_id: str, - slot: int, - profile: OutputFeedbackProfile | None = None, -) -> ImageGoal: - avoidance = preferred_crash_avoidance_profile() - generalized_avoidance = generalized_crash_avoidance_enabled() - goals = expand_image_goals( - target_id=target_id, - goals=image_goals_for_target(target_id), - profile=profile, - slot=slot, - ) - if not profile and not avoidance.hazards: - return goals[slot % len(goals)] - - def rank(goal: ImageGoal) -> tuple[int, int, int, int, str]: - objective_key = f"{target_id}|{goal.name}" - output_key = f"{target_id}|{goal.name}|{goal.output_format}" - format_key = f"{target_id}|{goal.output_format}" - objective_output_count = profile.objective_output_counts.get(output_key, 0) if profile else 0 - objective_count = profile.objective_counts.get(objective_key, 0) if profile else 0 - format_count = profile.format_counts.get(format_key, 0) if profile else 0 - avoidance_penalty = avoidance.goal_penalty( - target_id, - goal, - generalized=generalized_avoidance, - ) - salt = _stable_int(f"{target_id}|{goal.name}|{slot}") % 17 - return (avoidance_penalty, objective_output_count, objective_count, format_count + salt, goal.name) - - return min(goals, key=rank) - - -def _normalize_run_dirs(run_dir: str | Path | Iterable[str | Path]) -> list[Path]: - if isinstance(run_dir, (str, Path)): - return [Path(run_dir)] - roots = [Path(item) for item in run_dir] - return roots or [Path(".")] - - -def _iter_timeline(root: Path): - timeline = root / "timeline.jsonl" - if not timeline.exists(): - return - with timeline.open("r", encoding="utf-8", errors="replace") as handle: - for line in handle: - try: - yield json.loads(line) - except json.JSONDecodeError: - continue - - -def _record_objective(record: dict[str, Any]) -> str: - description = str(record.get("document_description") or "") - marker = " via " - if marker not in description: - return "" - objective = description.split(marker, 1)[1].split("/", 1)[0].strip() - if ":" in objective: - prefix, suffix = objective.split(":", 1) - if prefix in {"pdf", "postscript", "cups-raster", "pwg-raster"}: - return suffix - return objective - - -def _stable_int(value: str) -> int: - total = 0 - for char in value: - total = (total * 131 + ord(char)) & 0xFFFFFFFF - return total diff --git a/parser-fuzzers/src/parser_fuzzers/feedback/semantic_shapes.py b/parser-fuzzers/src/parser_fuzzers/feedback/semantic_shapes.py deleted file mode 100644 index f9f5c2d..0000000 --- a/parser-fuzzers/src/parser_fuzzers/feedback/semantic_shapes.py +++ /dev/null @@ -1,1137 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import re -import shlex -import struct -import zlib -from pathlib import Path -from typing import Any - - -def build_planned_shape( - *, - target_id: str, - ppd_kind: str, - document_kind: str, - input_mime: str, - output_mime: str, - expected_filters: list[str], - ppd_text: str, - document_data: bytes, - job_options: str = "", -) -> dict[str, Any]: - semantic_input = _semantic_input_shape( - target_id=target_id, - ppd_kind=ppd_kind, - document_kind=document_kind, - input_mime=input_mime, - output_mime=output_mime, - expected_filters=expected_filters, - ppd_shape=parse_ppd_text(ppd_text), - document_shape=parse_document_bytes(document_data), - job_options_shape=parse_job_options(job_options), - ) - return { - "semantic_input": semantic_input, - "semantic_input_hash": stable_hash(semantic_input), - "labels": _labels_for_semantic_input(semantic_input), - } - - -def build_result_shape_bundle(result: Any, stderr_text: str | None = None) -> dict[str, Any]: - ppd_text = _read_text(_get(result, "ppd_path", "")) - document_data = _read_bytes(_get(result, "document_path", "")) - output_data = _read_bytes(_get(result, "stdout_path", "")) - stderr = stderr_text if stderr_text is not None else _read_text(_get(result, "stderr_path", "")) - semantic_input = _semantic_input_shape( - target_id=str(_get(result, "target_id", "")), - ppd_kind=str(_get(result, "ppd_kind", "")), - document_kind=str(_get(result, "document_kind", "")), - input_mime="", - output_mime="", - expected_filters=[str(value) for value in _get(result, "filters", [])], - ppd_shape=parse_ppd_text(ppd_text), - document_shape=parse_document_bytes(document_data), - job_options_shape=parse_job_options(str(_get(result, "job_options", ""))), - ) - path_shape = parse_path_shape(result, stderr) - output_shape = parse_output_bytes(output_data) - failure_shape = parse_failure_shape(result, stderr) - semantic_input_hash = stable_hash(semantic_input) - path_shape_hash = stable_hash(path_shape) - output_shape_hash = stable_hash(output_shape) - failure_shape_hash = stable_hash(failure_shape) - compound = { - "semantic_input_hash": semantic_input_hash, - "path_shape_hash": path_shape_hash, - "output_shape_hash": output_shape_hash, - "failure_shape_hash": failure_shape_hash, - } - return { - "semantic_input": semantic_input, - "semantic_input_hash": semantic_input_hash, - "path_shape": path_shape, - "path_shape_hash": path_shape_hash, - "output_shape": output_shape, - "output_shape_hash": output_shape_hash, - "failure_shape": failure_shape, - "failure_shape_hash": failure_shape_hash, - "compound_shape_hash": stable_hash(compound), - "labels": { - **_labels_for_semantic_input(semantic_input), - **_labels_for_path(path_shape), - **_labels_for_output(output_shape), - **_labels_for_failure(failure_shape), - }, - } - - -def semantic_runtime_key(target_id: str, semantic_input_hash: str) -> str: - return f"target:{target_id}|semantic-input:{semantic_input_hash}" - - -def shape_feature_tokens(shape_bundle: dict[str, Any]) -> set[str]: - labels = shape_bundle.get("labels") if isinstance(shape_bundle.get("labels"), dict) else {} - path_shape = shape_bundle.get("path_shape") if isinstance(shape_bundle.get("path_shape"), dict) else {} - output_shape = shape_bundle.get("output_shape") if isinstance(shape_bundle.get("output_shape"), dict) else {} - failure_shape = shape_bundle.get("failure_shape") if isinstance(shape_bundle.get("failure_shape"), dict) else {} - semantic_hash = str(shape_bundle.get("semantic_input_hash") or "") - output_hash = str(shape_bundle.get("output_shape_hash") or "") - failure_hash = str(shape_bundle.get("failure_shape_hash") or "") - features: set[str] = set() - if semantic_hash: - features.add(f"shape-input:{semantic_hash}") - if output_hash: - features.add(f"shape-output:{output_hash}") - if failure_hash: - features.add(f"shape-failure:{failure_hash}") - for key, value in sorted(labels.items()): - if value not in {"", None}: - features.add(f"shape-{key}:{value}") - for state in path_shape.get("stderr_states", []): - if state: - features.add(f"shape-path-state:{state}") - depth = combined_depth_score(shape_bundle) - features.add(f"shape-path-depth:{_depth_bucket(depth)}") - if output_shape.get("format"): - features.add(f"shape-output-format:{output_shape.get('format', '')}") - if output_shape.get("structure"): - features.add(f"shape-output-structure:{str(output_shape.get('structure', ''))[:80]}") - if output_shape: - features.add(f"shape-output-depth:{_depth_bucket(output_depth_score(output_shape))}") - features.add(f"shape-pipeline:{_pipeline_label(path_shape, output_shape)}") - if failure_shape.get("location"): - features.add(f"shape-failure-site:{failure_shape['location']}") - return features - - -def compact_shape_record(shape_bundle: dict[str, Any]) -> dict[str, Any]: - semantic = shape_bundle.get("semantic_input", {}) - ppd = semantic.get("ppd", {}) if isinstance(semantic, dict) else {} - document = semantic.get("document", {}) if isinstance(semantic, dict) else {} - job_options = semantic.get("job_options", {}) if isinstance(semantic, dict) else {} - path_shape = shape_bundle.get("path_shape", {}) - output_shape = shape_bundle.get("output_shape", {}) - failure_shape = shape_bundle.get("failure_shape", {}) - depth = combined_depth_score(shape_bundle) - return { - "semantic_input_hash": shape_bundle.get("semantic_input_hash", ""), - "path_shape_hash": shape_bundle.get("path_shape_hash", ""), - "output_shape_hash": shape_bundle.get("output_shape_hash", ""), - "failure_shape_hash": shape_bundle.get("failure_shape_hash", ""), - "compound_shape_hash": shape_bundle.get("compound_shape_hash", ""), - "ppd": { - "filter_chain": ppd.get("filter_chain", []), - "page_class": ppd.get("page_class", ""), - "resolution_class": ppd.get("resolution_class", ""), - "color_model": ppd.get("default_color_model", ""), - }, - "document": { - "format": document.get("format", ""), - "image_class": document.get("image_class", ""), - "structure": document.get("structure", ""), - "validity": document.get("validity", ""), - }, - "job_options": { - "keys": job_options.get("keys", []), - "page_size": job_options.get("PageSize", ""), - "color_model": job_options.get("ColorModel", ""), - "resolution": job_options.get("Resolution", ""), - "scaling": job_options.get("scaling", ""), - "orientation": job_options.get("orientation-requested", ""), - }, - "path": { - "filter_chain": path_shape.get("filter_chain", []), - "reached": path_shape.get("reached_expected_filter", False), - "return_class": path_shape.get("return_class", ""), - "stderr_states": path_shape.get("stderr_states", []), - "depth_score": depth, - "depth_bucket": _depth_bucket(depth), - "path_only_depth_score": path_depth_score(path_shape), - "output_depth_score": output_depth_score(output_shape), - }, - "output": { - "format": output_shape.get("format", ""), - "size_bucket": output_shape.get("size_bucket", ""), - "validity": output_shape.get("validity", ""), - "structure": output_shape.get("structure", ""), - "object_bucket": output_shape.get("object_bucket", ""), - "stream_bucket": output_shape.get("stream_bucket", ""), - "page_bucket": output_shape.get("page_bucket", ""), - "has_xref": output_shape.get("has_xref", False), - "has_trailer": output_shape.get("has_trailer", False), - "has_eof": output_shape.get("has_eof", False), - }, - "failure": { - "kind": failure_shape.get("kind", ""), - "sanitizer": failure_shape.get("sanitizer", ""), - "location": failure_shape.get("location", ""), - "top_functions": failure_shape.get("top_functions", []), - }, - } - - -def path_depth_score(path_shape: dict[str, Any]) -> int: - score = 0 - if path_shape.get("cupstestppd_ok"): - score += 1 - if path_shape.get("filter_chain"): - score += 1 - if path_shape.get("reached_expected_filter"): - score += 2 - states = set(path_shape.get("stderr_states", [])) - stage_weights = { - "imagetops": 1, - "imagetoraster": 1, - "imagetopdf": 1, - "before-scaling": 2, - "image-colorspace": 2, - "cupswidth": 2, - "cupsheight": 2, - "cupsbytesperline": 2, - "formatting-page": 3, - "job-completed": 4, - } - score += sum(stage_weights.get(state, 1) for state in states) - if path_shape.get("return_class") == "zero": - score += 2 - if path_shape.get("stdout_size") not in {"missing", "zero"}: - score += 1 - return score - - -def output_depth_score(output_shape: dict[str, Any]) -> int: - fmt = str(output_shape.get("format", "")) - if fmt in {"", "empty", "missing"}: - return 0 - score = 1 - if fmt == "pdf": - score += 4 - score += _bucket_score(str(output_shape.get("object_bucket", "0"))) - score += _bucket_score(str(output_shape.get("stream_bucket", "0"))) - score += 2 * _bucket_score(str(output_shape.get("page_bucket", "0"))) - if output_shape.get("has_xref"): - score += 1 - if output_shape.get("has_trailer"): - score += 1 - if output_shape.get("has_eof"): - score += 1 - elif fmt == "postscript": - score += 3 - score += _bucket_score(str(output_shape.get("page_bucket", "0"))) - score += _bucket_score(str(output_shape.get("showpage_bucket", "0"))) - elif fmt in {"cups-raster", "pwg-raster"}: - score += 4 - if output_shape.get("validity") == "header-like": - score += 2 - score += _bucket_score(str(output_shape.get("height_bucket", "0"))) - elif fmt in {"png", "pnm"}: - score += 2 - if output_shape.get("size_bucket") not in {"", "zero", "lt64"}: - score += 1 - return score - - -def combined_depth_score(shape_bundle: dict[str, Any]) -> int: - path_shape = shape_bundle.get("path_shape") if isinstance(shape_bundle.get("path_shape"), dict) else {} - output_shape = shape_bundle.get("output_shape") if isinstance(shape_bundle.get("output_shape"), dict) else {} - return path_depth_score(path_shape) + output_depth_score(output_shape) - - -def stable_hash(payload: Any) -> str: - encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") - return hashlib.sha256(encoded).hexdigest()[:16] - - -def parse_ppd_text(text: str) -> dict[str, Any]: - filters = _parse_ppd_filters(text) - page_name, page_width, page_height = _parse_page_size(text) - x_res, y_res = _parse_resolution(text) - color_model = _parse_default_value(text, "ColorModel") - return { - "kind": "ppd" if text else "missing", - "line_bucket": _count_bucket(text.count("\n")), - "filter_chain": filters, - "filter_family": [_filter_family(item) for item in filters], - "default_page_size": page_name, - "page_class": _page_class(page_width, page_height), - "page_orientation": _orientation(page_width, page_height), - "default_resolution": _resolution_label(x_res, y_res), - "resolution_class": _resolution_class(x_res, y_res), - "default_color_model": color_model, - "has_cups_filter": bool(filters), - } - - -def parse_job_options(options: str) -> dict[str, Any]: - if not options: - return {"kind": "empty", "keys": []} - try: - parts = shlex.split(options) - parse_state = "ok" - except ValueError: - parts = options.split() - parse_state = "error" - values: dict[str, Any] = { - "kind": "cups-options", - "keys": [], - "parse_state": parse_state, - "option_bucket": _count_bucket(len(parts)), - } - keys: list[str] = [] - for part in parts: - if "=" in part: - key, value = part.split("=", 1) - else: - key, value = part, "true" - key = key.strip() - if not key: - continue - if key not in keys: - keys.append(key) - values[key] = _normalize_option_value(value) - values["keys"] = keys - return values - - -def parse_document_bytes(data: bytes) -> dict[str, Any]: - if data.startswith(b"\x89PNG\r\n\x1a\n"): - return _png_shape(data) - if len(data) >= 2 and data[:1] == b"P" and data[1:2] in {b"1", b"2", b"3", b"4", b"5", b"6"}: - return _pnm_shape(data) - if data.startswith(b"%PDF-"): - return { - "format": "pdf", - "size_bucket": _size_bucket(len(data)), - "structure": f"objects:{_count_bucket(data.count(b' obj'))}|streams:{_count_bucket(data.count(b'stream'))}", - "validity": "container-like", - } - if data.startswith(b"%!PS"): - return { - "format": "postscript", - "size_bucket": _size_bucket(len(data)), - "structure": f"showpage:{_count_bucket(data.count(b'showpage'))}|lines:{_count_bucket(data.count(bytes([10])))}", - "validity": "container-like", - } - if data.startswith(b"#CUPS-COMMAND"): - return { - "format": "cups-command", - "size_bucket": _size_bucket(len(data)), - "structure": f"lines:{_count_bucket(data.count(bytes([10])))}", - "validity": "container-like", - } - sync = data[:4].decode("latin1", errors="replace") - if sync in {"3SaR", "2SaR"}: - return _raster_shape(data, sync) - if not data: - return {"format": "empty", "size_bucket": "zero", "structure": "empty", "validity": "empty"} - if data.startswith(b"%") or data[:64].isascii(): - return { - "format": "text-like", - "size_bucket": _size_bucket(len(data)), - "structure": f"lines:{_count_bucket(data.count(bytes([10])))}", - "validity": "text", - } - return { - "format": "binary", - "size_bucket": _size_bucket(len(data)), - "structure": f"sync:{sync}", - "validity": "opaque", - } - - -def parse_output_bytes(data: bytes) -> dict[str, Any]: - if not data: - return { - "format": "empty", - "size_bucket": "zero", - "structure": "empty", - "validity": "empty", - } - prefix = data[:2048] - if data.startswith(b"%PDF-") or b"%PDF-" in prefix: - return _pdf_output_shape(data) - if data.startswith(b"%!PS") or b"%%Pages:" in prefix or b"showpage" in prefix: - return _postscript_output_shape(data) - sync = data[:4].decode("latin1", errors="replace") - if sync in {"3SaR", "2SaR"}: - shape = _raster_shape(data, sync) - shape["source"] = "output" - return shape - if data.startswith(b"\x89PNG\r\n\x1a\n") or ( - len(data) >= 2 and data[:1] == b"P" and data[1:2] in {b"1", b"2", b"3", b"4", b"5", b"6"} - ): - shape = parse_document_bytes(data) - shape["source"] = "output" - return shape - if prefix.lstrip().startswith(b"%"): - return { - "format": "text-percent", - "size_bucket": _size_bucket(len(data)), - "structure": f"lines:{_count_bucket(data.count(bytes([10])))}", - "validity": "text", - } - return { - "format": "binary", - "size_bucket": _size_bucket(len(data)), - "structure": f"sync:{sync}", - "validity": "opaque", - } - - -def parse_path_shape(result: Any, stderr_text: str) -> dict[str, Any]: - filters = [str(item) for item in _get(result, "filters", [])] - return { - "target_family": _target_family(str(_get(result, "target_id", ""))), - "filter_chain": [_filter_family(item) for item in filters], - "filter_count": _count_bucket(len(filters)), - "reached_expected_filter": bool(_get(result, "reached_expected_filter", False)), - "cupstestppd_ok": bool(_get(result, "cupstestppd_ok", False)), - "return_class": _return_class(_get(result, "returncode", None)), - "timed_out": bool(_get(result, "timed_out", False)), - "stderr_states": _stderr_state_tokens(stderr_text), - "stdout_size": _file_size_bucket(_get(result, "stdout_path", "")), - } - - -def parse_failure_shape(result: Any, stderr_text: str) -> dict[str, Any]: - if not bool(_get(result, "crashed", False)): - return { - "kind": "timeout" if bool(_get(result, "timed_out", False)) else "none", - "sanitizer": "", - "location": "", - "top_functions": [], - } - summary = _asan_summary(stderr_text) - sanitizer = _asan_kind(summary) - top_frames = _top_frames(stderr_text) - return { - "kind": "asan" if sanitizer else "signal-or-returncode", - "sanitizer": sanitizer, - "location": _failure_location(summary, top_frames), - "access": _access_class(stderr_text), - "address": _address_class(stderr_text), - "top_functions": [frame.get("function", "") for frame in top_frames[:3] if frame.get("function")], - "top_locations": [frame.get("location", "") for frame in top_frames[:3] if frame.get("location")], - "summary": _normalize_summary(summary), - "return_class": _return_class(_get(result, "returncode", None)), - } - - -def _semantic_input_shape( - *, - target_id: str, - ppd_kind: str, - document_kind: str, - input_mime: str, - output_mime: str, - expected_filters: list[str], - ppd_shape: dict[str, Any], - document_shape: dict[str, Any], - job_options_shape: dict[str, Any] | None = None, -) -> dict[str, Any]: - return { - "target": { - "id": target_id, - "family": _target_family(target_id), - }, - "generator": { - "ppd_kind": ppd_kind, - "document_kind": document_kind, - }, - "ppd": ppd_shape, - "document": document_shape, - "job_options": job_options_shape or {"kind": "empty", "keys": []}, - } - - -def _labels_for_semantic_input(semantic: dict[str, Any]) -> dict[str, str]: - ppd = semantic.get("ppd", {}) - document = semantic.get("document", {}) - job_options = semantic.get("job_options", {}) - filters = ppd.get("filter_family", []) if isinstance(ppd, dict) else [] - return { - "doc-format": str(document.get("format", "")), - "doc-image-class": str(document.get("image_class", "")), - "doc-structure": str(document.get("structure", ""))[:80], - "ppd-page-class": str(ppd.get("page_class", "")), - "ppd-resolution-class": str(ppd.get("resolution_class", "")), - "ppd-filter": "+".join(filters[:4]), - "job-option-keys": "+".join(job_options.get("keys", [])[:8]) if isinstance(job_options, dict) else "", - "job-option-color": str(job_options.get("ColorModel", "")) if isinstance(job_options, dict) else "", - "job-option-page": str(job_options.get("PageSize", "")) if isinstance(job_options, dict) else "", - "job-option-resolution": str(job_options.get("Resolution", "")) if isinstance(job_options, dict) else "", - } - - -def _labels_for_path(path_shape: dict[str, Any]) -> dict[str, str]: - return { - "path-filter": "+".join(path_shape.get("filter_chain", [])[:4]), - "path-return": str(path_shape.get("return_class", "")), - "path-reached": str(path_shape.get("reached_expected_filter", False)).lower(), - "path-stage": _deepest_path_stage(path_shape), - } - - -def _labels_for_failure(failure_shape: dict[str, Any]) -> dict[str, str]: - return { - "failure-kind": str(failure_shape.get("kind", "")), - "failure-sanitizer": str(failure_shape.get("sanitizer", "")), - "failure-location": str(failure_shape.get("location", ""))[:80], - } - - -def _labels_for_output(output_shape: dict[str, Any]) -> dict[str, str]: - return { - "output-format": str(output_shape.get("format", "")), - "output-size": str(output_shape.get("size_bucket", "")), - "output-validity": str(output_shape.get("validity", "")), - "output-structure": str(output_shape.get("structure", ""))[:80], - } - - -def _normalize_option_value(value: str) -> str: - value = str(value).strip() - if len(value) <= 48: - return value - return value[:48] - - -def _parse_ppd_filters(text: str) -> list[str]: - filters: list[str] = [] - for match in re.finditer(r'^\*cupsFilter2?\s*:\s*"([^"]+)"', text, flags=re.MULTILINE): - fields = match.group(1).split() - if fields: - filters.append(fields[-1]) - return filters - - -def _parse_page_size(text: str) -> tuple[str, int | None, int | None]: - default = _parse_default_value(text, "PageSize") - width = None - height = None - if default: - pattern = r"^\*PageSize\s+" + re.escape(default) + r'\b[^\n"]*:\s*"[^"]*?/PageSize\s*\[\s*([0-9.]+)\s+([0-9.]+)\s*\]' - match = re.search(pattern, text, flags=re.MULTILINE) - if match: - width = _safe_float_to_int(match.group(1)) - height = _safe_float_to_int(match.group(2)) - if width is None or height is None: - match = re.search(r"/PageSize\s*\[\s*([0-9.]+)\s+([0-9.]+)\s*\]", text) - if match: - width = _safe_float_to_int(match.group(1)) - height = _safe_float_to_int(match.group(2)) - return default, width, height - - -def _parse_resolution(text: str) -> tuple[int | None, int | None]: - value = _parse_default_value(text, "Resolution") - match = re.search(r"(\d+)\s*x\s*(\d+)\s*dpi", value or "", flags=re.IGNORECASE) - if match: - return int(match.group(1)), int(match.group(2)) - match = re.search(r"HWResolution\s*\[\s*(\d+)\s+(\d+)\s*\]", text) - if match: - return int(match.group(1)), int(match.group(2)) - return None, None - - -def _parse_default_value(text: str, keyword: str) -> str: - match = re.search(r"^\*Default" + re.escape(keyword) + r"\s*:\s*([^\s]+)", text, flags=re.MULTILINE) - return match.group(1).strip() if match else "" - - -def _png_shape(data: bytes) -> dict[str, Any]: - chunks: list[str] = [] - crc_state = "ok" - width = height = bit_depth = color_type = interlace = None - idat_bytes = 0 - pos = 8 - truncated = False - while pos + 8 <= len(data): - length = struct.unpack(">I", data[pos : pos + 4])[0] - raw_type = data[pos + 4 : pos + 8] - chunk_type = raw_type.decode("latin1", errors="replace") - payload_start = pos + 8 - payload_end = payload_start + length - crc_end = payload_end + 4 - if crc_end > len(data): - truncated = True - crc_state = "truncated" - chunks.append(chunk_type) - break - payload = data[payload_start:payload_end] - expected_crc = struct.unpack(">I", data[payload_end:crc_end])[0] - actual_crc = zlib.crc32(raw_type + payload) & 0xFFFFFFFF - if expected_crc != actual_crc and crc_state == "ok": - crc_state = "bad" - chunks.append(chunk_type) - if chunk_type == "IHDR" and len(payload) >= 13: - width, height = struct.unpack(">II", payload[:8]) - bit_depth = payload[8] - color_type = payload[9] - interlace = payload[12] - if chunk_type == "IDAT": - idat_bytes += length - pos = crc_end - if chunk_type == "IEND": - break - channels = _png_channels(color_type) - row_bytes = _row_bytes(width, bit_depth, channels) - return { - "format": "png", - "size_bucket": _size_bucket(len(data)), - "validity": "truncated" if truncated else f"crc:{crc_state}", - "width_bucket": _dimension_bucket(width), - "height_bucket": _dimension_bucket(height), - "image_class": _image_class(width, height), - "bit_depth": bit_depth, - "color_type": color_type, - "channels": channels, - "interlace": interlace, - "row_mod4": None if row_bytes is None else row_bytes % 4, - "structure": "+".join(chunks[:8]), - "has_plte": "PLTE" in chunks, - "has_trns": "tRNS" in chunks, - "idat_bucket": _size_bucket(idat_bytes), - } - - -def _pnm_shape(data: bytes) -> dict[str, Any]: - tokens: list[bytes] = [] - for raw_line in data.splitlines(): - line = raw_line.split(b"#", 1)[0].strip() - if not line: - continue - tokens.extend(line.split()) - if len(tokens) >= 4: - break - magic = tokens[0].decode("ascii", errors="replace") if tokens else "P?" - width = _safe_int_token(tokens[1]) if len(tokens) > 1 else None - height = _safe_int_token(tokens[2]) if len(tokens) > 2 else None - maxval = _safe_int_token(tokens[3]) if len(tokens) > 3 and magic not in {"P1", "P4"} else None - return { - "format": "pnm", - "size_bucket": _size_bucket(len(data)), - "validity": "header-like" if width and height else "short", - "magic": magic, - "width_bucket": _dimension_bucket(width), - "height_bucket": _dimension_bucket(height), - "image_class": _image_class(width, height), - "maxval_class": _maxval_class(maxval), - "structure": f"{magic}|max:{_maxval_class(maxval)}", - } - - -def _raster_shape(data: bytes, sync: str) -> dict[str, Any]: - if len(data) < 4 + 424: - return { - "format": "cups-raster" if sync == "3SaR" else "pwg-raster", - "size_bucket": _size_bucket(len(data)), - "structure": "short-header", - "validity": "short", - } - header = data[4 : 4 + 1796] - width = _u32(header, 372) - height = _u32(header, 376) - bpp = _u32(header, 388) - bpl = _u32(header, 392) - color_space = _u32(header, 400) - compression = _u32(header, 404) - row_count = _u32(header, 408) - format_name = "cups-raster" if sync == "3SaR" else "pwg-raster" - return { - "format": format_name, - "size_bucket": _size_bucket(len(data)), - "validity": "header-like", - "width_bucket": _dimension_bucket(width), - "height_bucket": _dimension_bucket(height), - "image_class": _image_class(width, height), - "bpp": bpp, - "bpl_bucket": _count_bucket(bpl), - "row_mod4": bpl % 4, - "color_space": color_space, - "compression": compression, - "row_count_class": _count_bucket(row_count), - "structure": ( - f"sync:{sync}|size:{_size_bucket(len(data))}|w:{_dimension_bucket(width)}|" - f"h:{_dimension_bucket(height)}|class:{_image_class(width, height)}|" - f"bpp:{bpp}|bpl:{_count_bucket(bpl)}|row:{_count_bucket(row_count)}|" - f"color:{color_space}|comp:{compression}|mod4:{bpl % 4}" - ), - } - - -def _pdf_output_shape(data: bytes) -> dict[str, Any]: - header = _pdf_header(data) - object_count = len(re.findall(rb"\b\d+\s+\d+\s+obj\b", data)) - stream_count = data.count(b"\nstream") + data.count(b"\r\nstream") - page_count = len(re.findall(rb"/Type\s*/Page\b", data)) - xobject_count = len(re.findall(rb"/Subtype\s*/Image\b", data)) - filter_names = sorted({item.decode("latin1", errors="replace") for item in re.findall(rb"/Filter\s*/([A-Za-z0-9]+)", data)}) - return { - "format": "pdf", - "size_bucket": _size_bucket(len(data)), - "validity": _pdf_validity(data, header, object_count), - "version": header, - "object_bucket": _count_bucket(object_count), - "stream_bucket": _count_bucket(stream_count), - "page_bucket": _count_bucket(page_count), - "image_xobject_bucket": _count_bucket(xobject_count), - "filter_names": filter_names[:6], - "has_xref": b"xref" in data[-4096:] or b"/XRef" in data, - "has_trailer": b"trailer" in data[-4096:] or b"/Root" in data, - "has_eof": b"%%EOF" in data[-1024:], - "structure": ( - f"pdf:{header}|size:{_size_bucket(len(data))}|obj:{_count_bucket(object_count)}|" - f"stream:{_count_bucket(stream_count)}|page:{_count_bucket(page_count)}|" - f"image:{_count_bucket(xobject_count)}|filter:{_filter_label(filter_names)}|" - f"xref:{int(b'xref' in data[-4096:] or b'/XRef' in data)}|" - f"eof:{int(b'%%EOF' in data[-1024:])}" - ), - } - - -def _postscript_output_shape(data: bytes) -> dict[str, Any]: - pages_match = re.search(rb"%%Pages:\s*(\d+)", data[:8192]) - declared_pages = int(pages_match.group(1)) if pages_match else 0 - showpage_count = data.count(b"showpage") - image_count = data.count(b"image") + data.count(b"imagemask") - return { - "format": "postscript", - "size_bucket": _size_bucket(len(data)), - "validity": "document-like" if data.startswith(b"%!PS") else "body-like", - "page_bucket": _count_bucket(declared_pages), - "showpage_bucket": _count_bucket(showpage_count), - "image_bucket": _count_bucket(image_count), - "has_bounding_box": b"%%BoundingBox:" in data[:8192], - "has_pages": bool(pages_match), - "has_eof": b"%%EOF" in data[-1024:], - "structure": ( - f"ps|size:{_size_bucket(len(data))}|pages:{_count_bucket(declared_pages)}|" - f"showpage:{_count_bucket(showpage_count)}|image:{_count_bucket(image_count)}|" - f"bbox:{int(b'%%BoundingBox:' in data[:8192])}|eof:{int(b'%%EOF' in data[-1024:])}" - ), - } - - -def _pdf_header(data: bytes) -> str: - match = re.search(rb"%PDF-(\d+\.\d+)", data[:2048]) - if not match: - return "unknown" - return match.group(1).decode("ascii", errors="replace") - - -def _pdf_validity(data: bytes, header: str, object_count: int) -> str: - traits = [] - traits.append("header" if header != "unknown" else "no-header") - traits.append("objects" if object_count else "no-objects") - if b"xref" in data[-4096:] or b"/XRef" in data: - traits.append("xref") - if b"trailer" in data[-4096:] or b"/Root" in data: - traits.append("trailer") - if b"%%EOF" in data[-1024:]: - traits.append("eof") - return "+".join(traits) - - -def _filter_label(filter_names: list[str]) -> str: - if not filter_names: - return "none" - return "+".join(filter_names[:3]) - - -def _stderr_state_tokens(stderr_text: str) -> list[str]: - markers = { - "cffilterimagetopdf:": "imagetopdf", - "cffilterimagetoraster:": "imagetoraster", - "ppdfilterimagetops": "imagetops", - "before scaling:": "before-scaling", - "using portrait orientation": "portrait", - "using landscape orientation": "landscape", - "formatting page": "formatting-page", - "cupswidth =": "cupswidth", - "cupsheight =": "cupsheight", - "cupsbytesperline =": "cupsbytesperline", - "img->colorspace =": "image-colorspace", - } - tokens: set[str] = set() - for line in stderr_text.splitlines(): - lowered = line.strip().lower() - for marker, token in markers.items(): - if marker in lowered: - tokens.add(token) - if "addresssanitizer" in lowered: - tokens.add("asan") - if "job completed" in lowered: - tokens.add("job-completed") - return sorted(tokens) - - -def _pipeline_label(path_shape: dict[str, Any], output_shape: dict[str, Any] | None = None) -> str: - output_shape = output_shape or {} - filters = "+".join(path_shape.get("filter_chain", [])[:3]) or "no-filter" - reached = "reached" if path_shape.get("reached_expected_filter") else "not-reached" - return_class = path_shape.get("return_class") or "unknown" - output_format = output_shape.get("format", "") or "no-output" - depth = path_depth_score(path_shape) + output_depth_score(output_shape) - return f"{filters}|{reached}|{return_class}|output:{output_format}|depth:{_depth_bucket(depth)}" - - -def _deepest_path_stage(path_shape: dict[str, Any]) -> str: - states = set(path_shape.get("stderr_states", [])) - ordered = [ - "job-completed", - "formatting-page", - "cupsbytesperline", - "cupsheight", - "cupswidth", - "image-colorspace", - "before-scaling", - "imagetopdf", - "imagetoraster", - "imagetops", - "asan", - ] - for state in ordered: - if state in states: - return state - return "none" - - -def _depth_bucket(score: int) -> str: - if score <= 0: - return "0" - if score <= 2: - return "1-2" - if score <= 5: - return "3-5" - if score <= 9: - return "6-9" - if score <= 14: - return "10-14" - return "15+" - - -def _asan_summary(stderr_text: str) -> str: - for line in stderr_text.splitlines(): - if line.startswith("SUMMARY: AddressSanitizer:"): - return " ".join(line.split()) - return "" - - -def _asan_kind(summary: str) -> str: - match = re.search(r"SUMMARY:\s+AddressSanitizer:\s+(\S+)", summary) - return match.group(1) if match else "" - - -def _top_frames(stderr_text: str) -> list[dict[str, str]]: - frames: list[dict[str, str]] = [] - for line in stderr_text.splitlines(): - stripped = line.strip() - if not stripped.startswith("#"): - continue - match = re.search(r"^#\d+\s+(?:0x[0-9a-fA-F]+\s+)?in\s+(.+?)(?:\s+(\S+:\d+)(?::\d+)?)?$", stripped) - if not match: - continue - function = _normalize_function(match.group(1)) - location = _normalize_location(match.group(2) or "") - frames.append({"function": function, "location": location}) - if len(frames) >= 5: - break - return frames - - -def _failure_location(summary: str, frames: list[dict[str, str]]) -> str: - if summary: - match = re.search(r"\s(\S+:\d+)\s+in\s+(\S+)", summary) - if match: - return f"{_normalize_location(match.group(1))}:in:{_normalize_function(match.group(2))}" - match = re.search(r"\sin\s+(\S+)", summary) - if match: - return f"in:{_normalize_function(match.group(1))}" - if frames: - location = frames[0].get("location", "") - function = frames[0].get("function", "") - return f"{location}:in:{function}" if location else f"in:{function}" - return "" - - -def _normalize_summary(summary: str) -> str: - summary = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", summary) - summary = re.sub(r"==\d+==", "==PID==", summary) - return " ".join(summary.split()) - - -def _access_class(stderr_text: str) -> str: - match = re.search(r"\b(READ|WRITE) of size (\d+)", stderr_text) - if match: - return f"{match.group(1).lower()}:{_count_bucket(int(match.group(2)))}" - return "" - - -def _address_class(stderr_text: str) -> str: - lowered = stderr_text.lower() - if "unknown address" in lowered: - return "unknown" - if "zero page" in lowered or "null pointer" in lowered: - return "null-ish" - if "wild pointer" in lowered: - return "wild" - return "" - - -def _parse_default_number(value: str) -> int | None: - try: - return int(value) - except (TypeError, ValueError): - return None - - -def _safe_float_to_int(value: str) -> int | None: - try: - return int(float(value)) - except ValueError: - return None - - -def _safe_int_token(value: bytes) -> int | None: - try: - return int(value) - except ValueError: - return None - - -def _read_text(path: str) -> str: - try: - return Path(path).read_text(encoding="utf-8", errors="replace") if path else "" - except OSError: - return "" - - -def _read_bytes(path: str) -> bytes: - try: - return Path(path).read_bytes() if path else b"" - except OSError: - return b"" - - -def _get(obj: Any, name: str, default: Any = None) -> Any: - if isinstance(obj, dict): - return obj.get(name, default) - return getattr(obj, name, default) - - -def _file_size_bucket(path: str) -> str: - try: - return _size_bucket(Path(path).stat().st_size) if path else "missing" - except OSError: - return "missing" - - -def _return_class(returncode: Any) -> str: - code = _parse_default_number(str(returncode)) if returncode is not None else None - if code is None: - return "none" - if code == 0: - return "zero" - if code < 0: - return "signal" - if code in {86, 134, 139}: - return f"crash-code:{code}" - return "nonzero" - - -def _target_family(target_id: str) -> str: - for suffix in ("_coverage", "_general", "_explore", "_structural", "_feedback"): - if target_id.endswith(suffix): - return target_id.removesuffix(suffix) - return target_id - - -def _filter_family(name: str) -> str: - value = Path(name).name if "/" in name else name - return re.sub(r"[^A-Za-z0-9_.+-]", "_", value) - - -def _page_class(width: int | None, height: int | None) -> str: - if not width or not height: - return "unknown" - area = width * height - if area <= 144 * 144: - return "tiny" - if 500 <= width <= 700 and 700 <= height <= 900: - return "letterish" - if width > 1000 or height > 1000: - return "large" - return "custom" - - -def _orientation(width: int | None, height: int | None) -> str: - if not width or not height: - return "unknown" - if width == height: - return "square" - return "landscape" if width > height else "portrait" - - -def _resolution_label(x_res: int | None, y_res: int | None) -> str: - if not x_res or not y_res: - return "unknown" - return f"{x_res}x{y_res}" - - -def _resolution_class(x_res: int | None, y_res: int | None) -> str: - if not x_res or not y_res: - return "unknown" - value = max(x_res, y_res) - if value <= 2: - return "unit-or-near-zero" - if value < 150: - return "low" - if value <= 720: - return "normal" - if value <= 2400: - return "high" - return "extreme" - - -def _image_class(width: int | None, height: int | None) -> str: - if not width or not height: - return "unknown" - area = width * height - traits = [] - if width == 1 or height == 1: - traits.append("line") - elif width <= 4 and height <= 4: - traits.append("tiny") - elif area <= 256: - traits.append("small") - elif area <= 16384: - traits.append("medium") - else: - traits.append("large") - if width % 2 or height % 2: - traits.append("odd") - if width in {1, 2, 3, 7, 15, 31, 63, 127, 255, 511, 1023}: - traits.append("width-boundary") - return "+".join(traits) - - -def _png_channels(color_type: int | None) -> int | None: - return {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}.get(color_type) - - -def _row_bytes(width: int | None, bit_depth: int | None, channels: int | None) -> int | None: - if not width or bit_depth is None or channels is None: - return None - return (width * bit_depth * channels + 7) // 8 - - -def _dimension_bucket(value: int | None) -> str: - if value is None: - return "unknown" - if value <= 1: - return "1" - if value <= 4: - return "2-4" - if value <= 16: - return "5-16" - if value <= 64: - return "17-64" - if value <= 256: - return "65-256" - if value <= 1024: - return "257-1024" - return "gt1024" - - -def _size_bucket(size: int) -> str: - if size <= 0: - return "zero" - if size < 64: - return "lt64" - if size < 512: - return "64-512" - if size < 4096: - return "512-4k" - if size < 65536: - return "4k-64k" - if size < 1024 * 1024: - return "64k-1m" - return "ge1m" - - -def _count_bucket(value: int) -> str: - if value <= 0: - return "0" - if value == 1: - return "1" - if value <= 4: - return "2-4" - if value <= 16: - return "5-16" - if value <= 64: - return "17-64" - if value <= 256: - return "65-256" - return "gt256" - - -def _bucket_score(bucket: str) -> int: - return { - "0": 0, - "1": 1, - "2-4": 2, - "5-16": 3, - "17-64": 4, - "65-256": 5, - "gt256": 6, - }.get(bucket, 0) - - -def _maxval_class(value: int | None) -> str: - if value is None: - return "none" - if value <= 1: - return "bitmap" - if value <= 255: - return "byte" - return "wide" - - -def _u32(buffer: bytes, offset: int) -> int: - return struct.unpack_from(" str: - value = re.sub(r"\s+", " ", value.strip()) - value = re.sub(r"\(.*\)$", "", value) - return value[:96] - - -def _normalize_location(value: str) -> str: - if not value: - return "" - return value.replace("/data/pre-gsoc/", "").replace("/usr/include/", "usr/include/") diff --git a/parser-fuzzers/src/parser_fuzzers/feedback/template_feedback.py b/parser-fuzzers/src/parser_fuzzers/feedback/template_feedback.py deleted file mode 100644 index ae84c8e..0000000 --- a/parser-fuzzers/src/parser_fuzzers/feedback/template_feedback.py +++ /dev/null @@ -1,538 +0,0 @@ -from __future__ import annotations - -import json -import struct -from collections import deque -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any, Iterable - - -HEADER_SIZE = 1796 -SYNC_CUPS_RASTER_V3 = b"3SaR" -SYNC_PWG_RASTER = b"2SaR" -PNG_MAGIC = b"\x89PNG\r\n\x1a\n" - -OFF_HW_RESOLUTION = 276 -OFF_CUPS_WIDTH = 372 -OFF_CUPS_HEIGHT = 376 -OFF_CUPS_BITS_PER_PIXEL = 388 -OFF_CUPS_BYTES_PER_LINE = 392 -OFF_CUPS_COLOR_ORDER = 396 -OFF_CUPS_COLOR_SPACE = 400 -OFF_CUPS_COMPRESSION = 404 -OFF_CUPS_ROW_COUNT = 408 -OFF_CUPS_NUM_COLORS = 420 - - -@dataclass(frozen=True) -class FeedbackSeed: - kind: str - source: str - target_id: str - case_id: int - document_path: str - crashed: bool - timed_out: bool - fields: dict[str, int] - - -@dataclass(frozen=True) -class FeedbackProfile: - source_run_dir: str - cups: list[FeedbackSeed] - pwg: list[FeedbackSeed] - images: list[FeedbackSeed] - - def to_dict(self) -> dict[str, Any]: - return { - "source_run_dir": self.source_run_dir, - "cups": [asdict(seed) for seed in self.cups], - "pwg": [asdict(seed) for seed in self.pwg], - "images": [asdict(seed) for seed in self.images], - } - - -@dataclass(frozen=True) -class _FeedbackCandidate: - seed: FeedbackSeed - score: float - diversity_key: tuple[Any, ...] - - -def build_feedback_profile( - run_dir: str | Path | Iterable[str | Path], - *, - max_cases_per_kind: int = 128, -) -> FeedbackProfile: - roots = _normalize_run_dirs(run_dir) - cups: list[_FeedbackCandidate] = [] - pwg: list[_FeedbackCandidate] = [] - images: list[_FeedbackCandidate] = [] - seen: set[tuple[str, tuple[tuple[str, int], ...]]] = set() - - for root in roots: - timeline = _timeline_records_by_case(root) - for source, meta_path in _candidate_meta_paths(root): - try: - seed = _seed_from_meta(root, meta_path, source) - except (OSError, json.JSONDecodeError, KeyError, struct.error, ValueError): - continue - key = (seed.kind, tuple(sorted(seed.fields.items()))) - if key in seen: - continue - seen.add(key) - record = timeline.get((seed.target_id, seed.case_id), {}) - candidate = _FeedbackCandidate( - seed=seed, - score=_candidate_score(seed, source, record), - diversity_key=_diversity_key(seed), - ) - if seed.kind == "cups": - cups.append(candidate) - elif seed.kind == "pwg": - pwg.append(candidate) - elif seed.kind == "image": - images.append(candidate) - - return FeedbackProfile( - source_run_dir=":".join(str(root) for root in roots), - cups=_select_frontier(cups, max_cases_per_kind), - pwg=_select_frontier(pwg, max_cases_per_kind), - images=_select_frontier(images, max_cases_per_kind), - ) - - -def write_feedback_profile(profile: FeedbackProfile, output_path: str | Path) -> None: - path = Path(output_path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(profile.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def load_feedback_profile(path: str | Path) -> FeedbackProfile: - data = json.loads(Path(path).read_text(encoding="utf-8")) - return FeedbackProfile( - source_run_dir=str(data.get("source_run_dir", "")), - cups=[_seed_from_dict(item) for item in data.get("cups", [])], - pwg=[_seed_from_dict(item) for item in data.get("pwg", [])], - images=[_seed_from_dict(item) for item in data.get("images", [])], - ) - - -def _candidate_meta_paths(root: Path) -> list[tuple[str, Path]]: - paths: list[tuple[str, Path]] = [] - interesting = root / "corpus" / "interesting" - if interesting.exists(): - paths.extend(("interesting", path) for path in sorted(interesting.glob("*/*/meta.json"))) - unique = root / "quarantine" / "unique" - if unique.exists(): - paths.extend(("unique-crash", path) for path in sorted(unique.glob("*/meta.json"))) - return paths - - -def _normalize_run_dirs(run_dir: str | Path | Iterable[str | Path]) -> list[Path]: - if isinstance(run_dir, (str, Path)): - return [Path(run_dir)] - roots = [Path(item) for item in run_dir] - return roots or [Path(".")] - - -def _timeline_records_by_case(root: Path, *, max_records: int = 200000) -> dict[tuple[str, int], dict[str, Any]]: - timeline_path = root / "timeline.jsonl" - if not timeline_path.exists(): - return {} - records: dict[tuple[str, int], dict[str, Any]] = {} - tail: deque[str] = deque(maxlen=max_records) - try: - with timeline_path.open("r", encoding="utf-8", errors="replace") as handle: - tail.extend(handle) - except OSError: - return records - for line in tail: - try: - record = json.loads(line) - except json.JSONDecodeError: - continue - target_id = str(record.get("target_id", "")) - if not target_id or "case_id" not in record: - continue - try: - case_id = int(record["case_id"]) - except (TypeError, ValueError): - continue - records[(target_id, case_id)] = record - return records - - -def _seed_from_meta(root: Path, meta_path: Path, source: str) -> FeedbackSeed: - meta = json.loads(meta_path.read_text(encoding="utf-8")) - document_path = Path(str(meta["document_path"])) - if not document_path.is_absolute(): - if document_path.exists(): - document_path = document_path - elif (root / document_path).exists(): - document_path = root / document_path - else: - document_path = Path.cwd() / document_path - data = document_path.read_bytes() - kind, fields = _parse_document_fields(data) - return FeedbackSeed( - kind=kind, - source=source, - target_id=str(meta.get("target_id", "")), - case_id=int(meta.get("case_id", 0)), - document_path=str(document_path), - crashed=bool(meta.get("crashed", False)), - timed_out=bool(meta.get("timed_out", False)), - fields=fields, - ) - - -def _parse_document_fields(data: bytes) -> tuple[str, dict[str, int]]: - if data.startswith(SYNC_CUPS_RASTER_V3): - return "cups", _parse_raster_fields(data) - if data.startswith(SYNC_PWG_RASTER): - return "pwg", _parse_raster_fields(data) - if data.startswith(PNG_MAGIC): - return "image", _parse_png_fields(data) - if len(data) >= 2 and data[:1] == b"P" and data[1:2] in {b"4", b"5", b"6"}: - return "image", _parse_pnm_fields(data) - raise ValueError("not a supported feedback document") - - -def _parse_raster_fields(data: bytes) -> dict[str, int]: - if not (data.startswith(SYNC_CUPS_RASTER_V3) or data.startswith(SYNC_PWG_RASTER)): - raise ValueError("not a supported raster document") - if len(data) < 4 + HEADER_SIZE: - raise ValueError("short raster document") - header = data[4 : 4 + HEADER_SIZE] - bytes_per_line = _u32(header, OFF_CUPS_BYTES_PER_LINE) - payload_rows = _infer_payload_rows(len(data), bytes_per_line) - return { - "width": _u32(header, OFF_CUPS_WIDTH), - "height": _u32(header, OFF_CUPS_HEIGHT), - "bits_per_pixel": _u32(header, OFF_CUPS_BITS_PER_PIXEL), - "bytes_per_line": bytes_per_line, - "row_count": _u32(header, OFF_CUPS_ROW_COUNT), - "payload_rows": payload_rows, - "color_space": _u32(header, OFF_CUPS_COLOR_SPACE), - "num_colors": _u32(header, OFF_CUPS_NUM_COLORS), - "color_order": _u32(header, OFF_CUPS_COLOR_ORDER), - "compression": _u32(header, OFF_CUPS_COMPRESSION), - "x_res": _u32(header, OFF_HW_RESOLUTION), - "y_res": _u32(header, OFF_HW_RESOLUTION + 4), - } - - -def _parse_png_fields(data: bytes) -> dict[str, int]: - if len(data) < 33 or data[12:16] != b"IHDR": - raise ValueError("short PNG document") - width, height = struct.unpack(">II", data[16:24]) - bit_depth = data[24] - color_type = data[25] - interlace = data[28] - idat_bytes = 0 - offset = 8 - while offset + 12 <= len(data): - length = struct.unpack(">I", data[offset : offset + 4])[0] - chunk_type = data[offset + 4 : offset + 8] - if chunk_type == b"IDAT": - idat_bytes += length - offset += 12 + length - if chunk_type == b"IEND": - break - channels = _png_channels(color_type) - return { - "format_id": _image_format_id_from_png(color_type), - "width": width, - "height": height, - "channels": channels, - "bit_depth": bit_depth, - "color_type": color_type, - "interlace": interlace, - "payload_len": idat_bytes, - "expected_payload_len": max(1, height * (1 + width * channels)), - } - - -def _parse_pnm_fields(data: bytes) -> dict[str, int]: - tokens, payload_offset = _pnm_tokens_and_payload_offset(data) - if len(tokens) < 3: - raise ValueError("short PNM document") - magic = tokens[0].decode("ascii", errors="replace") - if magic not in {"P4", "P5", "P6"}: - raise ValueError("unsupported PNM document") - width = int(tokens[1]) - height = int(tokens[2]) - maxval = 1 if magic == "P4" else int(tokens[3]) - channels = 3 if magic == "P6" else 1 - sample_bytes = 2 if maxval > 255 else 1 - if magic == "P4": - expected = ((width + 7) // 8) * height - else: - expected = width * height * channels * sample_bytes - return { - "format_id": {"P6": 3, "P5": 4, "P4": 5}[magic], - "width": width, - "height": height, - "channels": channels, - "maxval": maxval, - "payload_len": max(0, len(data) - payload_offset), - "expected_payload_len": expected, - "comment_style": 1 if b"#" in data[:payload_offset] else 0, - } - - -def _pnm_tokens_and_payload_offset(data: bytes) -> tuple[list[bytes], int]: - tokens: list[bytes] = [] - index = 0 - while index < len(data) and len(tokens) < 4: - while index < len(data) and data[index:index + 1].isspace(): - index += 1 - if index < len(data) and data[index:index + 1] == b"#": - while index < len(data) and data[index:index + 1] not in {b"\n", b"\r"}: - index += 1 - continue - start = index - while index < len(data) and not data[index:index + 1].isspace(): - index += 1 - if start < index: - tokens.append(data[start:index]) - if tokens and tokens[0] == b"P4" and len(tokens) >= 3: - break - while index < len(data) and data[index:index + 1].isspace(): - index += 1 - return tokens, index - - -def _image_format_id_from_png(color_type: int) -> int: - return {0: 0, 2: 1, 6: 2}.get(color_type, 1) - - -def _png_channels(color_type: int) -> int: - return {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}.get(color_type, 3) - - -def _infer_payload_rows(total_size: int, bytes_per_line: int) -> int: - if bytes_per_line <= 0 or total_size <= 4 + HEADER_SIZE: - return 1 - page_payload = total_size - 4 - HEADER_SIZE - return max(1, page_payload // bytes_per_line) - - -def _prioritize(seeds: list[FeedbackSeed]) -> list[FeedbackSeed]: - return sorted( - seeds, - key=lambda seed: ( - not seed.crashed, - seed.timed_out, - seed.target_id, - seed.case_id, - seed.source, - ), - ) - - -def _select_frontier(candidates: list[_FeedbackCandidate], limit: int) -> list[FeedbackSeed]: - if limit <= 0: - return [] - ordered = sorted( - candidates, - key=lambda candidate: ( - -candidate.score, - candidate.seed.kind, - candidate.seed.target_id, - candidate.seed.case_id, - candidate.seed.source, - ), - ) - selected: list[_FeedbackCandidate] = [] - used_diversity: set[tuple[Any, ...]] = set() - for candidate in ordered: - if candidate.diversity_key in used_diversity: - continue - selected.append(candidate) - used_diversity.add(candidate.diversity_key) - if len(selected) >= limit: - return [item.seed for item in selected] - for candidate in ordered: - if candidate in selected: - continue - selected.append(candidate) - if len(selected) >= limit: - break - return [item.seed for item in selected] - - -def _candidate_score(seed: FeedbackSeed, source: str, record: dict[str, Any]) -> float: - score = 0.0 - depth_score = _record_depth_score(record) - if source == "unique-crash": - score += 900.0 - elif seed.crashed: - score += 450.0 - if source == "interesting": - score += 80.0 - if record.get("retained_for_coverage"): - score += 120.0 - score += min(400.0, depth_score * 22.0) - if depth_score >= 15 and not seed.crashed: - score += 240.0 - elif depth_score >= 10 and not seed.crashed: - score += 90.0 - try: - score += 12.0 * int(record.get("new_feature_count", 0)) - except (TypeError, ValueError): - pass - if record.get("reached_expected_filter"): - score += 5.0 - if seed.timed_out or record.get("timed_out"): - score -= 30.0 - if seed.crashed and depth_score < 10: - score -= 250.0 - score += _field_edge_score(seed.fields) - if seed.kind == "image": - score += 10.0 - if seed.fields.get("payload_len") != seed.fields.get("expected_payload_len"): - score += 10.0 - if seed.fields.get("comment_style"): - score += 3.0 - return score - - -def _record_depth_score(record: dict[str, Any]) -> int: - semantic_shape = record.get("semantic_shape") - if not isinstance(semantic_shape, dict): - return 0 - path = semantic_shape.get("path") - path_score = 0 - if isinstance(path, dict): - try: - path_score = int(path.get("depth_score", 0)) - except (TypeError, ValueError): - path_score = 0 - output = semantic_shape.get("output") - if not isinstance(output, dict): - return path_score - return max(path_score, path_score + _output_feedback_score(output)) - - -def _output_feedback_score(output: dict[str, Any]) -> int: - fmt = str(output.get("format", "")) - if fmt == "pdf": - score = 6 - score += _bucket_score(str(output.get("object_bucket", "0"))) - score += _bucket_score(str(output.get("stream_bucket", "0"))) - score += 2 * _bucket_score(str(output.get("page_bucket", "0"))) - if output.get("has_xref"): - score += 1 - if output.get("has_trailer"): - score += 1 - if output.get("has_eof"): - score += 1 - return score - if fmt == "postscript": - return 4 + _bucket_score(str(output.get("page_bucket", "0"))) - if fmt in {"cups-raster", "pwg-raster"}: - return 5 - if fmt in {"png", "pnm"}: - return 2 - return 0 - - -def _bucket_score(bucket: str) -> int: - return { - "0": 0, - "1": 1, - "2-4": 2, - "5-16": 3, - "17-64": 4, - "65-256": 5, - "gt256": 6, - }.get(bucket, 0) - - -def _field_edge_score(fields: dict[str, int]) -> float: - width = fields.get("width", 0) - height = fields.get("height", 0) - bits = fields.get("bits_per_pixel", 0) - raw_bpl = max(1, (max(1, width) * max(1, bits) + 7) // 8) - score = 0.0 - if width in {1, 2, 3, 7, 8, 15, 16, 31, 32, 63, 64, 96, 127, 128, 192, 255, 256, 511, 512}: - score += 4.0 - if height in {1, 2, 3, 8, 16, 31, 32, 63, 64, 127, 128, 255, 256}: - score += 3.0 - if width * max(1, height) >= 3072: - score += 8.0 - if width >= max(1, height) * 4 or height >= max(1, width) * 4: - score += 5.0 - if fields.get("bytes_per_line", raw_bpl) != raw_bpl: - score += 6.0 - if fields.get("row_count", height) != height: - score += 6.0 - if fields.get("payload_rows", height) != height: - score += 6.0 - if "format_id" in fields: - score += 4.0 - if fields.get("payload_len", fields.get("expected_payload_len", 0)) != fields.get("expected_payload_len", 0): - score += 6.0 - return score - - -def _diversity_key(seed: FeedbackSeed) -> tuple[Any, ...]: - fields = seed.fields - width = fields.get("width", 0) - height = fields.get("height", 0) - bits = fields.get("bits_per_pixel", 0) - raw_bpl = max(1, (max(1, width) * max(1, bits) + 7) // 8) - return ( - seed.target_id, - seed.kind, - _bucket(width), - _bucket(height), - bits, - fields.get("color_space", 0), - fields.get("num_colors", 0), - _delta_bucket(fields.get("bytes_per_line", raw_bpl) - raw_bpl), - _delta_bucket(fields.get("row_count", height) - height), - _delta_bucket(fields.get("payload_rows", height) - height), - fields.get("format_id", -1), - _delta_bucket(fields.get("payload_len", fields.get("expected_payload_len", 0)) - fields.get("expected_payload_len", 0)), - ) - - -def _bucket(value: int) -> str: - if value <= 1: - return "one" - if value <= 8: - return "tiny" - if value <= 32: - return "small" - if value <= 128: - return "medium" - return "large" - - -def _delta_bucket(delta: int) -> str: - if delta == 0: - return "exact" - if delta < 0: - return "short" if delta >= -4 else "very-short" - return "long" if delta <= 4 else "very-long" - - -def _seed_from_dict(item: dict[str, Any]) -> FeedbackSeed: - return FeedbackSeed( - kind=str(item["kind"]), - source=str(item.get("source", "")), - target_id=str(item.get("target_id", "")), - case_id=int(item.get("case_id", 0)), - document_path=str(item.get("document_path", "")), - crashed=bool(item.get("crashed", False)), - timed_out=bool(item.get("timed_out", False)), - fields={str(key): int(value) for key, value in dict(item.get("fields", {})).items()}, - ) - - -def _u32(buffer: bytes, offset: int) -> int: - return struct.unpack_from(" dict[str, Any]: - return asdict(self) - - def concise_dict(self) -> dict[str, Any]: - return { - "run_id": self.run_id, - "work_dir": self.work_dir, - "duration_budget_sec": self.duration_budget_sec, - "elapsed_sec": self.elapsed_sec, - "workers": self.workers, - "cases": self.cases, - "crashes": self.crashes, - "valid_ppds": self.valid_ppds, - "timeouts": self.timeouts, - "reject_reasons": self.reject_reasons, - } - - -def iter_arithmetic_params() -> Iterable[ArithmeticParams]: - case_id = 0 - widths = [5, 8, 11, 16, 31, 64, 1, 2, 3, 4, 7, 13, 127, 256] - heights = [1, 2, 4, 6, 8] - bits_per_pixels = [16, 24, 32, 8, 1] - output_dpis = [1, 2, 3, 4, 5, 8, 10, 16, 72, 75, 100, 150, 300, 600, 1200] - generic_factors = [1, 2, 4, 8, 16, 64, 256, 1024, 65536, 2**20, 2**24, 2**28, 2**31] - - for width in widths: - for bits_per_pixel in bits_per_pixels: - bytes_per_line = max(1, (width * bits_per_pixel + 7) // 8) - risk_factors = _risk_factors(bytes_per_line) - for output_dpi in [1, 2, 4, 8, 16]: - for factor in risk_factors: - input_y_dpi = output_dpi * factor - if input_y_dpi < 1 or input_y_dpi > UINT32: - continue - for height in heights: - yield _make_params( - case_id=case_id, - output_dpi=output_dpi, - input_y_dpi=input_y_dpi, - width=width, - height=height, - bits_per_pixel=bits_per_pixel, - bytes_per_line=bytes_per_line, - factor=factor, - ) - case_id += 1 - - while True: - for width in widths: - for bits_per_pixel in bits_per_pixels: - bytes_per_line = max(1, (width * bits_per_pixel + 7) // 8) - factors = sorted(set(generic_factors + _overflow_factors(bytes_per_line))) - for output_dpi in output_dpis: - for factor in factors: - input_y_dpi = output_dpi * factor - if input_y_dpi < 1 or input_y_dpi > UINT32: - continue - yield _make_params( - case_id=case_id, - output_dpi=output_dpi, - input_y_dpi=input_y_dpi, - width=width, - height=heights[case_id % len(heights)], - bits_per_pixel=bits_per_pixel, - bytes_per_line=bytes_per_line, - factor=factor, - ) - case_id += 1 - - -def run_arithmetic_explore( - *, - work_dir: str | Path, - duration_sec: int, - workers: int, - timeout_sec: int, - filter_binary: str = "/usr/lib/cups/filter/pwgtoraster", -) -> ArithmeticSummary: - run_id = time.strftime("%Y%m%d-%H%M%S") - root = Path(work_dir) / run_id - root.mkdir(parents=True, exist_ok=True) - started = time.monotonic() - deadline = started + duration_sec - param_iter = iter_arithmetic_params() - results: list[ArithmeticResult] = [] - - with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: - futures: set[concurrent.futures.Future[ArithmeticResult]] = set() - while time.monotonic() < deadline or futures: - while time.monotonic() < deadline and len(futures) < workers: - params = next(param_iter) - case_dir = root / "cases" / f"case-{params.case_id:06d}" - futures.add(executor.submit(run_arithmetic_case, params, case_dir, filter_binary, timeout_sec)) - if not futures: - break - done, futures = concurrent.futures.wait( - futures, - timeout=max(0.1, min(1.0, deadline - time.monotonic())), - return_when=concurrent.futures.FIRST_COMPLETED, - ) - for future in done: - results.append(future.result()) - - results.sort(key=lambda item: item.case_id) - reject_reasons: dict[str, int] = {} - for result in results: - reject_reasons[result.reject_reason] = reject_reasons.get(result.reject_reason, 0) + 1 - - summary = ArithmeticSummary( - run_id=run_id, - work_dir=str(root), - duration_budget_sec=duration_sec, - elapsed_sec=round(time.monotonic() - started, 3), - workers=workers, - cases=len(results), - crashes=sum(1 for result in results if result.crashed), - valid_ppds=sum(1 for result in results if result.cupstestppd_ok), - timeouts=sum(1 for result in results if result.timed_out), - reject_reasons=dict(sorted(reject_reasons.items())), - results=results, - ) - (root / "summary.json").write_text(json.dumps(summary.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") - (root / "summary.concise.json").write_text( - json.dumps(summary.concise_dict(), indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - return summary - - -def run_arithmetic_case( - params: ArithmeticParams, - case_dir: Path, - filter_binary: str, - timeout_sec: int, -) -> ArithmeticResult: - case_dir.mkdir(parents=True, exist_ok=True) - ppd_path = case_dir / "candidate.ppd" - document_path = case_dir / "document.pwg" - command_path = case_dir / "command.txt" - stdout_path = case_dir / "stdout.bin" - stderr_path = case_dir / "stderr.txt" - meta_path = case_dir / "meta.json" - - ppd_path.write_text(make_pwg_resolution_ppd(params.output_dpi), encoding="utf-8") - document_path.write_bytes( - make_pwg_raster( - width=params.width, - height=params.height, - bits_per_pixel=params.bits_per_pixel, - x_res=params.input_x_dpi, - y_res=params.input_y_dpi, - ) - ) - - cupstestppd_ok = _run_cupstestppd(ppd_path, case_dir / "cupstestppd.txt") - command = [filter_binary, "1", "smt", "smt", "1", "", str(document_path)] - command_path.write_text(f"PPD={shlex.quote(str(ppd_path))} {shlex.join(command)}\n", encoding="utf-8") - - started = time.perf_counter() - stderr_text = "" - returncode: int | None = None - timed_out = False - try: - with stdout_path.open("wb") as stdout: - completed = subprocess.run( - command, - stdout=stdout, - stderr=subprocess.PIPE, - text=True, - timeout=timeout_sec, - check=False, - env=_build_env(ppd_path), - ) - returncode = completed.returncode - stderr_text = completed.stderr - except subprocess.TimeoutExpired as exc: - timed_out = True - stderr_text = (exc.stderr or "") if isinstance(exc.stderr, str) else "" - duration_ms = round((time.perf_counter() - started) * 1000.0, 3) - stderr_path.write_text(stderr_text, encoding="utf-8") - crashed, oracle = _classify(returncode, timed_out, stderr_text) - reject_reason = _reject_reason(stderr_text, returncode, timed_out, crashed) - - result = ArithmeticResult( - case_id=params.case_id, - work_dir=str(case_dir), - ppd_path=str(ppd_path), - document_path=str(document_path), - command_path=str(command_path), - stderr_path=str(stderr_path), - stdout_path=str(stdout_path), - meta_path=str(meta_path), - params=params, - cupstestppd_ok=cupstestppd_ok, - returncode=returncode, - timed_out=timed_out, - crashed=crashed, - oracle=oracle, - reject_reason=reject_reason, - duration_ms=duration_ms, - ) - meta_path.write_text(json.dumps(asdict(result), indent=2, sort_keys=True) + "\n", encoding="utf-8") - return result - - -def _overflow_factors(bytes_per_line: int) -> list[int]: - center = MOD32 // bytes_per_line - values = [] - for delta in [-3, -2, -1, 0, 1, 2, 3]: - if center + delta > 0: - values.append(center + delta) - return values - - -def _risk_factors(bytes_per_line: int) -> list[int]: - values = [2**31] - values.extend(_overflow_factors(bytes_per_line)) - return sorted(set(value for value in values if value > 0), reverse=True) - - -def _make_params( - *, - case_id: int, - output_dpi: int, - input_y_dpi: int, - width: int, - height: int, - bits_per_pixel: int, - bytes_per_line: int, - factor: int, -) -> ArithmeticParams: - strategy = "overflow-near" if (bytes_per_line * factor) >= MOD32 else "scale-sweep" - return ArithmeticParams( - case_id=case_id, - output_dpi=output_dpi, - input_x_dpi=output_dpi, - input_y_dpi=input_y_dpi, - width=width, - height=height, - bits_per_pixel=bits_per_pixel, - bytes_per_line=bytes_per_line, - y_factor=factor, - product_mod32=(bytes_per_line * factor) % MOD32, - strategy=strategy, - ) - - -def _run_cupstestppd(ppd_path: Path, output_path: Path) -> bool: - completed = subprocess.run( - ["cupstestppd", "-W", "none", str(ppd_path)], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - check=False, - ) - output_path.write_text(completed.stdout, encoding="utf-8") - return completed.returncode == 0 - - -def _build_env(ppd_path: Path) -> dict[str, str]: - env = os.environ.copy() - env["PPD"] = str(ppd_path) - return env - - -def _classify(returncode: int | None, timed_out: bool, stderr_text: str) -> tuple[bool, str]: - text = stderr_text.lower() - if timed_out: - return False, "timeout" - if "addresssanitizer" in text or "segmentation fault" in text or "crashed on signal" in text: - return True, "stderr crash/sanitizer" - if returncode is not None and returncode < 0: - return True, f"signal {-returncode}" - if returncode in {86, 134, 139}: - return True, f"returncode {returncode}" - return False, "" - - -def _reject_reason(stderr_text: str, returncode: int | None, timed_out: bool, crashed: bool) -> str: - text = stderr_text.lower() - if timed_out: - return "timeout" - if crashed: - return "crash" - if "not an integer multiple" in text: - return "resolution-not-multiple" - if "bad raster data" in text: - return "bad-raster-data" - if "unsupported" in text: - return "unsupported" - if returncode == 0: - return "ok" - return f"returncode-{returncode}" diff --git a/parser-fuzzers/src/parser_fuzzers/generator/auto_expand.py b/parser-fuzzers/src/parser_fuzzers/generator/auto_expand.py deleted file mode 100644 index b4ab2e1..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/auto_expand.py +++ /dev/null @@ -1,337 +0,0 @@ -from __future__ import annotations - -import json -import shlex -from collections import deque -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from parser_fuzzers.template_feedback import build_feedback_profile, write_feedback_profile - - -@dataclass(frozen=True) -class AutoExpandPlan: - search_root: str - run_dirs: list[str] - output_profile: str - expansion_level: int - stale_window: int - recent_records: int - recent_retained: int - recent_new_features: int - recent_crashes: int - recent_timeouts: int - profile_cups: int - profile_pwg: int - profile_images: int - recommended_env: dict[str, str] - recommended_command: str - target_actions: dict[str, str] - notes: list[str] - - def to_dict(self) -> dict[str, Any]: - return { - "search_root": self.search_root, - "run_dirs": self.run_dirs, - "output_profile": self.output_profile, - "expansion_level": self.expansion_level, - "stale_window": self.stale_window, - "recent_records": self.recent_records, - "recent_retained": self.recent_retained, - "recent_new_features": self.recent_new_features, - "recent_crashes": self.recent_crashes, - "recent_timeouts": self.recent_timeouts, - "profile_cups": self.profile_cups, - "profile_pwg": self.profile_pwg, - "profile_images": self.profile_images, - "recommended_env": self.recommended_env, - "recommended_command": self.recommended_command, - "target_actions": self.target_actions, - "notes": self.notes, - } - - -def build_auto_expand_plan( - *, - search_root: str | Path = "work", - output_profile: str | Path = "work/template-feedback/auto-expand-feedback.json", - max_runs: int = 8, - max_cases_per_kind: int = 160, - stale_window: int = 5000, - duration_sec: int = 1200, - workers: int = 10, - timeout_sec: int = 5, - max_run_gb: float = 10.0, - skip_probe_rate: float = 0.01, -) -> AutoExpandPlan: - runs = discover_campaign_runs(search_root, max_runs=max_runs) - if not runs: - raise ValueError(f"no campaign summaries found under {search_root}") - - profile = build_feedback_profile(runs, max_cases_per_kind=max_cases_per_kind) - output_path = Path(output_profile) - write_feedback_profile(profile, output_path) - - latest = runs[0] - recent_records = _read_recent_timeline(latest, stale_window) - latest_summary = _read_summary(latest) - recent_retained = sum(1 for record in recent_records if record.get("retained_for_coverage")) - recent_new_features = sum(_safe_int(record.get("new_feature_count")) for record in recent_records) - recent_crashes = sum(1 for record in recent_records if record.get("crashed")) - recent_timeouts = sum(1 for record in recent_records if record.get("timed_out")) - expansion_level = _recommend_expansion_level( - recent_count=len(recent_records), - recent_retained=recent_retained, - recent_new_features=recent_new_features, - ) - env = { - "SMT_FUZZER_TEMPLATE_FEEDBACK": str(output_path), - "SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL": str(expansion_level), - "SMT_FUZZER_TEMPLATE_CYCLE_EPOCHS": "8", - "SMT_FUZZER_HAZARD_SKIP_AFTER": "24", - "SMT_FUZZER_SEMANTIC_SKIP_AFTER": "2", - "SMT_FUZZER_SKIP_SHORT_IMAGE_ABORTS": "1", - "SMT_FUZZER_LOAD_LEGACY_SKIP_STATE": "1", - "SMT_FUZZER_MAX_RUN_GB": f"{max_run_gb:g}", - "SMT_FUZZER_SKIP_PROBE_RATE": f"{skip_probe_rate:g}", - } - command = _recommended_command( - output_path=output_path, - expansion_level=expansion_level, - duration_sec=duration_sec, - workers=workers, - timeout_sec=timeout_sec, - max_run_gb=max_run_gb, - skip_probe_rate=skip_probe_rate, - ) - notes = _notes( - expansion_level=expansion_level, - recent_count=len(recent_records), - recent_retained=recent_retained, - recent_new_features=recent_new_features, - profile_cups=len(profile.cups), - profile_pwg=len(profile.pwg), - profile_images=len(profile.images), - ) - return AutoExpandPlan( - search_root=str(search_root), - run_dirs=[str(path) for path in runs], - output_profile=str(output_path), - expansion_level=expansion_level, - stale_window=stale_window, - recent_records=len(recent_records), - recent_retained=recent_retained, - recent_new_features=recent_new_features, - recent_crashes=recent_crashes, - recent_timeouts=recent_timeouts, - profile_cups=len(profile.cups), - profile_pwg=len(profile.pwg), - profile_images=len(profile.images), - recommended_env=env, - recommended_command=command, - target_actions=_target_actions(latest_summary), - notes=notes, - ) - - -def write_auto_expand_plan(plan: AutoExpandPlan, output_path: str | Path) -> None: - path = Path(output_path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(plan.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def discover_campaign_runs(search_root: str | Path, *, max_runs: int = 8) -> list[Path]: - root = Path(search_root) - if not root.exists(): - return [] - candidates: list[tuple[float, Path]] = [] - for path in _iter_campaign_dirs(root, max_depth=3): - summary_path = path / "summary.concise.json" - if not summary_path.exists(): - continue - try: - candidates.append((summary_path.stat().st_mtime, path)) - except OSError: - continue - candidates.sort(key=lambda item: (item[0], str(item[1])), reverse=True) - return [path for _, path in candidates[:max(1, max_runs)]] - - -def _iter_campaign_dirs(root: Path, *, max_depth: int) -> list[Path]: - result: list[Path] = [] - stack: list[tuple[Path, int]] = [(root, 0)] - while stack: - path, depth = stack.pop() - if (path / "summary.concise.json").exists(): - result.append(path) - continue - if depth >= max_depth: - continue - try: - children = [child for child in path.iterdir() if child.is_dir()] - except OSError: - continue - for child in children: - stack.append((child, depth + 1)) - return result - - -def _read_recent_timeline(run_dir: Path, limit: int) -> list[dict[str, Any]]: - timeline_path = run_dir / "timeline.jsonl" - if not timeline_path.exists(): - return [] - tail: deque[str] = deque(maxlen=max(1, limit)) - try: - with timeline_path.open("r", encoding="utf-8", errors="replace") as handle: - tail.extend(handle) - except OSError: - return [] - records: list[dict[str, Any]] = [] - for line in tail: - try: - records.append(json.loads(line)) - except json.JSONDecodeError: - continue - return records - - -def _read_summary(run_dir: Path) -> dict[str, Any]: - summary_path = run_dir / "summary.concise.json" - try: - return json.loads(summary_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return {} - - -def _recommend_expansion_level(*, recent_count: int, recent_retained: int, recent_new_features: int) -> int: - if recent_count <= 0: - return 1 - retained_rate = recent_retained / max(1, recent_count) - if recent_retained == 0 and recent_new_features == 0: - return 2 - if retained_rate < 0.001: - return 1 - return 0 - - -def _target_actions(summary: dict[str, Any]) -> dict[str, str]: - actions: dict[str, str] = {} - target_stats = summary.get("target_stats", {}) - if not isinstance(target_stats, dict): - return actions - for target_id, raw_stats in sorted(target_stats.items()): - if not isinstance(raw_stats, dict): - continue - completed = _safe_int(raw_stats.get("completed")) - skipped = _safe_int(raw_stats.get("skipped")) - retained = _safe_int(raw_stats.get("retained_cases")) - timeouts = _safe_int(raw_stats.get("timeouts")) - crashes = _safe_int(raw_stats.get("crashes")) - if completed == 0 and skipped > 0: - actions[str(target_id)] = "probe-runtime-suppressed-family" - continue - if completed > 0 and timeouts / max(1, completed) > 0.05: - actions[str(target_id)] = "deprioritize-timeout-heavy-template" - continue - if completed > 0 and retained / max(1, completed) < 0.001: - actions[str(target_id)] = "expand-template-neighborhood" - continue - if crashes > 0 and retained == 0: - actions[str(target_id)] = "keep-skip-and-probe-lightly" - continue - actions[str(target_id)] = "continue-frontier-exploration" - return actions - - -def _recommended_command( - *, - output_path: Path, - expansion_level: int, - duration_sec: int, - workers: int, - timeout_sec: int, - max_run_gb: float, - skip_probe_rate: float, -) -> str: - env = { - "SMT_FUZZER_TEMPLATE_FEEDBACK": str(output_path), - "SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL": str(expansion_level), - "SMT_FUZZER_TEMPLATE_CYCLE_EPOCHS": "8", - "SMT_FUZZER_HAZARD_SKIP_AFTER": "24", - "SMT_FUZZER_SEMANTIC_SKIP_AFTER": "2", - "SMT_FUZZER_SKIP_SHORT_IMAGE_ABORTS": "1", - "SMT_FUZZER_LOAD_LEGACY_SKIP_STATE": "1", - "SMT_FUZZER_MAX_RUN_GB": f"{max_run_gb:g}", - "SMT_FUZZER_SKIP_PROBE_RATE": f"{skip_probe_rate:g}", - } - env_part = " ".join(f"{key}={shlex.quote(value)}" for key, value in env.items()) - args = [ - "PYTHONPATH=src", - "python3", - "-m", - "parser_fuzzers.cli", - "multitarget-monitor", - "--config", - "configs/parser_targets_auto_hybrid.yaml", - "--work-root", - "work/auto-hybrid-campaign", - "--workers", - str(workers), - "--timeout-sec", - str(timeout_sec), - "--duration-sec", - str(duration_sec), - "--max-run-gb", - f"{max_run_gb:g}", - "--discard-stdout", - "--discovery-mode", - "coverage", - "--scheduler", - "novelty", - "--runtime-skip", - "--auto-skip-state", - "--auto-skip-root", - "work", - "--generalized-skip", - "--family-skip-after", - "12", - "--skip-probe-rate", - f"{skip_probe_rate:g}", - "--prune-uninteresting", - ] - return f"{env_part} {' '.join(shlex.quote(arg) for arg in args)}" - - -def _notes( - *, - expansion_level: int, - recent_count: int, - recent_retained: int, - recent_new_features: int, - profile_cups: int, - profile_pwg: int, - profile_images: int, -) -> list[str]: - notes = [] - if expansion_level > 0: - notes.append("recent coverage yield is low; widen feedback template neighborhoods") - else: - notes.append("recent coverage yield is still useful; keep conservative neighborhoods") - if profile_cups == 0: - notes.append("no CUPS raster feedback seeds were found") - if profile_pwg == 0: - notes.append("no PWG raster feedback seeds were found") - if profile_images == 0: - notes.append("no image feedback seeds were found") - notes.append( - f"recent window: {recent_retained} retained / {recent_count} records, {recent_new_features} new features" - ) - return notes - - -def _safe_int(value: Any) -> int: - try: - return int(value) - except (TypeError, ValueError): - return 0 diff --git a/parser-fuzzers/src/parser_fuzzers/generator/constraint_repair.py b/parser-fuzzers/src/parser_fuzzers/generator/constraint_repair.py deleted file mode 100644 index 212a7b7..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/constraint_repair.py +++ /dev/null @@ -1,306 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from parser_fuzzers.crash_avoidance import ( - CrashAvoidanceProfile, - generalized_crash_avoidance_enabled, - preferred_crash_avoidance_profile, -) -from parser_fuzzers.format_specs import IMAGE_FORMATS, ImageGoal, image_channels, image_format_id, maxval_for_class -from parser_fuzzers.template_feedback import FeedbackSeed - -try: - import z3 -except ImportError: # pragma: no cover - depends on optional test environment - z3 = None - - -@dataclass(frozen=True) -class ImageRepairResult: - image_format: str - width: int - height: int - channels: int - maxval: int - payload_delta: int - comment_style: int - png_interlace: int - objective: str - solved_by: str - - -def repair_image_goal( - *, - goal: ImageGoal, - slot: int, - expansion_level: int, - seed: FeedbackSeed | None = None, - target_id: str = "", -) -> ImageRepairResult: - avoidance = preferred_crash_avoidance_profile() - fallback = _fallback(goal, slot, seed, target_id=target_id, avoidance=avoidance) - if z3 is None: - return fallback - - fmt = z3.Int("fmt") - width = z3.Int("width") - height = z3.Int("height") - payload_delta = z3.Int("payload_delta") - comment_style = z3.Int("comment_style") - interlace = z3.Int("interlace") - area = z3.Int("area") - - solver = z3.Solver() - _domain(solver, fmt, [image_format_id(item) for item in goal.allowed_formats]) - _domain(solver, width, _dimension_domain(goal.min_width, goal.max_width, slot, expansion_level)) - _domain(solver, height, _dimension_domain(goal.min_height, goal.max_height, slot // 3, expansion_level)) - _domain(solver, payload_delta, _payload_domain(goal.payload_policy, expansion_level)) - _domain(solver, comment_style, [goal.comment_style] if goal.comment_style is not None else [0, 1, 2, 3]) - _domain(solver, interlace, [goal.png_interlace]) - solver.add(area == width * height) - solver.add(width >= goal.min_width, width <= goal.max_width) - solver.add(height >= goal.min_height, height <= goal.max_height) - solver.add(area >= goal.min_area) - if goal.aspect == "wide": - solver.add(width >= height * 4) - elif goal.aspect == "tall": - solver.add(height >= width * 3) - if goal.payload_policy == "exact": - solver.add(payload_delta == 0) - elif goal.payload_policy == "short": - solver.add(payload_delta < 0) - elif goal.payload_policy == "extra": - solver.add(payload_delta > 0) - avoided = _add_crash_avoidance_constraints( - solver=solver, - target_id=target_id, - goal=goal, - fmt=fmt, - payload_delta=payload_delta, - interlace=interlace, - avoidance=avoidance, - ) - if seed and expansion_level <= 1: - if "width" in seed.fields: - solver.add(width >= max(1, seed.fields["width"] // 2)) - solver.add(width <= max(1, seed.fields["width"] * 3)) - if "height" in seed.fields: - solver.add(height >= max(1, seed.fields["height"] // 2)) - solver.add(height <= max(1, seed.fields["height"] * 3)) - - if solver.check() != z3.sat: - return fallback - model = solver.model() - image_format = IMAGE_FORMATS[int(model.evaluate(fmt, model_completion=True).as_long())] - return ImageRepairResult( - image_format=image_format, - width=int(model.evaluate(width, model_completion=True).as_long()), - height=int(model.evaluate(height, model_completion=True).as_long()), - channels=image_channels(image_format), - maxval=maxval_for_class(image_format, goal.maxval_class, slot), - payload_delta=int(model.evaluate(payload_delta, model_completion=True).as_long()), - comment_style=int(model.evaluate(comment_style, model_completion=True).as_long()), - png_interlace=int(model.evaluate(interlace, model_completion=True).as_long()), - objective=goal.name, - solved_by="z3-structure-avoid" if avoided else "z3-structure", - ) - - -def _fallback( - goal: ImageGoal, - slot: int, - seed: FeedbackSeed | None, - *, - target_id: str = "", - avoidance: CrashAvoidanceProfile | None = None, -) -> ImageRepairResult: - if seed and "format_id" in seed.fields: - allowed = tuple(item for item in goal.allowed_formats if image_format_id(item) == seed.fields["format_id"]) - image_format = allowed[0] if allowed else goal.allowed_formats[slot % len(goal.allowed_formats)] - else: - image_format = goal.allowed_formats[slot % len(goal.allowed_formats)] - width = max(goal.min_width, _fallback_dimension(goal.min_width, goal.max_width, slot)) - height = max(goal.min_height, _fallback_dimension(goal.min_height, goal.max_height, slot // 3)) - if goal.aspect == "wide": - width = max(width, height * 4) - elif goal.aspect == "tall": - height = max(height, width * 3) - payload_delta = 0 - if goal.payload_policy == "short": - payload_delta = -1 - elif goal.payload_policy == "extra": - payload_delta = 1 - image_format, payload_delta = _avoid_fallback_exact_hazard( - goal=goal, - target_id=target_id, - image_format=image_format, - payload_delta=payload_delta, - interlace=goal.png_interlace, - avoidance=avoidance, - ) - return ImageRepairResult( - image_format=image_format, - width=min(width, goal.max_width), - height=min(height, goal.max_height), - channels=image_channels(image_format), - maxval=maxval_for_class(image_format, goal.maxval_class, slot), - payload_delta=payload_delta, - comment_style=goal.comment_style if goal.comment_style is not None else slot % 4, - png_interlace=goal.png_interlace, - objective=goal.name, - solved_by="fallback-structure", - ) - - -def _add_crash_avoidance_constraints( - *, - solver, - target_id: str, - goal: ImageGoal, - fmt, - payload_delta, - interlace, - avoidance: CrashAvoidanceProfile, -) -> bool: - if not target_id or not avoidance.hazards: - return False - added = False - exact_hazards = avoidance.hazards_for_goal(target_id, goal) - hazards = list(exact_hazards) - if generalized_crash_avoidance_enabled(): - seen = {hazard.raw for hazard in hazards} - for hazard in avoidance.generalized_hazards_for_goal(target_id, goal): - if hazard.raw in seen: - continue - hazards.append(hazard) - seen.add(hazard.raw) - for hazard in hazards: - clauses = [] - fmt_id = image_format_id(hazard.image_format) - if hazard.image_format in goal.allowed_formats: - clauses.append(fmt != fmt_id) - payload_clause = _payload_difference_clause(payload_delta, hazard.payload) - if payload_clause is not None: - clauses.append(payload_clause) - if hazard.interlace is not None: - clauses.append(interlace != hazard.interlace) - if clauses: - solver.add(z3.Or(*clauses)) - added = True - return added - - -def _payload_difference_clause(payload_delta, payload: str): - if payload == "short": - return payload_delta >= 0 - if payload == "extra": - return payload_delta <= 0 - if payload == "exact": - return payload_delta != 0 - return None - - -def _avoid_fallback_exact_hazard( - *, - goal: ImageGoal, - target_id: str, - image_format: str, - payload_delta: int, - interlace: int, - avoidance: CrashAvoidanceProfile | None, -) -> tuple[str, int]: - if not target_id or avoidance is None or not avoidance.hazards: - return image_format, payload_delta - payload = _payload_label(payload_delta) - objective = f"{goal.output_format}:{goal.name}" - blocked = avoidance.blocks_exact( - target_id=target_id, - objective=objective, - image_format=image_format, - payload=payload, - interlace=interlace, - ) - if generalized_crash_avoidance_enabled(): - blocked = blocked or avoidance.blocks_generalized( - target_id=target_id, - image_format=image_format, - payload=payload, - interlace=interlace, - ) - if not blocked: - return image_format, payload_delta - for candidate in goal.allowed_formats: - candidate_blocked = avoidance.blocks_exact( - target_id=target_id, - objective=objective, - image_format=candidate, - payload=payload, - interlace=interlace, - ) - if generalized_crash_avoidance_enabled(): - candidate_blocked = candidate_blocked or avoidance.blocks_generalized( - target_id=target_id, - image_format=candidate, - payload=payload, - interlace=interlace, - ) - if not candidate_blocked: - return candidate, payload_delta - return image_format, payload_delta - - -def _payload_label(payload_delta: int) -> str: - if payload_delta < 0: - return "short" - if payload_delta > 0: - return "extra" - return "exact" - - -def _dimension_domain(min_value: int, max_value: int, salt: int, expansion_level: int) -> list[int]: - values = { - min_value, - min(max_value, min_value + 1), - min(max_value, min_value * 2), - min(max_value, min_value * 4), - 31, - 32, - 63, - 64, - 95, - 96, - 127, - 128, - 191, - 192, - 255, - 256, - } - if expansion_level >= 2: - values.update({511, 512}) - if expansion_level >= 3: - values.update({767, 768, 1023, 1024}) - ordered = sorted(value for value in values if min_value <= value <= max_value) - if not ordered: - return [min_value] - return ordered[salt % len(ordered):] + ordered[: salt % len(ordered)] - - -def _fallback_dimension(min_value: int, max_value: int, salt: int) -> int: - values = _dimension_domain(min_value, max_value, salt, 3) - return values[0] - - -def _payload_domain(policy: str, expansion_level: int) -> list[int]: - if policy == "short": - return [-1, -2, -4, -8] - if policy == "extra": - return [1, 2, 4, 8, 16, 32] - if expansion_level >= 2: - return [0, 0, 0, 1, -1] - return [0] - - -def _domain(solver, variable, values: list[int]) -> None: - solver.add(z3.Or(*[variable == value for value in sorted(set(values))])) diff --git a/parser-fuzzers/src/parser_fuzzers/generator/dimension_expander.py b/parser-fuzzers/src/parser_fuzzers/generator/dimension_expander.py deleted file mode 100644 index 5dbdb76..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/dimension_expander.py +++ /dev/null @@ -1,210 +0,0 @@ -from __future__ import annotations - -import os -from dataclasses import replace -from typing import Any - -from parser_fuzzers.format_specs import ImageGoal - - -PNM_FORMATS = frozenset({"ppm", "pgm", "pbm"}) -PNM_MAXVAL_FORMATS = frozenset({"ppm", "pgm"}) -PNG_FORMATS = frozenset({"png_gray", "png_rgb", "png_rgba"}) - - -def expand_image_goals( - *, - target_id: str, - goals: tuple[ImageGoal, ...], - profile: Any | None = None, - slot: int = 0, -) -> tuple[ImageGoal, ...]: - if not goals or not _enabled(): - return goals - budget = _budget() - if budget <= 0: - return goals - - ordered = sorted( - goals, - key=lambda goal: ( - _objective_count(profile, target_id, goal.name), - _stable_int(f"{target_id}|{goal.name}|{slot}"), - goal.name, - ), - ) - - generated: list[ImageGoal] = [] - seen = {goal.name for goal in goals} - for base in ordered: - for variant in _variants(base): - if variant.name in seen: - continue - generated.append(variant) - seen.add(variant.name) - if len(generated) >= budget: - break - if len(generated) >= budget: - break - - if not generated: - return goals - shift = slot % len(generated) - rotated = generated[shift:] + generated[:shift] - return goals + tuple(rotated) - - -def _variants(base: ImageGoal) -> tuple[ImageGoal, ...]: - variants: list[ImageGoal] = [] - variants.extend(_payload_variants(base)) - variants.extend(_pnm_comment_variants(base)) - variants.extend(_maxval_variants(base)) - variants.extend(_png_variants(base)) - variants.extend(_dimension_variants(base)) - return tuple(variants) - - -def _payload_variants(base: ImageGoal) -> tuple[ImageGoal, ...]: - return ( - _named_replace(base, "short-payload", payload_policy="short"), - _named_replace(base, "extra-payload", payload_policy="extra"), - ) - - -def _pnm_comment_variants(base: ImageGoal) -> tuple[ImageGoal, ...]: - pnm_formats = _allowed_subset(base, PNM_FORMATS) - if not pnm_formats: - return () - return ( - _named_replace(base, "inline-comment", allowed_formats=pnm_formats, comment_style=3), - _named_replace(base, "crlf-tabs", allowed_formats=pnm_formats, comment_style=2), - ) - - -def _maxval_variants(base: ImageGoal) -> tuple[ImageGoal, ...]: - maxval_formats = _allowed_subset(base, PNM_MAXVAL_FORMATS) - if not maxval_formats: - return () - return ( - _named_replace(base, "wide-maxval", allowed_formats=maxval_formats, maxval_class="wide"), - _named_replace(base, "low-maxval", allowed_formats=maxval_formats, maxval_class="low"), - ) - - -def _png_variants(base: ImageGoal) -> tuple[ImageGoal, ...]: - png_formats = _allowed_subset(base, PNG_FORMATS) - if not png_formats: - return () - return ( - _named_replace(base, "interlaced", allowed_formats=png_formats, png_interlace=1), - ) - - -def _dimension_variants(base: ImageGoal) -> tuple[ImageGoal, ...]: - variants: list[ImageGoal] = [] - large_width = max(base.min_width, 256) - large_height = max(base.min_height, 64) - large_area = max(base.min_area, 32768) - if _dimension_goal_is_feasible( - min_width=large_width, - min_height=large_height, - min_area=large_area, - max_width=max(base.max_width, large_width), - max_height=max(base.max_height, large_height), - aspect=base.aspect, - ): - variants.append( - _named_replace( - base, - "large-area", - min_width=large_width, - min_height=large_height, - min_area=large_area, - max_width=max(base.max_width, large_width), - max_height=max(base.max_height, large_height), - ) - ) - - edge_width = max(base.min_width, 31) - edge_height = max(base.min_height, 31) - edge_max_width = max(edge_width, min(base.max_width, 129)) - edge_max_height = max(edge_height, min(base.max_height, 129)) - edge_area = max(base.min_area, min(edge_max_width * edge_max_height, 4096)) - if _dimension_goal_is_feasible( - min_width=edge_width, - min_height=edge_height, - min_area=edge_area, - max_width=edge_max_width, - max_height=edge_max_height, - aspect="any", - ): - variants.append( - _named_replace( - base, - "edge-window", - min_width=edge_width, - min_height=edge_height, - min_area=edge_area, - max_width=edge_max_width, - max_height=edge_max_height, - aspect="any", - ) - ) - return tuple(variants) - - -def _dimension_goal_is_feasible( - *, - min_width: int, - min_height: int, - min_area: int, - max_width: int, - max_height: int, - aspect: str, -) -> bool: - if min_width > max_width or min_height > max_height: - return False - if max_width * max_height < min_area: - return False - if aspect == "wide" and max_width < min_height * 4: - return False - if aspect == "tall" and max_height < min_width * 3: - return False - return True - - -def _named_replace(base: ImageGoal, suffix: str, **changes: Any) -> ImageGoal: - return replace(base, name=f"auto-{base.name}-{suffix}", **changes) - - -def _allowed_subset(base: ImageGoal, allowed: frozenset[str]) -> tuple[str, ...]: - return tuple(item for item in base.allowed_formats if item in allowed) - - -def _objective_count(profile: Any | None, target_id: str, objective: str) -> int: - if profile is None: - return 0 - counts = getattr(profile, "objective_counts", {}) - try: - return int(counts.get(f"{target_id}|{objective}", 0)) - except (AttributeError, TypeError, ValueError): - return 0 - - -def _enabled() -> bool: - return os.environ.get("SMT_FUZZER_AUTO_DIMENSIONS", "").strip().lower() in {"1", "true", "yes", "on"} - - -def _budget() -> int: - value = os.environ.get("SMT_FUZZER_AUTO_DIMENSION_BUDGET", "64") - try: - return max(0, min(512, int(value))) - except ValueError: - return 64 - - -def _stable_int(value: str) -> int: - total = 0 - for char in value: - total = (total * 131 + ord(char)) & 0xFFFFFFFF - return total diff --git a/parser-fuzzers/src/parser_fuzzers/generator/dynamic_constraints.py b/parser-fuzzers/src/parser_fuzzers/generator/dynamic_constraints.py deleted file mode 100644 index 58cfbd8..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/dynamic_constraints.py +++ /dev/null @@ -1,140 +0,0 @@ -from __future__ import annotations - -import csv -import json -import re -from collections import Counter -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any - -from parser_fuzzers.source_constraints import OPTION_TOKENS - - -MAGIC_TOKENS = {"2SaR", "3SaR", "RaS2", "%PDF-", "%!PS"} -MIME_RE = re.compile(r"application/[A-Za-z0-9.+_-]+(?:/[A-Za-z0-9.+_-]+)?") - - -@dataclass(frozen=True) -class DynamicCompareRecord: - trace_path: str - pid: str - pc: str - op: str - ret: int - length: int - a_hex: str - b_hex: str - a_ascii: str - b_ascii: str - tokens: tuple[str, ...] - - -def build_dynamic_compare_profile( - run_dir: str | Path, - *, - max_records: int = 2000, -) -> dict[str, Any]: - root = Path(run_dir) - records: list[DynamicCompareRecord] = [] - op_counts: Counter[str] = Counter() - token_counts: Counter[str] = Counter() - pc_counts: Counter[str] = Counter() - total = 0 - equal = 0 - nonzero = 0 - trace_files = 0 - - for trace_path in sorted(root.rglob("compare_trace.tsv")): - trace_files += 1 - for record in _read_trace(trace_path): - total += 1 - op_counts[record.op] += 1 - pc_counts[record.pc] += 1 - if record.ret == 0: - equal += 1 - else: - nonzero += 1 - for token in record.tokens: - token_counts[token] += 1 - if len(records) < max_records: - records.append(record) - - return { - "schema_version": "dynamic-compare-hints-v1", - "run_dir": str(root), - "summary": { - "trace_files": trace_files, - "compare_records": total, - "records": len(records), - "records_truncated": len(records) >= max_records, - "equal_compares": equal, - "nonzero_compares": nonzero, - "op_counts": dict(sorted(op_counts.items())), - "top_pcs": dict(pc_counts.most_common(32)), - }, - "tokens": dict(token_counts.most_common(128)), - "ppd_options": { - token: count for token, count in token_counts.most_common(128) if token in OPTION_TOKENS - }, - "magic_tokens": { - token: count for token, count in token_counts.most_common(128) if token in MAGIC_TOKENS - }, - "records": [asdict(record) for record in records], - } - - -def write_dynamic_compare_profile(profile: dict[str, Any], output_path: str | Path) -> None: - destination = Path(output_path) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(json.dumps(profile, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def _read_trace(path: Path): - try: - with path.open("r", encoding="utf-8", errors="replace", newline="") as handle: - reader = csv.DictReader(handle, delimiter="\t") - for row in reader: - record = _record_from_row(path, row) - if record is not None: - yield record - except OSError: - return - - -def _record_from_row(path: Path, row: dict[str, str]) -> DynamicCompareRecord | None: - try: - ret = int(row.get("ret", "0")) - length = int(row.get("len", "0")) - except ValueError: - return None - a_ascii = row.get("a_ascii", "") - b_ascii = row.get("b_ascii", "") - tokens = _tokens_from_compare(a_ascii, b_ascii) - return DynamicCompareRecord( - trace_path=str(path), - pid=row.get("pid", ""), - pc=row.get("pc", ""), - op=row.get("op", ""), - ret=ret, - length=length, - a_hex=row.get("a_hex", ""), - b_hex=row.get("b_hex", ""), - a_ascii=a_ascii, - b_ascii=b_ascii, - tokens=tokens, - ) - - -def _tokens_from_compare(a_ascii: str, b_ascii: str) -> tuple[str, ...]: - tokens = set() - for side in (a_ascii, b_ascii): - compact = side.strip() - if not compact: - continue - for token in OPTION_TOKENS | MAGIC_TOKENS: - if token and token in compact: - tokens.add(token) - for match in MIME_RE.finditer(compact): - tokens.add(match.group(0).rstrip(".")) - return tuple(sorted(tokens)) diff --git a/parser-fuzzers/src/parser_fuzzers/generator/image_templates.py b/parser-fuzzers/src/parser_fuzzers/generator/image_templates.py deleted file mode 100644 index 803b404..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/image_templates.py +++ /dev/null @@ -1,386 +0,0 @@ -from __future__ import annotations - -import os -from dataclasses import dataclass -from functools import lru_cache - -from parser_fuzzers.output_feedback import load_output_feedback_profile -from parser_fuzzers.structure_mutator import mutate_image_structure -from parser_fuzzers.template_feedback import FeedbackSeed, load_feedback_profile -from parser_fuzzers.z3_guard import Z3_LOCK - -try: - import z3 -except ImportError: # pragma: no cover - exercised only in minimal environments - z3 = None - - -IMAGE_FEEDBACK_PERIOD = 960 - -IMAGE_FORMATS = ("png_gray", "png_rgb", "png_rgba", "ppm", "pgm", "pbm") -IMAGE_WIDTHS = (1, 2, 3, 4, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 95, 96, 97, 127, 128, 129, 191, 192, 255) -IMAGE_HEIGHTS = (1, 2, 3, 4, 5, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64) - - -@dataclass(frozen=True) -class ImageTemplateInstance: - image_format: str - width: int - height: int - channels: int - maxval: int - payload_delta: int - comment_style: int - png_interlace: int - objective: str - solved_by: str - - -def image_feedback_instance(case_index: int, target_id: str = "") -> ImageTemplateInstance: - with Z3_LOCK: - feedback_path = os.environ.get("SMT_FUZZER_IMAGE_FEEDBACK") or os.environ.get("SMT_FUZZER_TEMPLATE_FEEDBACK", "") - output_feedback_path = _output_feedback_path() - target_id = target_id or _target_id() - structure_mutator = _structure_mutator_enabled() - expansion_level = _expansion_level() - cycle_epoch = _cycle_epoch(case_index) - slot = case_index % IMAGE_FEEDBACK_PERIOD - if not feedback_path: - return _image_feedback_instance( - slot, - "", - output_feedback_path, - target_id, - expansion_level, - cycle_epoch, - structure_mutator, - ) - return _image_feedback_instance( - slot, - feedback_path, - output_feedback_path, - target_id, - expansion_level, - cycle_epoch, - structure_mutator, - ) - - -@lru_cache(maxsize=None) -def _image_feedback_instance( - slot: int, - feedback_path: str, - output_feedback_path: str, - target_id: str, - expansion_level: int, - cycle_epoch: int, - structure_mutator: bool, -) -> ImageTemplateInstance: - seeds = _feedback_seeds(feedback_path) - seed_index = (slot + cycle_epoch * 37 + (slot // 17) * cycle_epoch) % len(seeds) if seeds else 0 - seed = seeds[seed_index] if seeds else None - salted_slot = slot + cycle_epoch * 997 - effective_expansion = max(expansion_level, min(3, expansion_level + cycle_epoch)) - if structure_mutator: - output_feedback = _loaded_output_feedback(output_feedback_path) if output_feedback_path else None - mutated = mutate_image_structure( - target_id=target_id, - slot=salted_slot, - expansion_level=effective_expansion, - seed=seed, - output_feedback=output_feedback, - ) - return ImageTemplateInstance( - image_format=mutated.image_format, - width=mutated.width, - height=mutated.height, - channels=mutated.channels, - maxval=mutated.maxval, - payload_delta=mutated.payload_delta, - comment_style=mutated.comment_style, - png_interlace=mutated.png_interlace, - objective=f"{mutated.output_format_goal}:{mutated.objective}", - solved_by=mutated.solved_by, - ) - return _solve_image(seed, salted_slot, effective_expansion) - - -def _solve_image(seed: FeedbackSeed | None, slot: int, expansion_level: int) -> ImageTemplateInstance: - fallback = _fallback_image(seed, slot, expansion_level) - if z3 is None: - return fallback - fmt = z3.Int("fmt") - width = z3.Int("width") - height = z3.Int("height") - payload_delta = z3.Int("payload_delta") - comment_style = z3.Int("comment_style") - interlace = z3.Int("interlace") - pixel_area = z3.Int("pixel_area") - solver = z3.Solver() - _domain(solver, fmt, list(range(len(IMAGE_FORMATS)))) - _domain(solver, width, list(_width_domain(seed, expansion_level))) - _domain(solver, height, list(_height_domain(seed, expansion_level))) - _domain(solver, payload_delta, _payload_delta_domain(expansion_level)) - _domain(solver, comment_style, [0, 1, 2, 3]) - _domain(solver, interlace, [0, 0, 0, 1]) - solver.add(pixel_area == width * height) - if seed: - seed_fmt = seed.fields.get("format_id") - if seed_fmt is not None and 0 <= seed_fmt < len(IMAGE_FORMATS): - if slot % 5 != 0: - solver.add(fmt == seed_fmt) - if expansion_level <= 1 and "width" in seed.fields: - solver.add(width >= max(1, seed.fields["width"] // 2)) - solver.add(width <= max(1, seed.fields["width"] * (2 + expansion_level))) - if expansion_level <= 1 and "height" in seed.fields: - solver.add(height >= max(1, seed.fields["height"] // 2)) - solver.add(height <= max(1, seed.fields["height"] * (2 + expansion_level))) - else: - solver.add(fmt == (slot * 5 + slot // 7) % len(IMAGE_FORMATS)) - objective = _objective_name(slot, expansion_level) - if objective == "valid": - solver.add(payload_delta == 0, interlace == 0) - elif objective == "short_payload": - solver.add(payload_delta < 0, interlace == 0) - elif objective == "extra_payload": - solver.add(payload_delta > 0, interlace == 0) - elif objective == "commented": - solver.add(comment_style > 0, payload_delta == 0) - elif objective == "png_interlace_flag": - solver.add(fmt <= 2, interlace == 1, payload_delta == 0) - elif objective == "edge_dimensions": - solver.add(z3.Or(width <= 3, height <= 2, width >= 127, height >= 31), payload_delta == 0) - elif objective == "post_scaling_valid": - solver.add(width >= 96, height >= 16, payload_delta == 0, interlace == 0) - solver.add(z3.Or(fmt == 1, fmt == 2, fmt == 3)) - solver.add(pixel_area >= 3072) - elif objective == "wide_aspect_valid": - solver.add(width >= 192, height <= 32, payload_delta == 0, interlace == 0) - elif objective == "tall_aspect_valid": - solver.add(width <= 64, height >= 64, payload_delta == 0, interlace == 0) - elif objective == "maxval_sweep": - solver.add(fmt >= 3, payload_delta == 0, interlace == 0) - if solver.check() != z3.sat: - return fallback - model = solver.model() - fmt_index = int(model.evaluate(fmt, model_completion=True).as_long()) - image_format = IMAGE_FORMATS[fmt_index] - return ImageTemplateInstance( - image_format=image_format, - width=int(model.evaluate(width, model_completion=True).as_long()), - height=int(model.evaluate(height, model_completion=True).as_long()), - channels=_channels_for_format(image_format), - maxval=_maxval_for_format(image_format, slot), - payload_delta=int(model.evaluate(payload_delta, model_completion=True).as_long()), - comment_style=int(model.evaluate(comment_style, model_completion=True).as_long()), - png_interlace=int(model.evaluate(interlace, model_completion=True).as_long()), - objective=objective, - solved_by="z3-image", - ) - - -def _fallback_image(seed: FeedbackSeed | None, slot: int, expansion_level: int) -> ImageTemplateInstance: - if seed and "format_id" in seed.fields: - fmt_index = seed.fields["format_id"] % len(IMAGE_FORMATS) - width = _select_near(IMAGE_WIDTHS, seed.fields.get("width", 8), slot) - height = _select_near(IMAGE_HEIGHTS, seed.fields.get("height", 4), slot // 3) - else: - fmt_index = (slot * 5 + slot // 7) % len(IMAGE_FORMATS) - width = IMAGE_WIDTHS[(slot * 7 + 3) % len(IMAGE_WIDTHS)] - height = IMAGE_HEIGHTS[(slot * 5 + 1) % len(IMAGE_HEIGHTS)] - image_format = IMAGE_FORMATS[fmt_index] - deltas = _payload_delta_domain(expansion_level) - objective = _objective_name(slot, expansion_level) - exact_payload_objectives = { - "valid", - "commented", - "png_interlace_flag", - "edge_dimensions", - "post_scaling_valid", - "wide_aspect_valid", - "tall_aspect_valid", - "maxval_sweep", - } - payload_delta = 0 if objective in exact_payload_objectives else deltas[slot % len(deltas)] - if objective == "short_payload" and payload_delta >= 0: - payload_delta = -1 - if objective == "extra_payload" and payload_delta <= 0: - payload_delta = 1 - return ImageTemplateInstance( - image_format=image_format, - width=width, - height=height, - channels=_channels_for_format(image_format), - maxval=_maxval_for_format(image_format, slot), - payload_delta=payload_delta, - comment_style=slot % 4, - png_interlace=1 if objective == "png_interlace_flag" and image_format.startswith("png") else 0, - objective=objective, - solved_by="fallback-image", - ) - - -def _feedback_seeds(path: str) -> tuple[FeedbackSeed, ...]: - if not path: - return () - profile = load_feedback_profile(path) - return tuple(profile.images) - - -@lru_cache(maxsize=32) -def _loaded_output_feedback(path: str): - return load_output_feedback_profile(path) - - -def _expansion_level() -> int: - value = os.environ.get("SMT_FUZZER_IMAGE_EXPANSION_LEVEL") or os.environ.get("SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL", "0") - try: - return max(0, min(3, int(value))) - except ValueError: - return 0 - - -def _cycle_epoch(case_index: int) -> int: - epochs = _cycle_epochs() - if epochs <= 1: - return 0 - return (case_index // IMAGE_FEEDBACK_PERIOD) % epochs - - -def _cycle_epochs() -> int: - value = os.environ.get("SMT_FUZZER_IMAGE_CYCLE_EPOCHS") or os.environ.get("SMT_FUZZER_TEMPLATE_CYCLE_EPOCHS", "1") - try: - return max(1, min(64, int(value))) - except ValueError: - return 1 - - -def _target_id() -> str: - return ( - os.environ.get("SMT_FUZZER_TARGET_ID") - or os.environ.get("SMT_FUZZER_IMAGE_TARGET") - or "image_to_imagetoraster_feedback" - ) - - -def _output_feedback_path() -> str: - value = os.environ.get("SMT_FUZZER_OUTPUT_FEEDBACK", "").strip() - if not value or value == "auto": - return "" - return value - - -def _structure_mutator_enabled() -> bool: - return os.environ.get("SMT_FUZZER_STRUCTURE_MUTATOR", "").strip().lower() in {"1", "true", "yes", "on"} - - -def _domain(solver, variable, values: list[int]) -> None: - solver.add(z3.Or(*[variable == value for value in sorted(set(values))])) - - -def _width_domain(seed: FeedbackSeed | None, expansion_level: int) -> tuple[int, ...]: - values = set(IMAGE_WIDTHS) - if seed and "width" in seed.fields: - values.update(_near_values(seed.fields["width"], 1 + expansion_level)) - if expansion_level >= 2: - values.update({191, 192, 193, 256, 257, 511, 512}) - if expansion_level >= 3: - values.update({767, 768, 769, 1023, 1024}) - return tuple(sorted(value for value in values if value >= 1 and value <= 1024)) - - -def _height_domain(seed: FeedbackSeed | None, expansion_level: int) -> tuple[int, ...]: - values = set(IMAGE_HEIGHTS) - if seed and "height" in seed.fields: - values.update(_near_values(seed.fields["height"], 1 + expansion_level)) - if expansion_level >= 2: - values.update({63, 64, 65, 127, 128, 129}) - if expansion_level >= 3: - values.update({191, 192, 193, 255, 256}) - return tuple(sorted(value for value in values if value >= 1 and value <= 256)) - - -def _near_values(center: int, scale: int) -> set[int]: - return { - max(1, center - scale), - center, - center + scale, - max(1, center // 2), - max(1, center * min(4, 1 + scale)), - } - - -def _payload_delta_domain(expansion_level: int) -> list[int]: - if expansion_level <= 0: - return [0, 0, -1, 1, 2] - if expansion_level == 1: - return [0, -1, 1, -2, 2, 4, 8] - return [0, -1, 1, -2, 2, -4, 4, 8, 16, 32] - - -def _objective_name(slot: int, expansion_level: int) -> str: - if _image_valid_bias_enabled(): - short_every = _short_payload_every() - if short_every > 0 and slot % short_every == 2: - return "short_payload" - if expansion_level >= 2 and slot % 20 == 9: - return "post_scaling_valid" - if expansion_level >= 2 and slot % 24 == 11: - return "wide_aspect_valid" - if expansion_level >= 3 and slot % 30 == 17: - return "tall_aspect_valid" - if expansion_level >= 3 and slot % 28 == 13: - return "maxval_sweep" - if expansion_level >= 1 and slot % 18 == 5: - return "png_interlace_flag" - names = [ - "valid", - "commented", - "edge_dimensions", - "valid", - "extra_payload", - "edge_dimensions", - "post_scaling_valid" if expansion_level >= 2 else "valid", - "valid", - "commented", - ] - return names[slot % len(names)] - names = ["valid", "commented", "short_payload", "extra_payload", "edge_dimensions"] - if expansion_level >= 1: - names.append("png_interlace_flag") - if expansion_level >= 2: - names.extend(["post_scaling_valid", "wide_aspect_valid"]) - if expansion_level >= 3: - names.extend(["tall_aspect_valid", "maxval_sweep"]) - return names[slot % len(names)] - - -def _image_valid_bias_enabled() -> bool: - return os.environ.get("SMT_FUZZER_IMAGE_VALID_BIAS", "").strip().lower() in {"1", "true", "yes", "on"} - - -def _short_payload_every() -> int: - value = os.environ.get("SMT_FUZZER_IMAGE_SHORT_PAYLOAD_EVERY", "0") - try: - return max(0, int(value)) - except ValueError: - return 0 - - -def _channels_for_format(image_format: str) -> int: - if image_format in {"png_rgb", "ppm"}: - return 3 - if image_format == "png_rgba": - return 4 - return 1 - - -def _maxval_for_format(image_format: str, slot: int) -> int: - if image_format in {"pbm"}: - return 1 - return [1, 2, 15, 31, 127, 255, 65535][slot % 7] - - -def _select_near(values: tuple[int, ...], center: int, salt: int) -> int: - choices = sorted(set(values) | _near_values(center, 2)) - return choices[salt % len(choices)] diff --git a/parser-fuzzers/src/parser_fuzzers/generator/patcher.py b/parser-fuzzers/src/parser_fuzzers/generator/patcher.py deleted file mode 100644 index 12b42ab..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/patcher.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -from parser_fuzzers.hashing import sha256_bytes, sha256_file -from parser_fuzzers.models import Patch, SolverResult - - -def apply_patches(input_bytes: bytes, patches: list[Patch]) -> bytes: - output = bytearray(input_bytes) - for patch in patches: - start = patch.offset - end = patch.offset + patch.width - if end > len(output): - raise ValueError(f"patch [{start}, {end}) exceeds input length {len(output)}") - old_bytes = bytes.fromhex(patch.old_hex) - current = bytes(output[start:end]) - if current != old_bytes: - raise ValueError( - f"patch old bytes mismatch at offset {start}: " - f"expected {old_bytes.hex()}, got {current.hex()}" - ) - output[start:end] = bytes.fromhex(patch.new_hex) - return bytes(output) - - -def load_solver_result(path: str | Path) -> SolverResult: - with Path(path).open("r", encoding="utf-8") as handle: - return SolverResult.from_dict(json.load(handle)) - - -def write_solver_result(result: SolverResult, path: str | Path) -> None: - destination = Path(path) - destination.parent.mkdir(parents=True, exist_ok=True) - with destination.open("w", encoding="utf-8") as handle: - json.dump(result.to_dict(), handle, indent=2, sort_keys=True) - handle.write("\n") - - -def apply_solver_result( - result: SolverResult, - input_path: str | Path, - output_dir: str | Path, -) -> Path: - source = Path(input_path) - input_bytes = source.read_bytes() - expected_hash = result.event.input_sha256 - actual_hash = sha256_file(source) - if actual_hash != expected_hash: - raise ValueError(f"input hash mismatch: expected {expected_hash}, got {actual_hash}") - - output_bytes = apply_patches(input_bytes, result.patches) - output_hash = sha256_bytes(output_bytes)[:12] - target = result.event.target_id.replace("/", "_") - output_name = f"smt-{target}-{expected_hash[:12]}-{output_hash}.bin" - destination = Path(output_dir) / output_name - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(output_bytes) - return destination diff --git a/parser-fuzzers/src/parser_fuzzers/generator/ppd_templates.py b/parser-fuzzers/src/parser_fuzzers/generator/ppd_templates.py deleted file mode 100644 index e289958..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/ppd_templates.py +++ /dev/null @@ -1,279 +0,0 @@ -from __future__ import annotations - -from parser_fuzzers.template_synth import synthesize_ppd_slots - - -GENERIC_STRING_VALUES = ["normal", "0", "", "A", "END", "reset", "job", "literal text"] -GENERAL_STRING_VALUES = [ - "normal", - "", - "A", - "END", - "reset", - "job-end", - "literal text", - "x" * 64, - "\\033E", - "0 1 2 3", - "printer-ready", - "SMT-Fuzzer-General", - "quotes-\"-escaped", - "backslash-\\-escaped", -] -GENERIC_RESOLUTIONS = [300, 600, 150, 72, 1200, 2, 75, 1] -GENERAL_RESOLUTIONS = [300, 600, 150, 72, 203, 200, 360, 720, 1200, 2400, 75, 100, 1, 2, 10, 65535, 65536] -COVERAGE_RESOLUTIONS = [300, 600, 150, 72, 203, 200, 360, 720, 1200, 2400, 75, 100, 1, 2, 10, 65535] -PAGE_SIZES = [ - ("Letter", "612 792", "18 36 594 756"), - ("A4", "595 842", "12 12 583 830"), - ("Small", "144 144", "0 0 144 144"), - ("Wide", "1008 612", "18 18 990 594"), -] -FILTER_COVERAGE_PPDS = { - "pdftopdf_coverage_options": ("PDFToPDF Coverage Options", "application/pdf", "pdftopdf"), - "pdftops_coverage_options": ("PDFToPS Coverage Options", "application/pdf", "pdftops"), - "pdftoraster_coverage_options": ("PDFToRaster Coverage Options", "application/pdf", "pdftoraster"), - "mupdftopwg_coverage_options": ("MuPDFToPWG Coverage Options", "application/pdf", "mupdftopwg"), - "imagetoraster_coverage_options": ("ImageToRaster Coverage Options", "image/x-portable-anymap", "imagetoraster"), - "imagetopdf_coverage_options": ("ImageToPDF Coverage Options", "image/x-portable-anymap", "imagetopdf"), - "imagetops_coverage_options": ("ImageToPS Coverage Options", "image/x-portable-anymap", "imagetops"), - "texttopdf_coverage_options": ("TextToPDF Coverage Options", "text/plain", "texttopdf"), - "texttotext_coverage_options": ("TextToText Coverage Options", "text/plain", "texttotext"), - "gstoraster_coverage_options": ("GSToRaster Coverage Options", "application/postscript", "gstoraster"), - "gstopdf_coverage_options": ("GSToPDF Coverage Options", "application/postscript", "gstopdf"), - "gstopxl_coverage_options": ("GSToPXL Coverage Options", "application/postscript", "gstopxl"), - "pwgtopclm_coverage_options": ("PWGToPCLm Coverage Options", "application/vnd.cups-pwg", "pwgtopclm"), - "commandtoescpx_coverage_options": ("CommandToESCPX Coverage Options", "application/vnd.cups-command", "commandtoescpx"), - "commandtopclx_coverage_options": ("CommandToPCLX Coverage Options", "application/vnd.cups-command", "commandtopclx"), -} - - -def make_ppd(kind: str, case_index: int) -> str: - if kind in FILTER_COVERAGE_PPDS: - model, input_mime, filter_name = FILTER_COVERAGE_PPDS[kind] - slots = synthesize_ppd_slots(case_index) - return _base_ppd( - model=model, - filter_line=f'*cupsFilter: "{input_mime} 0 {filter_name}"', - extra=_coverage_options(case_index), - page_size=PAGE_SIZES[slots.page_size_index], - ) - if kind == "rastertopclx": - payload = GENERAL_STRING_VALUES[case_index % len(GENERAL_STRING_VALUES)] - return _base_ppd( - model="Rastertopclx Template", - filter_line='*cupsFilter: "application/vnd.cups-raster 0 rastertopclx"', - extra=f'*cupsPCL EndJob: "{_escape(payload)}"\n', - ) - if kind == "rastertopclx_string_sweep": - value = GENERIC_STRING_VALUES[case_index % len(GENERIC_STRING_VALUES)] - return _base_ppd( - model="Rastertopclx String Sweep", - filter_line='*cupsFilter: "application/vnd.cups-raster 0 rastertopclx"', - extra=f'*cupsPCL EndJob: "{_escape(value)}"\n', - ) - if kind == "rastertopclx_general_strings": - value = GENERAL_STRING_VALUES[case_index % len(GENERAL_STRING_VALUES)] - return _base_ppd( - model="Rastertopclx General Strings", - filter_line='*cupsFilter: "application/vnd.cups-raster 0 rastertopclx"', - extra=f'*cupsPCL EndJob: "{_escape(value)}"\n', - ) - if kind == "rastertopclx_plain": - return _base_ppd( - model="Rastertopclx Plain Template", - filter_line='*cupsFilter: "application/vnd.cups-raster 0 rastertopclx"', - extra="", - ) - if kind == "rastertoescpx_single_pagesize": - return _base_ppd( - model="Rastertoescpx Single PageSize", - filter_line='*cupsFilter: "application/vnd.cups-raster 0 rastertoescpx"', - extra="", - ) - if kind == "rastertoescpx_size_sweep": - return _base_ppd( - model="Rastertoescpx Size Sweep", - filter_line='*cupsFilter: "application/vnd.cups-raster 0 rastertoescpx"', - extra="", - ) - if kind == "raster_coverage_options": - slots = synthesize_ppd_slots(case_index) - return _base_ppd( - model="Raster Coverage Options", - filter_line='*cupsFilter: "application/vnd.cups-raster 0 rastertopclx"', - extra=_coverage_options(case_index), - page_size=PAGE_SIZES[slots.page_size_index], - ) - if kind == "rastertops_plain": - return _base_ppd( - model="Rastertops Plain Template", - filter_line='*cupsFilter: "application/vnd.cups-raster 0 rastertops"', - extra="", - ) - if kind == "rastertopwg_plain": - return _base_ppd( - model="Rastertopwg Plain Template", - filter_line='*cupsFilter: "application/vnd.cups-raster 0 rastertopwg"', - extra="", - ) - if kind == "pwgtopdf_plain": - return _base_ppd( - model="PWGToPDF Plain Template", - filter_line='*cupsFilter: "application/vnd.cups-pwg 0 pwgtopdf"', - extra="", - ) - if kind == "pwgtopdf_coverage_options": - slots = synthesize_ppd_slots(case_index) - return _base_ppd( - model="PWGToPDF Coverage Options", - filter_line='*cupsFilter: "application/vnd.cups-pwg 0 pwgtopdf"', - extra=_coverage_options(case_index), - page_size=PAGE_SIZES[slots.page_size_index], - ) - if kind == "pwgtoraster_1dpi": - return _base_ppd( - model="PWG 1dpi Template", - filter_line='*cupsFilter: "application/vnd.cups-pwg 0 pwgtoraster"', - extra=( - '*OpenUI *Resolution: PickOne\n' - '*DefaultResolution: 1x1dpi\n' - '*Resolution 1x1dpi/1 dpi: "<>setpagedevice"\n' - '*CloseUI: *Resolution\n' - ), - ) - if kind == "pwg_resolution_sweep": - dpi = GENERIC_RESOLUTIONS[case_index % len(GENERIC_RESOLUTIONS)] - return _base_ppd( - model="PWG Resolution Sweep", - filter_line='*cupsFilter: "application/vnd.cups-pwg 0 pwgtoraster"', - extra=( - '*OpenUI *Resolution: PickOne\n' - f'*DefaultResolution: {dpi}x{dpi}dpi\n' - f'*Resolution {dpi}x{dpi}dpi/{dpi} dpi: "<>setpagedevice"\n' - '*CloseUI: *Resolution\n' - ), - ) - if kind == "pwg_resolution_general": - dpi = GENERAL_RESOLUTIONS[case_index % len(GENERAL_RESOLUTIONS)] - return _base_ppd( - model="PWG General Resolution", - filter_line='*cupsFilter: "application/vnd.cups-pwg 0 pwgtoraster"', - extra=( - '*OpenUI *Resolution: PickOne\n' - f'*DefaultResolution: {dpi}x{dpi}dpi\n' - f'*Resolution {dpi}x{dpi}dpi/{dpi} dpi: "<>setpagedevice"\n' - '*CloseUI: *Resolution\n' - ), - ) - if kind == "pwg_resolution_coverage": - slots = synthesize_ppd_slots(case_index) - dpi = slots.resolution - return _base_ppd( - model="PWG Coverage Resolution", - filter_line='*cupsFilter: "application/vnd.cups-pwg 0 pwgtoraster"', - extra=( - '*OpenUI *Resolution: PickOne\n' - f'*DefaultResolution: {dpi}x{dpi}dpi\n' - f'*Resolution {dpi}x{dpi}dpi/{dpi} dpi: "<>setpagedevice"\n' - '*CloseUI: *Resolution\n' - + _coverage_options(case_index) - ), - page_size=PAGE_SIZES[slots.page_size_index], - ) - raise ValueError(f"unknown PPD kind: {kind}") - - -def make_pwg_resolution_ppd(dpi: int) -> str: - return _base_ppd( - model="PWG Resolution Boundary", - filter_line='*cupsFilter: "application/vnd.cups-pwg 0 pwgtoraster"', - extra=( - '*OpenUI *Resolution: PickOne\n' - f'*DefaultResolution: {dpi}x{dpi}dpi\n' - f'*Resolution {dpi}x{dpi}dpi/{dpi} dpi: "<>setpagedevice"\n' - '*CloseUI: *Resolution\n' - ), - ) - - -def _base_ppd( - *, - model: str, - filter_line: str, - extra: str, - single_pagesize: bool = False, - page_size: tuple[str, str, str] = ("Letter", "612 792", "18 36 594 756"), -) -> str: - page_name, page_dimension, imageable_area = page_size - page_region = "" if single_pagesize else ( - '*OpenUI *PageRegion: PickOne\n' - f'*DefaultPageRegion: {page_name}\n' - f'*PageRegion {page_name}: "<>setpagedevice"\n' - '*CloseUI: *PageRegion\n' - ) - return f"""*PPD-Adobe: "4.3" -*FormatVersion: "4.3" -*FileVersion: "1.0" -*LanguageVersion: English -*LanguageEncoding: ISOLatin1 -*Manufacturer: "SMT-Fuzzer" -*ModelName: "{_escape(model)}" -*ShortNickName: "SMTTemplate" -*NickName: "{_escape(model)}" -*PCFileName: "SMTPPD.PPD" -*Product: "(SMTTemplate)" -*PSVersion: "(3010) 0" -*cupsVersion: 1.0 -*cupsModelNumber: 0 -*cupsManualCopies: False -{filter_line} -{extra}*OpenUI *PageSize: PickOne -*DefaultPageSize: {page_name} -*PageSize {page_name}: "<>setpagedevice" -*CloseUI: *PageSize -{page_region}*DefaultImageableArea: {page_name} -*ImageableArea {page_name}: "{imageable_area}" -*DefaultPaperDimension: {page_name} -*PaperDimension {page_name}: "{page_dimension}" -""" - - -def _escape(value: str) -> str: - return value.replace("\\", "\\\\").replace('"', '\\"') - - -def _coverage_options(case_index: int) -> str: - slots = synthesize_ppd_slots(case_index) - color_model = ["Gray", "RGB", "CMYK", "Black"][slots.color_model_index] - quality = ["Draft", "Normal", "High", "Photo"][slots.quality_index] - media = ["Plain", "Glossy", "Transparency", "Envelope"][slots.media_index] - duplex = ["None", "DuplexNoTumble", "DuplexTumble"][slots.duplex_index] - return f"""*OpenUI *ColorModel: PickOne -*DefaultColorModel: {color_model} -*ColorModel Gray/Gray: "<>setpagedevice" -*ColorModel RGB/RGB: "<>setpagedevice" -*ColorModel CMYK/CMYK: "<>setpagedevice" -*ColorModel Black/Black: "<>setpagedevice" -*CloseUI: *ColorModel -*OpenUI *PrintQuality: PickOne -*DefaultPrintQuality: {quality} -*PrintQuality Draft/Draft: "<>setpagedevice" -*PrintQuality Normal/Normal: "<>setpagedevice" -*PrintQuality High/High: "<>setpagedevice" -*PrintQuality Photo/Photo: "<>setpagedevice" -*CloseUI: *PrintQuality -*OpenUI *MediaType: PickOne -*DefaultMediaType: {media} -*MediaType Plain/Plain: "<>setpagedevice" -*MediaType Glossy/Glossy: "<>setpagedevice" -*MediaType Transparency/Transparency: "<>setpagedevice" -*MediaType Envelope/Envelope: "<>setpagedevice" -*CloseUI: *MediaType -*OpenUI *Duplex: PickOne -*DefaultDuplex: {duplex} -*Duplex None/Off: "<>setpagedevice" -*Duplex DuplexNoTumble/Long edge: "<>setpagedevice" -*Duplex DuplexTumble/Short edge: "<>setpagedevice" -*CloseUI: *Duplex -""" diff --git a/parser-fuzzers/src/parser_fuzzers/generator/solver.py b/parser-fuzzers/src/parser_fuzzers/generator/solver.py deleted file mode 100644 index ac7ab5e..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/solver.py +++ /dev/null @@ -1,180 +0,0 @@ -from __future__ import annotations - -import time -from typing import Callable - -from parser_fuzzers.models import BranchEvent, Patch, SolverResult - - -class MissingSolverError(RuntimeError): - pass - - -def read_event_value(event: BranchEvent, input_bytes: bytes) -> int: - end = event.offset + event.width - if end > len(input_bytes): - raise ValueError( - f"event reads bytes [{event.offset}, {end}), but input has {len(input_bytes)} bytes" - ) - return int.from_bytes(input_bytes[event.offset:end], event.endianness, signed=False) - - -def _signed_value(value: int, bits: int) -> int: - sign_bit = 1 << (bits - 1) - mask = (1 << bits) - 1 - value &= mask - return value - (1 << bits) if value & sign_bit else value - - -def condition_holds(event: BranchEvent, value: int) -> bool: - bits = event.width * 8 - mask = (1 << bits) - 1 - lhs_unsigned = value & mask - rhs_unsigned = event.rhs & mask - lhs_signed = _signed_value(value, bits) - rhs_signed = _signed_value(event.rhs, bits) - - checks: dict[str, Callable[[], bool]] = { - "eq": lambda: lhs_unsigned == rhs_unsigned, - "ne": lambda: lhs_unsigned != rhs_unsigned, - "ult": lambda: lhs_unsigned < rhs_unsigned, - "ule": lambda: lhs_unsigned <= rhs_unsigned, - "ugt": lambda: lhs_unsigned > rhs_unsigned, - "uge": lambda: lhs_unsigned >= rhs_unsigned, - "slt": lambda: lhs_signed < rhs_signed, - "sle": lambda: lhs_signed <= rhs_signed, - "sgt": lambda: lhs_signed > rhs_signed, - "sge": lambda: lhs_signed >= rhs_signed, - } - return checks[event.op]() - - -def solve_event(event: BranchEvent, input_bytes: bytes, *, allow_fallback: bool = False) -> SolverResult: - start = time.perf_counter() - old_value = read_event_value(event, input_bytes) - if condition_holds(event, old_value): - return SolverResult( - status="already_satisfied", - solver_ms=_elapsed_ms(start), - patches=[], - reason="input already satisfies branch event", - event=event, - ) - - try: - patch_value = _solve_with_z3(event) - backend = "z3" - except MissingSolverError: - if not allow_fallback: - raise - patch_value = _solve_by_bounded_search(event) - backend = "bounded-fallback" - - if patch_value is None: - return SolverResult( - status="unsat", - solver_ms=_elapsed_ms(start), - patches=[], - reason=f"no value found by {backend}", - event=event, - ) - - old_bytes = old_value.to_bytes(event.width, event.endianness, signed=False) - new_bytes = patch_value.to_bytes(event.width, event.endianness, signed=False) - return SolverResult( - status="sat", - solver_ms=_elapsed_ms(start), - patches=[ - Patch( - offset=event.offset, - old_hex=old_bytes.hex(), - new_hex=new_bytes.hex(), - width=event.width, - ) - ], - reason=f"solved with {backend}", - event=event, - ) - - -def _elapsed_ms(start: float) -> float: - return round((time.perf_counter() - start) * 1000.0, 3) - - -def _solve_with_z3(event: BranchEvent) -> int | None: - try: - import z3 # type: ignore - except ImportError as exc: - raise MissingSolverError( - "z3-solver is not installed. Install project dependencies with " - "`python3 -m pip install -r requirements.txt`." - ) from exc - - bits = event.width * 8 - mask = (1 << bits) - 1 - field = z3.BitVec("field", bits) - rhs = z3.BitVecVal(event.rhs & mask, bits) - solver = z3.Solver() - solver.add(_z3_constraint(z3, field, rhs, event.op)) - result = solver.check() - if result != z3.sat: - return None - model_value = solver.model()[field] - if model_value is None: - return None - return int(model_value.as_long()) & mask - - -def _z3_constraint(z3_module, lhs, rhs, op: str): - if op == "eq": - return lhs == rhs - if op == "ne": - return lhs != rhs - if op == "ult": - return z3_module.ULT(lhs, rhs) - if op == "ule": - return z3_module.ULE(lhs, rhs) - if op == "ugt": - return z3_module.UGT(lhs, rhs) - if op == "uge": - return z3_module.UGE(lhs, rhs) - if op == "slt": - return lhs < rhs - if op == "sle": - return lhs <= rhs - if op == "sgt": - return lhs > rhs - if op == "sge": - return lhs >= rhs - raise ValueError(f"unsupported op: {op}") - - -def _solve_by_bounded_search(event: BranchEvent) -> int | None: - bits = event.width * 8 - max_value = 1 << bits - if bits <= 16: - candidates = range(max_value) - else: - candidates = _wide_candidates(event, max_value) - for candidate in candidates: - if condition_holds(event, candidate): - return candidate - return None - - -def _wide_candidates(event: BranchEvent, max_value: int) -> list[int]: - mask = max_value - 1 - rhs = event.rhs & mask - sign_bit = 1 << (event.width * 8 - 1) - candidates = { - 0, - 1, - mask, - rhs, - (rhs - 1) & mask, - (rhs + 1) & mask, - sign_bit, - (sign_bit - 1) & mask, - (sign_bit + 1) & mask, - } - return sorted(candidates) diff --git a/parser-fuzzers/src/parser_fuzzers/generator/source_constraints.py b/parser-fuzzers/src/parser_fuzzers/generator/source_constraints.py deleted file mode 100644 index 04c5389..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/source_constraints.py +++ /dev/null @@ -1,499 +0,0 @@ -from __future__ import annotations - -import json -import os -import re -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any, Iterable, Sequence - - -SOURCE_EXTENSIONS = {".c", ".cc", ".cpp", ".cxx", ".h", ".hh", ".hpp", ".hxx"} -CONTROL_RE = re.compile( - r"\b(if|else\s+if|switch|case|while|for|assert|return)\b|" - r"\b(strcasecmp|strcmp|strncmp|memcmp|memchr|strstr)\b" -) -OP_RE = re.compile(r"==|!=|<=|>=|<|>") -STRING_RE = re.compile(r'"((?:[^"\\]|\\.)*)"') - - -FIELD_PATTERNS: tuple[tuple[str, tuple[str, ...], tuple[str, ...]], ...] = ( - ("width", ("cups", "pwg"), (r"\bcupsWidth\b", r"\bwidth\b", r"\bWidth\b")), - ("height", ("cups", "pwg"), (r"\bcupsHeight\b", r"\bheight\b", r"\bHeight\b")), - ( - "bytes_per_line", - ("cups", "pwg"), - (r"\bcupsBytesPerLine\b", r"\bbytes[_-]?per[_-]?line\b", r"\bBytesPerLine\b"), - ), - ( - "bits_per_pixel", - ("cups", "pwg"), - (r"\bcupsBitsPerPixel\b", r"\bbits[_-]?per[_-]?pixel\b", r"\bBitsPerPixel\b"), - ), - ("row_count", ("cups", "pwg"), (r"\bcupsRowCount\b", r"\brow[_-]?count\b", r"\bRowCount\b")), - ("payload_rows", ("cups", "pwg"), (r"\bpayload[_-]?rows\b", r"\bdata[_-]?rows\b", r"\brows[_-]?written\b")), - ("x_res", ("cups", "pwg", "ppd"), (r"\bHWResolution\b", r"\bx[_-]?res\b", r"\bResolution\b")), - ("y_res", ("cups", "pwg", "ppd"), (r"\bHWResolution\b", r"\by[_-]?res\b", r"\bResolution\b")), - ("color_space", ("cups",), (r"\bcupsColorSpace\b", r"\bColorSpace\b")), - ("num_colors", ("cups",), (r"\bcupsNumColors\b", r"\bNumColors\b")), - ("color_order", ("cups",), (r"\bcupsColorOrder\b", r"\bColorOrder\b")), - ("compression", ("cups",), (r"\bcupsCompression\b", r"\bCompression\b")), -) - -OPTION_TOKENS = { - "PageSize", - "PageRegion", - "ColorModel", - "PrintQuality", - "MediaType", - "Duplex", - "Resolution", - "HWResolution", - "cupsFilter", - "cupsManualCopies", - "cupsColorSpace", - "cupsBitsPerPixel", - "cupsBytesPerLine", -} - -DYNAMIC_TOKEN_FIELD_MAP: dict[str, tuple[tuple[str, tuple[str, ...]], ...]] = { - "cupsBytesPerLine": (("bytes_per_line", ("cups", "pwg")),), - "cupsBitsPerPixel": (("bits_per_pixel", ("cups", "pwg")),), - "cupsColorSpace": (("color_space", ("cups",)),), - "HWResolution": (("x_res", ("cups", "pwg", "ppd")), ("y_res", ("cups", "pwg", "ppd"))), - "Resolution": (("x_res", ("cups", "pwg", "ppd")), ("y_res", ("cups", "pwg", "ppd"))), - "PageSize": (("width", ("ppd",)), ("height", ("ppd",))), - "PageRegion": (("width", ("ppd",)), ("height", ("ppd",))), - "ColorModel": (("color_space", ("cups", "ppd")),), - "cupsFilter": (), -} - - -@dataclass(frozen=True) -class SourceHint: - source_path: str - line: int - text: str - families: tuple[str, ...] - fields: tuple[str, ...] - operators: tuple[str, ...] - strings: tuple[str, ...] - kind: str - - -def mine_source_constraints( - source_roots: Sequence[str | Path], - *, - max_records: int = 2000, -) -> dict[str, Any]: - records: list[SourceHint] = [] - files_scanned = 0 - lines_scanned = 0 - field_counts: dict[str, dict[str, int]] = {"cups": {}, "pwg": {}, "ppd": {}} - option_counts: dict[str, int] = {} - kind_counts: dict[str, int] = {} - - for source_file in _iter_source_files(source_roots): - files_scanned += 1 - path_text = str(source_file) - path_family = _family_from_path(path_text) - try: - lines = source_file.read_text(encoding="utf-8", errors="replace").splitlines() - except OSError: - continue - for line_no, line in enumerate(lines, 1): - lines_scanned += 1 - hint = _hint_from_line(source_file, line_no, line, path_family) - if hint is None: - continue - for family in hint.families: - counts = field_counts.setdefault(family, {}) - for field in hint.fields: - counts[field] = counts.get(field, 0) + 1 - for token in hint.strings: - if token in OPTION_TOKENS: - option_counts[token] = option_counts.get(token, 0) + 1 - kind_counts[hint.kind] = kind_counts.get(hint.kind, 0) + 1 - if len(records) < max_records: - records.append(hint) - - profile = { - "schema_version": "source-constraint-hints-v1", - "source_roots": [str(Path(root)) for root in source_roots], - "summary": { - "files_scanned": files_scanned, - "lines_scanned": lines_scanned, - "records": len(records), - "records_truncated": len(records) >= max_records, - "kind_counts": dict(sorted(kind_counts.items())), - }, - "families": { - family: {"fields": dict(sorted(counts.items(), key=lambda item: (-item[1], item[0])))} - for family, counts in sorted(field_counts.items()) - }, - "ppd_options": dict(sorted(option_counts.items(), key=lambda item: (-item[1], item[0]))), - "records": [asdict(record) for record in records], - "template_bias": _template_bias(field_counts), - } - return profile - - -def write_source_constraint_profile(profile: dict[str, Any], output_path: str | Path) -> None: - destination = Path(output_path) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(json.dumps(profile, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def load_source_constraint_profile(path: str | Path) -> dict[str, Any]: - try: - return json.loads(Path(path).read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return {} - - -def active_source_constraint_key() -> str: - source_path = os.environ.get("SMT_FUZZER_SOURCE_CONSTRAINTS", "").strip() - dynamic_path = os.environ.get("SMT_FUZZER_DYNAMIC_CONSTRAINTS", "").strip() - if not source_path and not dynamic_path: - return "" - parts = [] - for path in (source_path, dynamic_path): - if path: - parts.append(_profile_identity(path)) - rate = os.environ.get("SMT_FUZZER_SOURCE_CONSTRAINT_RATE", "") - return f"{'|'.join(parts)}:rate={rate}" - - -def _profile_identity(path: str) -> str: - resolved = Path(path) - try: - stat = resolved.stat() - return f"{resolved.resolve()}:{stat.st_mtime_ns}:{stat.st_size}" - except OSError: - return str(resolved) - - -def choose_source_objective( - spec_name: str, - objectives: Sequence[Any], - slot: int, -) -> tuple[Any, bool]: - default = objectives[slot % len(objectives)] - profile = _active_profile() - if not profile or not _source_rate_allows(slot): - return default, False - family = _family_from_spec(spec_name) - preferred = _preferred_objective_names(profile, family) - if not preferred: - return default, False - by_name = {getattr(objective, "name", ""): objective for objective in objectives} - candidates = [by_name[name] for name in preferred if name in by_name] - if not candidates: - return default, False - return candidates[(slot // max(1, len(objectives))) % len(candidates)], True - - -def choose_source_feedback_variant(family: str, variant_count: int, slot: int) -> tuple[int, bool]: - default = slot % variant_count - profile = _active_profile() - if not profile or not _source_rate_allows(slot): - return default, False - variants = _preferred_feedback_variants(profile, family) - variants = [variant for variant in variants if 0 <= variant < variant_count] - if not variants: - return default, False - return variants[(slot // max(1, variant_count)) % len(variants)], True - - -def _iter_source_files(source_roots: Sequence[str | Path]) -> Iterable[Path]: - seen: set[Path] = set() - for root in source_roots: - path = Path(root) - if path.is_file() and path.suffix in SOURCE_EXTENSIONS: - resolved = path.resolve() - if resolved not in seen: - seen.add(resolved) - yield path - continue - if not path.exists(): - continue - for source_file in sorted(path.rglob("*")): - if source_file.is_file() and source_file.suffix in SOURCE_EXTENSIONS: - resolved = source_file.resolve() - if resolved in seen: - continue - seen.add(resolved) - yield source_file - - -def _hint_from_line(source_file: Path, line_no: int, line: str, path_family: str) -> SourceHint | None: - stripped = line.strip() - if not stripped or stripped.startswith(("//", "/*", "*")): - return None - fields, field_families = _fields_in_line(stripped) - strings = _strings_in_line(stripped) - operators = tuple(sorted(set(OP_RE.findall(stripped)))) - control = CONTROL_RE.search(stripped) is not None - interesting_string = any(token in OPTION_TOKENS or token.startswith("application/") for token in strings) - if not fields and not interesting_string: - return None - if not control and not operators and not interesting_string: - return None - - families = set(field_families) - if path_family: - families.add(path_family) - if interesting_string: - families.add("ppd") - kind = _hint_kind(stripped, operators, strings) - return SourceHint( - source_path=str(source_file), - line=line_no, - text=stripped[:240], - families=tuple(sorted(families or {"unknown"})), - fields=tuple(sorted(fields)), - operators=operators, - strings=strings, - kind=kind, - ) - - -def _fields_in_line(line: str) -> tuple[set[str], set[str]]: - fields: set[str] = set() - families: set[str] = set() - for name, mapped_families, patterns in FIELD_PATTERNS: - if any(re.search(pattern, line, flags=re.IGNORECASE) for pattern in patterns): - fields.add(name) - families.update(mapped_families) - return fields, families - - -def _strings_in_line(line: str) -> tuple[str, ...]: - values = [] - for match in STRING_RE.finditer(line): - value = match.group(1) - if len(value) > 96: - continue - if value in OPTION_TOKENS or value.startswith("application/") or value in {"RaS2", "3SaR", "2SaR"}: - values.append(value) - return tuple(sorted(set(values))) - - -def _hint_kind(line: str, operators: tuple[str, ...], strings: tuple[str, ...]) -> str: - if any(func in line for func in ("strcmp", "strcasecmp", "strncmp", "memcmp", "strstr")): - return "string-or-memory-compare" - if "switch" in line or re.search(r"\bcase\b", line): - return "switch-or-case" - if operators: - return "bounds-or-equality" - if strings: - return "string-token" - return "field-reference" - - -def _family_from_path(path_text: str) -> str: - lower = path_text.lower() - if "ppd" in lower: - return "ppd" - if "pwg" in lower: - return "pwg" - if "raster" in lower or "cups" in lower: - return "cups" - return "" - - -def _family_from_spec(spec_name: str) -> str: - lower = spec_name.lower() - if "pwg" in lower: - return "pwg" - if "cups" in lower: - return "cups" - return "cups" - - -def _template_bias(field_counts: dict[str, dict[str, int]]) -> dict[str, Any]: - return { - family: { - "preferred_objectives": _objective_names_from_fields(counts, cups=(family == "cups")), - "preferred_feedback_variants": _feedback_variants_from_fields(counts), - } - for family, counts in sorted(field_counts.items()) - } - - -def _preferred_objective_names(profile: dict[str, Any], family: str) -> list[str]: - bias = profile.get("template_bias", {}).get(family, {}) - names = bias.get("preferred_objectives", []) - return [str(name) for name in names] - - -def _preferred_feedback_variants(profile: dict[str, Any], family: str) -> list[int]: - bias = profile.get("template_bias", {}).get(family, {}) - variants = bias.get("preferred_feedback_variants", []) - parsed = [] - for variant in variants: - try: - parsed.append(int(variant)) - except (TypeError, ValueError): - continue - return parsed - - -def _objective_names_from_fields(counts: dict[str, int], *, cups: bool) -> list[str]: - names: list[str] = [] - if counts.get("bytes_per_line", 0): - names.extend(["short_line", "padded_line", "valid_tight" if cups else "valid_exact"]) - if counts.get("row_count", 0): - names.extend(["row_count_short", "row_count_long"]) - if counts.get("payload_rows", 0): - names.extend(["payload_short", "payload_extra"]) - if counts.get("width", 0) or counts.get("height", 0) or counts.get("bits_per_pixel", 0): - names.extend(["valid_aligned" if cups else "valid_exact", "padded_line"]) - if counts.get("x_res", 0) or counts.get("y_res", 0): - names.extend(["valid_aligned" if cups else "valid_exact"]) - return _unique(names) - - -def _feedback_variants_from_fields(counts: dict[str, int]) -> list[int]: - variants: list[int] = [] - if counts.get("bytes_per_line", 0): - variants.extend([1, 2, 3, 4, 9, 10]) - if counts.get("row_count", 0): - variants.extend([5, 6, 11, 12]) - if counts.get("payload_rows", 0): - variants.extend([7, 8, 12, 13]) - if counts.get("width", 0) or counts.get("height", 0) or counts.get("bits_per_pixel", 0): - variants.extend([0, 2, 10]) - return _unique_ints(variants) - - -def _unique(values: Iterable[str]) -> list[str]: - seen: set[str] = set() - result: list[str] = [] - for value in values: - if value in seen: - continue - seen.add(value) - result.append(value) - return result - - -def _unique_ints(values: Iterable[int]) -> list[int]: - seen: set[int] = set() - result: list[int] = [] - for value in values: - if value in seen: - continue - seen.add(value) - result.append(value) - return result - - -def _source_rate_allows(slot: int) -> bool: - rate = _source_rate() - if rate <= 0: - return False - if rate >= 1: - return True - bucket = ((slot + 1) * 1103515245 + 12345) % 10000 - return bucket < int(rate * 10000) - - -def _source_rate() -> float: - value = os.environ.get("SMT_FUZZER_SOURCE_CONSTRAINT_RATE", "0.5") - try: - return max(0.0, min(1.0, float(value))) - except ValueError: - return 0.5 - - -def _active_profile() -> dict[str, Any]: - source_path = os.environ.get("SMT_FUZZER_SOURCE_CONSTRAINTS", "").strip() - dynamic_path = os.environ.get("SMT_FUZZER_DYNAMIC_CONSTRAINTS", "").strip() - profiles = [] - if source_path: - profiles.append(_cached_profile(source_path)) - if dynamic_path: - dynamic_profile = _cached_profile(dynamic_path) - profiles.append(_source_profile_from_dynamic(dynamic_profile)) - if not profiles: - return {} - return _merge_profiles(profiles) - - -def _cached_profile(path: str) -> dict[str, Any]: - # Manual cache keeps patched test environments predictable because the key is - # the file identity, not unrelated global state. - cache = getattr(_cached_profile, "_cache", {}) - try: - stat = Path(path).stat() - key = f"{path}:{stat.st_mtime_ns}:{stat.st_size}" - except OSError: - key = path - if key not in cache: - cache[key] = load_source_constraint_profile(path) - setattr(_cached_profile, "_cache", cache) - return cache[key] - - -def _source_profile_from_dynamic(profile: dict[str, Any]) -> dict[str, Any]: - field_counts: dict[str, dict[str, int]] = {"cups": {}, "pwg": {}, "ppd": {}} - tokens: dict[str, int] = {} - for source in (profile.get("tokens", {}), profile.get("ppd_options", {}), profile.get("magic_tokens", {})): - if isinstance(source, dict): - for token, count in source.items(): - try: - tokens[str(token)] = tokens.get(str(token), 0) + int(count) - except (TypeError, ValueError): - continue - for token, count in tokens.items(): - for field, families in DYNAMIC_TOKEN_FIELD_MAP.get(token, ()): - for family in families: - family_counts = field_counts.setdefault(family, {}) - family_counts[field] = family_counts.get(field, 0) + count - return { - "schema_version": "source-constraint-hints-v1+dynamic", - "summary": {"dynamic_compare_records": profile.get("summary", {}).get("compare_records", 0)}, - "families": { - family: {"fields": dict(sorted(counts.items(), key=lambda item: (-item[1], item[0])))} - for family, counts in sorted(field_counts.items()) - }, - "ppd_options": dict(profile.get("ppd_options", {})), - "records": [], - "template_bias": _template_bias(field_counts), - } - - -def _merge_profiles(profiles: list[dict[str, Any]]) -> dict[str, Any]: - field_counts: dict[str, dict[str, int]] = {"cups": {}, "pwg": {}, "ppd": {}} - ppd_options: dict[str, int] = {} - for profile in profiles: - families = profile.get("families", {}) - if isinstance(families, dict): - for family, payload in families.items(): - fields = payload.get("fields", {}) if isinstance(payload, dict) else {} - if not isinstance(fields, dict): - continue - family_counts = field_counts.setdefault(str(family), {}) - for field, count in fields.items(): - try: - family_counts[str(field)] = family_counts.get(str(field), 0) + int(count) - except (TypeError, ValueError): - continue - options = profile.get("ppd_options", {}) - if isinstance(options, dict): - for token, count in options.items(): - try: - ppd_options[str(token)] = ppd_options.get(str(token), 0) + int(count) - except (TypeError, ValueError): - continue - return { - "schema_version": "source-constraint-hints-v1+merged", - "families": { - family: {"fields": dict(sorted(counts.items(), key=lambda item: (-item[1], item[0])))} - for family, counts in sorted(field_counts.items()) - }, - "ppd_options": dict(sorted(ppd_options.items(), key=lambda item: (-item[1], item[0]))), - "records": [], - "template_bias": _template_bias(field_counts), - } diff --git a/parser-fuzzers/src/parser_fuzzers/generator/structure_mutator.py b/parser-fuzzers/src/parser_fuzzers/generator/structure_mutator.py deleted file mode 100644 index b8db9c2..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/structure_mutator.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from parser_fuzzers.constraint_repair import ImageRepairResult, repair_image_goal -from parser_fuzzers.output_feedback import OutputFeedbackProfile, choose_image_goal -from parser_fuzzers.template_feedback import FeedbackSeed - - -@dataclass(frozen=True) -class MutatedImageStructure: - image_format: str - width: int - height: int - channels: int - maxval: int - payload_delta: int - comment_style: int - png_interlace: int - objective: str - solved_by: str - output_format_goal: str - - -def mutate_image_structure( - *, - target_id: str, - slot: int, - expansion_level: int, - seed: FeedbackSeed | None = None, - output_feedback: OutputFeedbackProfile | None = None, -) -> MutatedImageStructure: - goal = choose_image_goal(target_id=target_id, slot=slot, profile=output_feedback) - repaired = repair_image_goal( - goal=goal, - slot=slot, - expansion_level=expansion_level, - seed=seed, - target_id=target_id, - ) - return _from_repair(repaired, goal.output_format) - - -def _from_repair(repaired: ImageRepairResult, output_format: str) -> MutatedImageStructure: - return MutatedImageStructure( - image_format=repaired.image_format, - width=repaired.width, - height=repaired.height, - channels=repaired.channels, - maxval=repaired.maxval, - payload_delta=repaired.payload_delta, - comment_style=repaired.comment_style, - png_interlace=repaired.png_interlace, - objective=repaired.objective, - solved_by=repaired.solved_by, - output_format_goal=output_format, - ) diff --git a/parser-fuzzers/src/parser_fuzzers/generator/structured_templates.py b/parser-fuzzers/src/parser_fuzzers/generator/structured_templates.py deleted file mode 100644 index a4d5c5d..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/structured_templates.py +++ /dev/null @@ -1,816 +0,0 @@ -from __future__ import annotations - -import os -from dataclasses import dataclass -from functools import lru_cache -from typing import Any, Callable, Iterable, Mapping - -from parser_fuzzers.source_constraints import ( - active_source_constraint_key, - choose_source_feedback_variant, - choose_source_objective, -) -from parser_fuzzers.template_feedback import FeedbackSeed, load_feedback_profile -from parser_fuzzers.z3_guard import Z3_LOCK - -try: - import z3 -except ImportError: # pragma: no cover - exercised only in minimal environments - z3 = None - - -ConstraintBuilder = Callable[[Mapping[str, Any], int], Iterable[Any]] -FallbackBuilder = Callable[[int], dict[str, int]] - - -CUPS_STRUCTURAL_PERIOD = 384 -PWG_STRUCTURAL_PERIOD = 320 -CUPS_FEEDBACK_PERIOD = 768 -PWG_FEEDBACK_PERIOD = 640 - - -@dataclass(frozen=True) -class FieldSpec: - name: str - values: tuple[int, ...] | None = None - min_value: int | None = None - max_value: int | None = None - pin: bool = True - salt: int = 1 - offset: int = 0 - stride: int = 11 - - -@dataclass(frozen=True) -class TemplateObjective: - name: str - constraints: ConstraintBuilder - - -@dataclass(frozen=True) -class TemplateSpec: - name: str - period: int - fields: tuple[FieldSpec, ...] - constraints: tuple[ConstraintBuilder, ...] - objectives: tuple[TemplateObjective, ...] - fallback: FallbackBuilder - - -@dataclass(frozen=True) -class TemplateInstance: - spec_name: str - objective: str - case_index: int - fields: dict[str, int] - solved_by: str - - def get(self, name: str) -> int: - return self.fields[name] - - -def solve_template(spec: TemplateSpec, case_index: int) -> TemplateInstance: - slot = case_index % spec.period - objective, source_biased = choose_source_objective(spec.name, spec.objectives, slot) - if z3 is None: - return _fallback_instance(spec, objective, slot) - - for loosen in range(4): - solver = z3.Solver() - variables = {field.name: z3.Int(field.name) for field in spec.fields} - for field in spec.fields: - _add_field_domain(solver, variables[field.name], field) - if field.pin and field.values and loosen == 0: - solver.add(variables[field.name] == _selected_value(field, slot)) - elif field.pin and field.values and loosen == 1 and field.name not in {"x_res", "y_res"}: - solver.add(variables[field.name] == _selected_value(field, slot)) - elif field.pin and field.values and loosen == 2 and field.name in {"width", "height", "bits_per_pixel"}: - solver.add(variables[field.name] == _selected_value(field, slot)) - for builder in spec.constraints: - solver.add(*builder(variables, slot)) - solver.add(*objective.constraints(variables, slot)) - if solver.check() == z3.sat: - model = solver.model() - return TemplateInstance( - spec_name=spec.name, - objective=objective.name, - case_index=case_index, - fields={ - field.name: int(model.evaluate(variables[field.name], model_completion=True).as_long()) - for field in spec.fields - }, - solved_by="z3-source" if source_biased else "z3", - ) - return _fallback_instance(spec, objective, slot) - - -def cups_structural_instance(case_index: int) -> TemplateInstance: - with Z3_LOCK: - slot = case_index % CUPS_STRUCTURAL_PERIOD - source_key = active_source_constraint_key() - if source_key: - return _cups_source_structural_instance(slot, source_key) - return _cups_structural_instance(slot) - - -@lru_cache(maxsize=None) -def _cups_structural_instance(slot: int) -> TemplateInstance: - return solve_template(CUPS_RASTER_STRUCTURAL_SPEC, slot) - - -@lru_cache(maxsize=None) -def _cups_source_structural_instance(slot: int, source_key: str) -> TemplateInstance: - _ = source_key - return solve_template(CUPS_RASTER_STRUCTURAL_SPEC, slot) - - -def pwg_structural_instance(case_index: int) -> TemplateInstance: - with Z3_LOCK: - slot = case_index % PWG_STRUCTURAL_PERIOD - source_key = active_source_constraint_key() - if source_key: - return _pwg_source_structural_instance(slot, source_key) - return _pwg_structural_instance(slot) - - -@lru_cache(maxsize=None) -def _pwg_structural_instance(slot: int) -> TemplateInstance: - return solve_template(PWG_RASTER_STRUCTURAL_SPEC, slot) - - -@lru_cache(maxsize=None) -def _pwg_source_structural_instance(slot: int, source_key: str) -> TemplateInstance: - _ = source_key - return solve_template(PWG_RASTER_STRUCTURAL_SPEC, slot) - - -def cups_feedback_instance(case_index: int) -> TemplateInstance: - with Z3_LOCK: - feedback_path = os.environ.get("SMT_FUZZER_TEMPLATE_FEEDBACK", "") - if not feedback_path: - return cups_structural_instance(case_index + CUPS_STRUCTURAL_PERIOD) - expansion_level = _feedback_expansion_level() - source_key = active_source_constraint_key() - return _cups_feedback_instance(case_index % CUPS_FEEDBACK_PERIOD, feedback_path, expansion_level, source_key) - - -@lru_cache(maxsize=None) -def _cups_feedback_instance(slot: int, feedback_path: str, expansion_level: int, source_key: str) -> TemplateInstance: - _ = source_key - seeds = _feedback_seeds(feedback_path, "cups") - if not seeds: - return cups_structural_instance(slot + CUPS_STRUCTURAL_PERIOD) - return _solve_cups_feedback(seeds[slot % len(seeds)], slot, expansion_level) - - -def pwg_feedback_instance(case_index: int) -> TemplateInstance: - with Z3_LOCK: - feedback_path = os.environ.get("SMT_FUZZER_TEMPLATE_FEEDBACK", "") - if not feedback_path: - return pwg_structural_instance(case_index + PWG_STRUCTURAL_PERIOD) - expansion_level = _feedback_expansion_level() - source_key = active_source_constraint_key() - return _pwg_feedback_instance(case_index % PWG_FEEDBACK_PERIOD, feedback_path, expansion_level, source_key) - - -@lru_cache(maxsize=None) -def _pwg_feedback_instance(slot: int, feedback_path: str, expansion_level: int, source_key: str) -> TemplateInstance: - _ = source_key - seeds = _feedback_seeds(feedback_path, "pwg") - if not seeds: - return pwg_structural_instance(slot + PWG_STRUCTURAL_PERIOD) - return _solve_pwg_feedback(seeds[slot % len(seeds)], slot, expansion_level) - - -def _add_field_domain(solver: Any, variable: Any, field: FieldSpec) -> None: - if field.values is not None: - solver.add(z3.Or(*[variable == value for value in sorted(set(field.values))])) - return - if field.min_value is not None: - solver.add(variable >= field.min_value) - if field.max_value is not None: - solver.add(variable <= field.max_value) - - -def _selected_value(field: FieldSpec, slot: int) -> int: - assert field.values is not None - values = field.values - index = (slot * field.salt + slot // max(1, field.stride) + field.offset) % len(values) - return values[index] - - -def _fallback_instance(spec: TemplateSpec, objective: TemplateObjective, slot: int) -> TemplateInstance: - return TemplateInstance( - spec_name=spec.name, - objective=objective.name, - case_index=slot, - fields=spec.fallback(slot), - solved_by="fallback", - ) - - -def _ceil_div(numerator: Any, denominator: int) -> Any: - return (numerator + denominator - 1) / denominator - - -def _align(value: Any, alignment: int) -> Any: - return _ceil_div(value, alignment) * alignment - - -CUPS_WIDTHS = (1, 2, 3, 4, 7, 8, 9, 15, 16, 17, 31, 32, 33, 48, 63, 64, 65, 96, 127, 128, 129, 255, 511) -CUPS_HEIGHTS = (1, 2, 3, 4, 5, 8, 16) -CUPS_RESOLUTIONS = (72, 75, 100, 150, 203, 300, 360, 600, 720, 1200, 2400, 32768, 65535) -CUPS_COLOR_TUPLES = ( - (3, 1, 8), - (18, 1, 8), - (18, 1, 16), - (1, 3, 24), - (1, 3, 32), - (6, 4, 32), - (6, 4, 24), -) - - -def _cups_base_constraints(v: Mapping[str, Any], slot: int) -> Iterable[Any]: - raw_bpl = _ceil_div(v["width"] * v["bits_per_pixel"], 8) - return [ - z3.Or( - *[ - z3.And(v["color_space"] == color_space, v["num_colors"] == colors, v["bits_per_pixel"] == bpp) - for color_space, colors, bpp in CUPS_COLOR_TUPLES - ] - ), - v["bytes_per_line"] >= 1, - v["bytes_per_line"] <= _align(raw_bpl, 8) + 32, - v["row_count"] >= 1, - v["row_count"] <= v["height"] + 2, - v["payload_rows"] >= 1, - v["payload_rows"] <= v["height"] + 2, - ] - - -def _cups_objective(kind: str) -> ConstraintBuilder: - def constraints(v: Mapping[str, Any], slot: int) -> Iterable[Any]: - raw_bpl = _ceil_div(v["width"] * v["bits_per_pixel"], 8) - aligned_bpl = _align(raw_bpl, 8) - delta = [1, 3, 7, 15][slot % 4] - if kind == "valid_aligned": - return [v["bytes_per_line"] == aligned_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if kind == "valid_tight": - return [v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if kind == "padded_line": - return [v["bytes_per_line"] == aligned_bpl + delta, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if kind == "short_line": - return [raw_bpl > 1, v["bytes_per_line"] == raw_bpl - 1, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if kind == "row_count_short": - return [v["height"] > 1, v["bytes_per_line"] == aligned_bpl, v["row_count"] == v["height"] - 1, v["payload_rows"] == v["height"]] - if kind == "row_count_long": - return [v["bytes_per_line"] == aligned_bpl, v["row_count"] == v["height"] + 1, v["payload_rows"] == v["height"]] - if kind == "payload_short": - return [v["height"] > 1, v["bytes_per_line"] == aligned_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"] - 1] - if kind == "payload_extra": - return [v["bytes_per_line"] == aligned_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"] + 1] - raise ValueError(kind) - - return constraints - - -def _fallback_cups(slot: int) -> dict[str, int]: - color_space, num_colors, bits_per_pixel = CUPS_COLOR_TUPLES[(slot * 3 + slot // 7) % len(CUPS_COLOR_TUPLES)] - width = CUPS_WIDTHS[(slot * 7 + slot // 5) % len(CUPS_WIDTHS)] - height = CUPS_HEIGHTS[(slot * 5 + slot // 11) % len(CUPS_HEIGHTS)] - raw_bpl = (width * bits_per_pixel + 7) // 8 - aligned_bpl = ((raw_bpl + 7) // 8) * 8 - objective = slot % 8 - bytes_per_line = aligned_bpl - row_count = height - payload_rows = height - if objective == 1: - bytes_per_line = raw_bpl - elif objective == 2: - bytes_per_line = aligned_bpl + [1, 3, 7, 15][slot % 4] - elif objective == 3 and raw_bpl > 1: - bytes_per_line = raw_bpl - 1 - elif objective == 4 and height > 1: - row_count = height - 1 - elif objective == 5: - row_count = height + 1 - elif objective == 6 and height > 1: - payload_rows = height - 1 - elif objective == 7: - payload_rows = height + 1 - return { - "width": width, - "height": height, - "compression": (0, 0, 1, 10)[slot % 4], - "num_colors": num_colors, - "color_space": color_space, - "color_order": (0, 0, 1, 2)[slot % 4], - "bits_per_pixel": bits_per_pixel, - "pages": (1, 1, 2, 3)[slot % 4], - "x_res": CUPS_RESOLUTIONS[(slot * 3 + 1) % len(CUPS_RESOLUTIONS)], - "y_res": CUPS_RESOLUTIONS[(slot * 5 + 2) % len(CUPS_RESOLUTIONS)], - "bytes_per_line": bytes_per_line, - "row_count": row_count, - "payload_rows": payload_rows, - } - - -CUPS_RASTER_STRUCTURAL_SPEC = TemplateSpec( - name="cups_raster_structural", - period=CUPS_STRUCTURAL_PERIOD, - fields=( - FieldSpec("width", values=CUPS_WIDTHS, salt=7, stride=5), - FieldSpec("height", values=CUPS_HEIGHTS, salt=5, stride=11), - FieldSpec("compression", values=(0, 0, 1, 10), salt=3), - FieldSpec("num_colors", values=(1, 3, 4), pin=False), - FieldSpec("color_space", values=(1, 3, 6, 18), pin=False), - FieldSpec("color_order", values=(0, 0, 1, 2), salt=5), - FieldSpec("bits_per_pixel", values=(8, 16, 24, 32), pin=False), - FieldSpec("pages", values=(1, 1, 2, 3), salt=11), - FieldSpec("x_res", values=CUPS_RESOLUTIONS, salt=3, offset=1), - FieldSpec("y_res", values=CUPS_RESOLUTIONS, salt=5, offset=2), - FieldSpec("bytes_per_line", min_value=1, max_value=4096, pin=False), - FieldSpec("row_count", min_value=1, max_value=64, pin=False), - FieldSpec("payload_rows", min_value=1, max_value=64, pin=False), - ), - constraints=(_cups_base_constraints,), - objectives=tuple( - TemplateObjective(name, _cups_objective(name)) - for name in ( - "valid_aligned", - "valid_tight", - "padded_line", - "short_line", - "row_count_short", - "row_count_long", - "payload_short", - "payload_extra", - ) - ), - fallback=_fallback_cups, -) - - -PWG_WIDTHS = (1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 65, 127, 128, 129, 255, 511) -PWG_HEIGHTS = (1, 2, 3, 4, 8, 16) -PWG_BPP = (1, 8, 16, 24, 32) -PWG_RESOLUTIONS = (72, 150, 203, 300, 360, 600, 720, 1200, 2400, 32768, 65535, 65536, 2147483647) - - -def _pwg_base_constraints(v: Mapping[str, Any], slot: int) -> Iterable[Any]: - raw_bpl = _ceil_div(v["width"] * v["bits_per_pixel"], 8) - return [ - v["bytes_per_line"] >= 1, - v["bytes_per_line"] <= raw_bpl + 64, - v["row_count"] >= 1, - v["row_count"] <= v["height"] + 2, - v["payload_rows"] >= 1, - v["payload_rows"] <= v["height"] + 2, - ] - - -def _pwg_objective(kind: str) -> ConstraintBuilder: - def constraints(v: Mapping[str, Any], slot: int) -> Iterable[Any]: - raw_bpl = _ceil_div(v["width"] * v["bits_per_pixel"], 8) - delta = [1, 2, 4, 8, 16][slot % 5] - if kind == "valid_exact": - return [v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if kind == "padded_line": - return [v["bytes_per_line"] == raw_bpl + delta, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if kind == "short_line": - return [raw_bpl > 1, v["bytes_per_line"] == raw_bpl - 1, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if kind == "row_count_short": - return [v["height"] > 1, v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"] - 1, v["payload_rows"] == v["height"]] - if kind == "row_count_long": - return [v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"] + 1, v["payload_rows"] == v["height"]] - if kind == "payload_short": - return [v["height"] > 1, v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"] - 1] - if kind == "payload_extra": - return [v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"] + 1] - raise ValueError(kind) - - return constraints - - -def _fallback_pwg(slot: int) -> dict[str, int]: - width = PWG_WIDTHS[(slot * 5 + slot // 3) % len(PWG_WIDTHS)] - height = PWG_HEIGHTS[(slot * 7 + slot // 13) % len(PWG_HEIGHTS)] - bits_per_pixel = PWG_BPP[(slot * 3 + slot // 17) % len(PWG_BPP)] - raw_bpl = max(1, (width * bits_per_pixel + 7) // 8) - objective = slot % 7 - bytes_per_line = raw_bpl - row_count = height - payload_rows = height - if objective == 1: - bytes_per_line = raw_bpl + [1, 2, 4, 8, 16][slot % 5] - elif objective == 2 and raw_bpl > 1: - bytes_per_line = raw_bpl - 1 - elif objective == 3 and height > 1: - row_count = height - 1 - elif objective == 4: - row_count = height + 1 - elif objective == 5 and height > 1: - payload_rows = height - 1 - elif objective == 6: - payload_rows = height + 1 - return { - "width": width, - "height": height, - "bits_per_pixel": bits_per_pixel, - "x_res": PWG_RESOLUTIONS[(slot * 2 + 3) % len(PWG_RESOLUTIONS)], - "y_res": PWG_RESOLUTIONS[(slot * 5 + 1) % len(PWG_RESOLUTIONS)], - "pages": (1, 1, 2, 3)[slot % 4], - "bytes_per_line": bytes_per_line, - "row_count": row_count, - "payload_rows": payload_rows, - } - - -PWG_RASTER_STRUCTURAL_SPEC = TemplateSpec( - name="pwg_raster_structural", - period=PWG_STRUCTURAL_PERIOD, - fields=( - FieldSpec("width", values=PWG_WIDTHS, salt=5, stride=3), - FieldSpec("height", values=PWG_HEIGHTS, salt=7, stride=13), - FieldSpec("bits_per_pixel", values=PWG_BPP, salt=3, stride=17), - FieldSpec("x_res", values=PWG_RESOLUTIONS, salt=2, offset=3), - FieldSpec("y_res", values=PWG_RESOLUTIONS, salt=5, offset=1), - FieldSpec("pages", values=(1, 1, 2, 3), salt=11), - FieldSpec("bytes_per_line", min_value=1, max_value=4096, pin=False), - FieldSpec("row_count", min_value=1, max_value=64, pin=False), - FieldSpec("payload_rows", min_value=1, max_value=64, pin=False), - ), - constraints=(_pwg_base_constraints,), - objectives=tuple( - TemplateObjective(name, _pwg_objective(name)) - for name in ( - "valid_exact", - "padded_line", - "short_line", - "row_count_short", - "row_count_long", - "payload_short", - "payload_extra", - ) - ), - fallback=_fallback_pwg, -) - - -@lru_cache(maxsize=None) -def _loaded_feedback(path: str): - return load_feedback_profile(path) - - -def _feedback_seeds(path: str, kind: str) -> tuple[FeedbackSeed, ...]: - profile = _loaded_feedback(path) - seeds = profile.cups if kind == "cups" else profile.pwg - return tuple(seeds) - - -def _solve_cups_feedback(seed: FeedbackSeed, slot: int, expansion_level: int = 0) -> TemplateInstance: - seed = _sanitize_feedback_seed(seed, CUPS_RASTER_STRUCTURAL_SPEC) - variant, source_biased = choose_source_feedback_variant( - "cups", - _cups_feedback_variant_count(expansion_level), - slot, - ) - if z3 is None: - return _feedback_fallback("cups_raster_feedback", seed, slot, _cups_feedback_fields(seed, slot, variant)) - - for loosen in range(_feedback_loosen_limit(expansion_level)): - solver = z3.Solver() - variables = {field.name: z3.Int(field.name) for field in CUPS_RASTER_STRUCTURAL_SPEC.fields} - for field in CUPS_RASTER_STRUCTURAL_SPEC.fields: - _add_field_domain(solver, variables[field.name], field) - solver.add(*_cups_base_constraints(variables, slot)) - solver.add( - *_feedback_neighborhood_constraints( - variables, - seed, - CUPS_RASTER_STRUCTURAL_SPEC, - loosen, - cups=True, - expansion_level=expansion_level, - ) - ) - solver.add(*_cups_feedback_objective(variables, seed, slot, variant)) - if solver.check() == z3.sat: - model = solver.model() - return TemplateInstance( - spec_name="cups_raster_feedback", - objective=f"feedback:{_feedback_variant_name(variant)}:{seed.source}", - case_index=slot, - fields={ - field.name: int(model.evaluate(variables[field.name], model_completion=True).as_long()) - for field in CUPS_RASTER_STRUCTURAL_SPEC.fields - }, - solved_by="z3-feedback-source" if source_biased else "z3-feedback", - ) - return _feedback_fallback("cups_raster_feedback", seed, slot, _cups_feedback_fields(seed, slot, variant)) - - -def _solve_pwg_feedback(seed: FeedbackSeed, slot: int, expansion_level: int = 0) -> TemplateInstance: - seed = _sanitize_feedback_seed(seed, PWG_RASTER_STRUCTURAL_SPEC) - variant, source_biased = choose_source_feedback_variant( - "pwg", - _pwg_feedback_variant_count(expansion_level), - slot, - ) - if z3 is None: - return _feedback_fallback("pwg_raster_feedback", seed, slot, _pwg_feedback_fields(seed, slot, variant)) - - for loosen in range(_feedback_loosen_limit(expansion_level)): - solver = z3.Solver() - variables = {field.name: z3.Int(field.name) for field in PWG_RASTER_STRUCTURAL_SPEC.fields} - for field in PWG_RASTER_STRUCTURAL_SPEC.fields: - _add_field_domain(solver, variables[field.name], field) - solver.add(*_pwg_base_constraints(variables, slot)) - solver.add( - *_feedback_neighborhood_constraints( - variables, - seed, - PWG_RASTER_STRUCTURAL_SPEC, - loosen, - cups=False, - expansion_level=expansion_level, - ) - ) - solver.add(*_pwg_feedback_objective(variables, seed, slot, variant)) - if solver.check() == z3.sat: - model = solver.model() - return TemplateInstance( - spec_name="pwg_raster_feedback", - objective=f"feedback:{_feedback_variant_name(variant)}:{seed.source}", - case_index=slot, - fields={ - field.name: int(model.evaluate(variables[field.name], model_completion=True).as_long()) - for field in PWG_RASTER_STRUCTURAL_SPEC.fields - }, - solved_by="z3-feedback-source" if source_biased else "z3-feedback", - ) - return _feedback_fallback("pwg_raster_feedback", seed, slot, _pwg_feedback_fields(seed, slot, variant)) - - -def _feedback_neighborhood_constraints( - variables: Mapping[str, Any], - seed: FeedbackSeed, - spec: TemplateSpec, - loosen: int, - *, - cups: bool, - expansion_level: int = 0, -) -> list[Any]: - fields = seed.fields - constraints: list[Any] = [] - radius_scale = 1 + max(0, expansion_level) - for field in spec.fields: - if field.name not in fields or field.values is None: - continue - value = fields[field.name] - if loosen == 0 and field.name in {"width", "height", "bits_per_pixel", "color_space", "num_colors"}: - if value in field.values: - constraints.append(variables[field.name] == value) - continue - if loosen <= 1 and field.name in {"width", "height", "bits_per_pixel"}: - allowed = _near_values(field.values, value, _feedback_radius(field.name, value) * radius_scale) - if allowed: - constraints.append(z3.Or(*[variables[field.name] == item for item in allowed])) - elif loosen <= 1 and cups and field.name in {"color_space", "num_colors"} and value in field.values: - constraints.append(variables[field.name] == value) - elif loosen <= 1 and field.name in {"x_res", "y_res"}: - allowed = _near_values(field.values, value, max(1, (value // 2) * radius_scale)) - if allowed: - constraints.append(z3.Or(*[variables[field.name] == item for item in allowed])) - return constraints - - -def _cups_feedback_objective(v: Mapping[str, Any], seed: FeedbackSeed, slot: int, variant: int) -> list[Any]: - raw_bpl = _ceil_div(v["width"] * v["bits_per_pixel"], 8) - aligned_bpl = _align(raw_bpl, 8) - seed_bpl = max(1, seed.fields.get("bytes_per_line", 1)) - delta = [1, 2, 4, 8][slot % 4] - if variant == 0: - return [v["bytes_per_line"] == seed_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 1: - return [raw_bpl > delta, v["bytes_per_line"] == raw_bpl - delta, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 2: - return [v["bytes_per_line"] == raw_bpl + delta, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 3: - return [v["bytes_per_line"] == _seed_delta(seed_bpl, -delta), v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 4: - return [v["bytes_per_line"] == seed_bpl + delta, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 5: - return [v["height"] > 1, v["bytes_per_line"] == aligned_bpl, v["row_count"] == v["height"] - 1, v["payload_rows"] == v["height"]] - if variant == 6: - return [v["bytes_per_line"] == aligned_bpl, v["row_count"] == v["height"] + 1, v["payload_rows"] == v["height"]] - if variant == 7: - return [v["height"] > 1, v["bytes_per_line"] == aligned_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"] - 1] - if variant == 8: - return [v["bytes_per_line"] == aligned_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"] + 1] - if variant == 9: - return [v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 10: - return [v["bytes_per_line"] == aligned_bpl + 16, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 11: - return [v["bytes_per_line"] == aligned_bpl + 32, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 12: - return [v["bytes_per_line"] == aligned_bpl, v["row_count"] == v["height"] + 2, v["payload_rows"] == v["height"]] - if variant == 13: - return [v["bytes_per_line"] == aligned_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"] + 2] - return [v["bytes_per_line"] == aligned_bpl, v["row_count"] == v["height"] + 2, v["payload_rows"] == v["height"] + 2] - - -def _pwg_feedback_objective(v: Mapping[str, Any], seed: FeedbackSeed, slot: int, variant: int) -> list[Any]: - raw_bpl = _ceil_div(v["width"] * v["bits_per_pixel"], 8) - seed_bpl = max(1, seed.fields.get("bytes_per_line", 1)) - delta = [1, 2, 4, 8][slot % 4] - if variant == 0: - return [v["bytes_per_line"] == seed_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 1: - return [raw_bpl > delta, v["bytes_per_line"] == raw_bpl - delta, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 2: - return [v["bytes_per_line"] == raw_bpl + delta, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 3: - return [v["bytes_per_line"] == _seed_delta(seed_bpl, -delta), v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 4: - return [v["bytes_per_line"] == seed_bpl + delta, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 5: - return [v["height"] > 1, v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"] - 1, v["payload_rows"] == v["height"]] - if variant == 6: - return [v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"] + 1, v["payload_rows"] == v["height"]] - if variant == 7: - return [v["height"] > 1, v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"] - 1] - if variant == 8: - return [v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"] + 1] - if variant == 9: - return [v["bytes_per_line"] == raw_bpl + 16, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 10: - return [v["bytes_per_line"] == raw_bpl + 32, v["row_count"] == v["height"], v["payload_rows"] == v["height"]] - if variant == 11: - return [v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"] + 2, v["payload_rows"] == v["height"]] - return [v["bytes_per_line"] == raw_bpl, v["row_count"] == v["height"], v["payload_rows"] == v["height"] + 2] - - -def _cups_feedback_fields(seed: FeedbackSeed, slot: int, variant: int) -> dict[str, int]: - fields = dict(_fallback_cups(slot)) - fields.update({key: value for key, value in seed.fields.items() if key in fields}) - return _apply_feedback_fallback_variant(fields, slot, variant, cups=True) - - -def _pwg_feedback_fields(seed: FeedbackSeed, slot: int, variant: int) -> dict[str, int]: - fields = dict(_fallback_pwg(slot)) - fields.update({key: value for key, value in seed.fields.items() if key in fields}) - return _apply_feedback_fallback_variant(fields, slot, variant, cups=False) - - -def _sanitize_feedback_seed(seed: FeedbackSeed, spec: TemplateSpec) -> FeedbackSeed: - sanitized = dict(seed.fields) - for field in spec.fields: - if field.name not in sanitized: - continue - sanitized[field.name] = _sanitize_feedback_value(field, sanitized[field.name]) - return FeedbackSeed( - kind=seed.kind, - source=seed.source, - target_id=seed.target_id, - case_id=seed.case_id, - document_path=seed.document_path, - crashed=seed.crashed, - timed_out=seed.timed_out, - fields=sanitized, - ) - - -def _sanitize_feedback_value(field: FieldSpec, value: int) -> int: - if field.values is not None: - return min(sorted(set(field.values)), key=lambda item: (abs(item - value), item)) - if field.min_value is not None: - value = max(field.min_value, value) - if field.max_value is not None: - value = min(field.max_value, value) - return value - - -def _apply_feedback_fallback_variant(fields: dict[str, int], slot: int, variant: int, *, cups: bool) -> dict[str, int]: - raw_bpl = max(1, (fields["width"] * fields["bits_per_pixel"] + 7) // 8) - aligned_bpl = ((raw_bpl + 7) // 8) * 8 if cups else raw_bpl - seed_bpl = max(1, fields.get("bytes_per_line", raw_bpl)) - delta = [1, 2, 4, 8][slot % 4] - if variant == 1: - fields["bytes_per_line"] = max(1, raw_bpl - delta) - elif variant == 2: - fields["bytes_per_line"] = raw_bpl + delta - elif variant == 3: - fields["bytes_per_line"] = max(1, seed_bpl - delta) - elif variant == 4: - fields["bytes_per_line"] = seed_bpl + delta - elif variant == 5: - fields["bytes_per_line"] = aligned_bpl - fields["row_count"] = max(1, fields["height"] - 1) - elif variant == 6: - fields["bytes_per_line"] = aligned_bpl - fields["row_count"] = fields["height"] + 1 - elif variant == 7: - fields["bytes_per_line"] = aligned_bpl - fields["payload_rows"] = max(1, fields["height"] - 1) - elif variant == 8: - fields["bytes_per_line"] = aligned_bpl - fields["payload_rows"] = fields["height"] + 1 - elif variant == 9 and not cups: - fields["bytes_per_line"] = raw_bpl + 16 - elif variant == 10: - fields["bytes_per_line"] = aligned_bpl + 16 if cups else raw_bpl + 32 - elif variant == 11: - if cups: - fields["bytes_per_line"] = aligned_bpl + 32 - else: - fields["bytes_per_line"] = raw_bpl - fields["row_count"] = fields["height"] + 2 - elif variant == 12: - fields["bytes_per_line"] = aligned_bpl - if cups: - fields["row_count"] = fields["height"] + 2 - else: - fields["payload_rows"] = fields["height"] + 2 - elif variant == 13: - fields["bytes_per_line"] = aligned_bpl - fields["payload_rows"] = fields["height"] + 2 - elif variant >= 14: - fields["bytes_per_line"] = aligned_bpl - fields["row_count"] = fields["height"] + 2 - fields["payload_rows"] = fields["height"] + 2 - else: - fields["bytes_per_line"] = seed_bpl if variant == 0 else raw_bpl - fields.setdefault("row_count", fields["height"]) - fields.setdefault("payload_rows", fields["height"]) - return fields - - -def _feedback_fallback(spec_name: str, seed: FeedbackSeed, slot: int, fields: dict[str, int]) -> TemplateInstance: - return TemplateInstance( - spec_name=spec_name, - objective=f"feedback:{_feedback_variant_name(slot)}:{seed.source}", - case_index=slot, - fields=fields, - solved_by="fallback-feedback", - ) - - -def _near_values(values: tuple[int, ...], center: int, radius: int) -> list[int]: - allowed = [value for value in sorted(set(values)) if abs(value - center) <= radius] - return allowed or ([center] if center in values else []) - - -def _feedback_radius(name: str, value: int) -> int: - if name == "width": - return max(2, value // 4) - if name == "height": - return 2 - return max(1, value // 2) - - -def _seed_delta(value: int, delta: int) -> int: - return max(1, value + delta) - - -def _feedback_variant_name(variant: int) -> str: - names = ( - "seed_line", - "raw_minus", - "raw_plus", - "seed_minus", - "seed_plus", - "row_short", - "row_long", - "payload_short", - "payload_extra", - "valid_tight", - "wide_pad_16", - "wide_pad_32", - "row_plus_two", - "payload_plus_two", - "row_payload_plus_two", - ) - return names[variant % len(names)] - - -def _feedback_expansion_level() -> int: - value = os.environ.get("SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL", "0") - try: - return max(0, min(3, int(value))) - except ValueError: - return 0 - - -def _feedback_loosen_limit(expansion_level: int) -> int: - return max(3, min(6, 3 + expansion_level)) - - -def _cups_feedback_variant_count(expansion_level: int) -> int: - return 10 if expansion_level <= 0 else min(15, 10 + (2 * expansion_level)) - - -def _pwg_feedback_variant_count(expansion_level: int) -> int: - return 8 if expansion_level <= 0 else min(13, 8 + (2 * expansion_level)) diff --git a/parser-fuzzers/src/parser_fuzzers/generator/template_synth.py b/parser-fuzzers/src/parser_fuzzers/generator/template_synth.py deleted file mode 100644 index d8cd457..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/template_synth.py +++ /dev/null @@ -1,344 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from functools import lru_cache -from typing import Any - -from parser_fuzzers.z3_guard import Z3_LOCK - -try: - import z3 -except ImportError: # pragma: no cover - exercised only in minimal environments - z3 = None - - -PPD_SYNTH_PERIOD = 240 -CUPS_RASTER_SYNTH_PERIOD = 192 -PWG_RASTER_SYNTH_PERIOD = 160 -IMAGE_SYNTH_PERIOD = 72 - -CUPS_WIDTHS = [1, 2, 3, 4, 7, 8, 9, 15, 16, 17, 31, 32, 33, 48, 63, 64, 65, 96, 127, 128, 129, 255] -CUPS_HEIGHTS = [1, 2, 3, 4, 5, 8] -CUPS_RESOLUTIONS = [72, 75, 100, 150, 203, 300, 360, 600, 720, 1200, 2400] -CUPS_COLOR_MODES = [ - (3, 1, 8), - (18, 1, 8), - (1, 3, 24), - (6, 4, 32), - (1, 3, 32), - (18, 1, 16), -] - -PWG_WIDTHS = [1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 65, 127, 128, 129, 255] -PWG_HEIGHTS = [1, 2, 3, 4, 8] -PWG_BPP = [1, 8, 16, 24, 32] -PWG_RESOLUTIONS = [72, 150, 203, 300, 360, 600, 720, 1200, 2400, 32768, 65535, 2147483647] - -IMAGE_FORMATS = ["png_rgb", "png_gray", "ppm", "pgm", "pbm"] -IMAGE_WIDTHS = [1, 2, 3, 4, 7, 8, 9, 15, 16, 17, 31, 32, 63, 64] -IMAGE_HEIGHTS = [1, 2, 3, 4, 5, 8] - -PAGE_SIZE_COUNT = 4 -COLOR_MODEL_COUNT = 4 -QUALITY_COUNT = 4 -MEDIA_COUNT = 4 -DUPLEX_COUNT = 3 -PPD_RESOLUTIONS = [72, 75, 100, 150, 203, 300, 360, 600, 720, 1200, 2400, 65535] - - -@dataclass(frozen=True) -class CUPSRasterSlots: - width: int - height: int - compression: int - num_colors: int - color_space: int - color_order: int - bits_per_pixel: int - pages: int - x_res: int - y_res: int - - -@dataclass(frozen=True) -class PWGRasterSlots: - width: int - height: int - bits_per_pixel: int - x_res: int - y_res: int - pages: int - - -@dataclass(frozen=True) -class ImageSlots: - image_format: str - width: int - height: int - channels: int - - -@dataclass(frozen=True) -class PPDSlots: - page_size_index: int - color_model_index: int - quality_index: int - media_index: int - duplex_index: int - resolution: int - - -def synthesize_cups_raster_slots(case_index: int) -> CUPSRasterSlots: - with Z3_LOCK: - return _synthesize_cups_raster_slot(case_index % CUPS_RASTER_SYNTH_PERIOD) - - -@lru_cache(maxsize=None) -def _synthesize_cups_raster_slot(slot: int) -> CUPSRasterSlots: - fallback = _fallback_cups(slot) - if z3 is None: - return fallback - width = z3.Int("width") - height = z3.Int("height") - compression = z3.Int("compression") - color_space = z3.Int("color_space") - num_colors = z3.Int("num_colors") - bits_per_pixel = z3.Int("bits_per_pixel") - color_order = z3.Int("color_order") - pages = z3.Int("pages") - x_res = z3.Int("x_res") - y_res = z3.Int("y_res") - bytes_per_line = z3.Int("bytes_per_line") - raw_bpl = (width * bits_per_pixel + 7) / 8 - - solver = z3.Solver() - _domain(solver, width, CUPS_WIDTHS) - _domain(solver, height, CUPS_HEIGHTS) - _domain(solver, compression, [0, 0, 0, 1, 10]) - _domain(solver, color_order, [0, 0, 1]) - _domain(solver, pages, [1, 1, 2, 3]) - _domain(solver, x_res, CUPS_RESOLUTIONS) - _domain(solver, y_res, CUPS_RESOLUTIONS) - solver.add( - z3.Or( - *[ - z3.And(color_space == cs, num_colors == colors, bits_per_pixel == bpp) - for cs, colors, bpp in CUPS_COLOR_MODES - ] - ) - ) - solver.add(bytes_per_line >= raw_bpl) - solver.add(bytes_per_line <= raw_bpl + 7) - solver.add(bytes_per_line % 8 == 0) - solver.add(width == CUPS_WIDTHS[(slot * 7 + slot // 5) % len(CUPS_WIDTHS)]) - solver.add(height == CUPS_HEIGHTS[(slot * 5 + slot // 11) % len(CUPS_HEIGHTS)]) - cs, colors, bpp = CUPS_COLOR_MODES[(slot * 3 + slot // 7) % len(CUPS_COLOR_MODES)] - solver.add(color_space == cs, num_colors == colors, bits_per_pixel == bpp) - solver.add(x_res == CUPS_RESOLUTIONS[(slot * 3 + 1) % len(CUPS_RESOLUTIONS)]) - solver.add(y_res == CUPS_RESOLUTIONS[(slot * 5 + 2) % len(CUPS_RESOLUTIONS)]) - solver.add(compression == [0, 0, 0, 1, 10][slot % 5]) - solver.add(pages == [1, 2, 1, 3][slot % 4]) - solver.add(color_order == [0, 0, 1][slot % 3]) - if solver.check() != z3.sat: - return fallback - model = solver.model() - return CUPSRasterSlots( - width=_model_int(model, width), - height=_model_int(model, height), - compression=_model_int(model, compression), - num_colors=_model_int(model, num_colors), - color_space=_model_int(model, color_space), - color_order=_model_int(model, color_order), - bits_per_pixel=_model_int(model, bits_per_pixel), - pages=_model_int(model, pages), - x_res=_model_int(model, x_res), - y_res=_model_int(model, y_res), - ) - - -def synthesize_pwg_raster_slots(case_index: int) -> PWGRasterSlots: - with Z3_LOCK: - return _synthesize_pwg_raster_slot(case_index % PWG_RASTER_SYNTH_PERIOD) - - -@lru_cache(maxsize=None) -def _synthesize_pwg_raster_slot(slot: int) -> PWGRasterSlots: - fallback = _fallback_pwg(slot) - if z3 is None: - return fallback - width = z3.Int("width") - height = z3.Int("height") - bits_per_pixel = z3.Int("bits_per_pixel") - x_res = z3.Int("x_res") - y_res = z3.Int("y_res") - pages = z3.Int("pages") - row_bytes = z3.Int("row_bytes") - - solver = z3.Solver() - _domain(solver, width, PWG_WIDTHS) - _domain(solver, height, PWG_HEIGHTS) - _domain(solver, bits_per_pixel, PWG_BPP) - _domain(solver, x_res, PWG_RESOLUTIONS) - _domain(solver, y_res, PWG_RESOLUTIONS) - _domain(solver, pages, [1, 1, 2, 3]) - solver.add(row_bytes == (width * bits_per_pixel + 7) / 8) - solver.add(row_bytes >= 1) - solver.add(width == PWG_WIDTHS[(slot * 5 + slot // 3) % len(PWG_WIDTHS)]) - solver.add(height == PWG_HEIGHTS[(slot * 7 + slot // 13) % len(PWG_HEIGHTS)]) - solver.add(bits_per_pixel == PWG_BPP[(slot * 3 + slot // 17) % len(PWG_BPP)]) - solver.add(x_res == PWG_RESOLUTIONS[(slot * 2 + 3) % len(PWG_RESOLUTIONS)]) - solver.add(y_res == PWG_RESOLUTIONS[(slot * 5 + 1) % len(PWG_RESOLUTIONS)]) - solver.add(pages == [1, 2, 1, 3][slot % 4]) - if solver.check() != z3.sat: - return fallback - model = solver.model() - return PWGRasterSlots( - width=_model_int(model, width), - height=_model_int(model, height), - bits_per_pixel=_model_int(model, bits_per_pixel), - x_res=_model_int(model, x_res), - y_res=_model_int(model, y_res), - pages=_model_int(model, pages), - ) - - -def synthesize_image_slots(case_index: int) -> ImageSlots: - with Z3_LOCK: - return _synthesize_image_slot(case_index % IMAGE_SYNTH_PERIOD) - - -@lru_cache(maxsize=None) -def _synthesize_image_slot(slot: int) -> ImageSlots: - fallback = _fallback_image(slot) - if z3 is None: - return fallback - format_index = z3.Int("format_index") - width = z3.Int("width") - height = z3.Int("height") - channels = z3.Int("channels") - solver = z3.Solver() - _domain(solver, format_index, list(range(len(IMAGE_FORMATS)))) - _domain(solver, width, IMAGE_WIDTHS) - _domain(solver, height, IMAGE_HEIGHTS) - solver.add(format_index == slot % len(IMAGE_FORMATS)) - solver.add(width == IMAGE_WIDTHS[(slot * 7 + 3) % len(IMAGE_WIDTHS)]) - solver.add(height == IMAGE_HEIGHTS[(slot * 5 + 1) % len(IMAGE_HEIGHTS)]) - solver.add( - z3.Or( - z3.And(format_index == 0, channels == 3), - z3.And(format_index == 1, channels == 1), - z3.And(format_index == 2, channels == 3), - z3.And(format_index == 3, channels == 1), - z3.And(format_index == 4, channels == 1), - ) - ) - if solver.check() != z3.sat: - return fallback - model = solver.model() - fmt_index = _model_int(model, format_index) - return ImageSlots( - image_format=IMAGE_FORMATS[fmt_index], - width=_model_int(model, width), - height=_model_int(model, height), - channels=_model_int(model, channels), - ) - - -def synthesize_ppd_slots(case_index: int) -> PPDSlots: - with Z3_LOCK: - return _synthesize_ppd_slot(case_index % PPD_SYNTH_PERIOD) - - -@lru_cache(maxsize=None) -def _synthesize_ppd_slot(slot: int) -> PPDSlots: - fallback = _fallback_ppd(slot) - if z3 is None: - return fallback - page_size_index = z3.Int("page_size_index") - color_model_index = z3.Int("color_model_index") - quality_index = z3.Int("quality_index") - media_index = z3.Int("media_index") - duplex_index = z3.Int("duplex_index") - resolution = z3.Int("resolution") - solver = z3.Solver() - _domain(solver, page_size_index, list(range(PAGE_SIZE_COUNT))) - _domain(solver, color_model_index, list(range(COLOR_MODEL_COUNT))) - _domain(solver, quality_index, list(range(QUALITY_COUNT))) - _domain(solver, media_index, list(range(MEDIA_COUNT))) - _domain(solver, duplex_index, list(range(DUPLEX_COUNT))) - _domain(solver, resolution, PPD_RESOLUTIONS) - solver.add(page_size_index == (slot * 5 + slot // 13) % PAGE_SIZE_COUNT) - solver.add(color_model_index == (slot * 3 + slot // 7) % COLOR_MODEL_COUNT) - solver.add(quality_index == (slot * 5 + 1) % QUALITY_COUNT) - solver.add(media_index == (slot * 7 + 2) % MEDIA_COUNT) - solver.add(duplex_index == (slot * 11 + slot // 17) % DUPLEX_COUNT) - solver.add(resolution == PPD_RESOLUTIONS[(slot * 7 + 3) % len(PPD_RESOLUTIONS)]) - if solver.check() != z3.sat: - return fallback - model = solver.model() - return PPDSlots( - page_size_index=_model_int(model, page_size_index), - color_model_index=_model_int(model, color_model_index), - quality_index=_model_int(model, quality_index), - media_index=_model_int(model, media_index), - duplex_index=_model_int(model, duplex_index), - resolution=_model_int(model, resolution), - ) - - -def _domain(solver: Any, variable: Any, values: list[int]) -> None: - solver.add(z3.Or(*[variable == value for value in sorted(set(values))])) - - -def _model_int(model: Any, variable: Any) -> int: - return int(model.evaluate(variable, model_completion=True).as_long()) - - -def _fallback_cups(slot: int) -> CUPSRasterSlots: - color_space, num_colors, bits_per_pixel = CUPS_COLOR_MODES[(slot * 3 + slot // 7) % len(CUPS_COLOR_MODES)] - return CUPSRasterSlots( - width=CUPS_WIDTHS[(slot * 7 + slot // 5) % len(CUPS_WIDTHS)], - height=CUPS_HEIGHTS[(slot * 5 + slot // 11) % len(CUPS_HEIGHTS)], - compression=[0, 0, 0, 1, 10][slot % 5], - num_colors=num_colors, - color_space=color_space, - color_order=[0, 0, 1][slot % 3], - bits_per_pixel=bits_per_pixel, - pages=[1, 2, 1, 3][slot % 4], - x_res=CUPS_RESOLUTIONS[(slot * 3 + 1) % len(CUPS_RESOLUTIONS)], - y_res=CUPS_RESOLUTIONS[(slot * 5 + 2) % len(CUPS_RESOLUTIONS)], - ) - - -def _fallback_pwg(slot: int) -> PWGRasterSlots: - return PWGRasterSlots( - width=PWG_WIDTHS[(slot * 5 + slot // 3) % len(PWG_WIDTHS)], - height=PWG_HEIGHTS[(slot * 7 + slot // 13) % len(PWG_HEIGHTS)], - bits_per_pixel=PWG_BPP[(slot * 3 + slot // 17) % len(PWG_BPP)], - x_res=PWG_RESOLUTIONS[(slot * 2 + 3) % len(PWG_RESOLUTIONS)], - y_res=PWG_RESOLUTIONS[(slot * 5 + 1) % len(PWG_RESOLUTIONS)], - pages=[1, 2, 1, 3][slot % 4], - ) - - -def _fallback_image(slot: int) -> ImageSlots: - fmt_index = slot % len(IMAGE_FORMATS) - image_format = IMAGE_FORMATS[fmt_index] - channels = 3 if image_format in {"png_rgb", "ppm"} else 1 - return ImageSlots( - image_format=image_format, - width=IMAGE_WIDTHS[(slot * 7 + 3) % len(IMAGE_WIDTHS)], - height=IMAGE_HEIGHTS[(slot * 5 + 1) % len(IMAGE_HEIGHTS)], - channels=channels, - ) - - -def _fallback_ppd(slot: int) -> PPDSlots: - return PPDSlots( - page_size_index=(slot * 5 + slot // 13) % PAGE_SIZE_COUNT, - color_model_index=(slot * 3 + slot // 7) % COLOR_MODEL_COUNT, - quality_index=(slot * 5 + 1) % QUALITY_COUNT, - media_index=(slot * 7 + 2) % MEDIA_COUNT, - duplex_index=(slot * 11 + slot // 17) % DUPLEX_COUNT, - resolution=PPD_RESOLUTIONS[(slot * 7 + 3) % len(PPD_RESOLUTIONS)], - ) diff --git a/parser-fuzzers/src/parser_fuzzers/generator/z3_guard.py b/parser-fuzzers/src/parser_fuzzers/generator/z3_guard.py deleted file mode 100644 index 04d772d..0000000 --- a/parser-fuzzers/src/parser_fuzzers/generator/z3_guard.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import annotations - -import threading - - -Z3_LOCK = threading.RLock() diff --git a/parser-fuzzers/src/parser_fuzzers/hashing.py b/parser-fuzzers/src/parser_fuzzers/hashing.py deleted file mode 100644 index a7f7f85..0000000 --- a/parser-fuzzers/src/parser_fuzzers/hashing.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.core.hashing`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.core.hashing") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/image_templates.py b/parser-fuzzers/src/parser_fuzzers/image_templates.py deleted file mode 100644 index 4de7465..0000000 --- a/parser-fuzzers/src/parser_fuzzers/image_templates.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.image_templates`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.image_templates") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/loop_metrics.py b/parser-fuzzers/src/parser_fuzzers/loop_metrics.py deleted file mode 100644 index 5ac9759..0000000 --- a/parser-fuzzers/src/parser_fuzzers/loop_metrics.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.metrics.loop_metrics`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.metrics.loop_metrics") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/metrics/__init__.py b/parser-fuzzers/src/parser_fuzzers/metrics/__init__.py deleted file mode 100644 index b05b17c..0000000 --- a/parser-fuzzers/src/parser_fuzzers/metrics/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Layer package for parser-fuzzers.""" diff --git a/parser-fuzzers/src/parser_fuzzers/metrics/baseline_compare.py b/parser-fuzzers/src/parser_fuzzers/metrics/baseline_compare.py deleted file mode 100644 index b8b22ee..0000000 --- a/parser-fuzzers/src/parser_fuzzers/metrics/baseline_compare.py +++ /dev/null @@ -1,576 +0,0 @@ -from __future__ import annotations - -import contextlib -import json -import os -import shutil -import subprocess -import time -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any, Iterator - -from parser_fuzzers.crash_dedup import dedup_run -from parser_fuzzers.multitarget_runner import load_profiles, run_multitarget_monitor -from parser_fuzzers.run_metrics import summarize_run_metrics, write_run_metrics - - -@dataclass(frozen=True) -class OssFuzzStatus: - oss_fuzz_dir: str - project_dir: str - project_yaml: str - dockerfile: str - run_tests: str - docker_available: bool - project_exists: bool - has_project_build_sh: bool - build_sh_source: str - official_available_locally: bool - reason: str - helper_commands: list[str] - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - -@dataclass(frozen=True) -class LocalComparisonResult: - comparison_id: str - work_root: str - baseline_run_dir: str - optimized_run_dir: str - baseline_metrics: str - optimized_metrics: str - comparison_json: str - comparison_md: str - baseline_coverage_json: str - optimized_coverage_json: str - oss_fuzz_status: OssFuzzStatus - - def to_dict(self) -> dict[str, Any]: - payload = asdict(self) - payload["oss_fuzz_status"] = self.oss_fuzz_status.to_dict() - return payload - - -def inspect_oss_fuzz_cups_filters(oss_fuzz_dir: str | Path) -> OssFuzzStatus: - root = Path(oss_fuzz_dir) - project_dir = root / "projects" / "cups-filters" - project_yaml = project_dir / "project.yaml" - dockerfile = project_dir / "Dockerfile" - run_tests = project_dir / "run_tests.sh" - has_project_build_sh = (project_dir / "build.sh").exists() - build_sh_source = _extract_build_sh_source(dockerfile) - docker_available = shutil.which("docker") is not None - project_exists = project_yaml.exists() and dockerfile.exists() - official_available = project_exists and docker_available - if not project_exists: - reason = "missing OSS-Fuzz cups-filters project metadata" - elif not docker_available: - reason = "docker is not available; official OSS-Fuzz helper commands cannot run locally" - elif not has_project_build_sh and build_sh_source: - reason = "official build script is supplied through the Dockerfile from OpenPrinting/fuzzing" - else: - reason = "official OSS-Fuzz helper commands should be usable" - return OssFuzzStatus( - oss_fuzz_dir=str(root), - project_dir=str(project_dir), - project_yaml=str(project_yaml) if project_yaml.exists() else "", - dockerfile=str(dockerfile) if dockerfile.exists() else "", - run_tests=str(run_tests) if run_tests.exists() else "", - docker_available=docker_available, - project_exists=project_exists, - has_project_build_sh=has_project_build_sh, - build_sh_source=build_sh_source, - official_available_locally=official_available, - reason=reason, - helper_commands=[ - "python3 infra/helper.py build_image cups-filters", - "python3 infra/helper.py build_fuzzers --sanitizer address --engine libfuzzer cups-filters", - "python3 infra/helper.py run_fuzzer cups-filters ", - "python3 infra/helper.py coverage cups-filters", - ], - ) - - -def run_local_baseline_comparison( - *, - config_path: str | Path, - work_root: str | Path, - oss_fuzz_dir: str | Path, - duration_sec: int, - workers: int, - timeout_sec: int, - max_run_gb: float = 0.0, - enable_llvm_profiles: bool = False, - export_llvm_coverage: bool = False, - optimized_policy: str = "avoidance", -) -> LocalComparisonResult: - comparison_id = time.strftime("%Y%m%d-%H%M%S") - root = Path(work_root) / comparison_id - root.mkdir(parents=True, exist_ok=True) - config = Path(config_path) - oss_status = inspect_oss_fuzz_cups_filters(oss_fuzz_dir) - objects = _coverage_objects_from_config(config) - - baseline_env = { - "SMT_FUZZER_ENABLE_LLVM_PROFILES": "1" if enable_llvm_profiles else "", - "SMT_FUZZER_HAZARD_SKIP_AFTER": "0", - "SMT_FUZZER_SEMANTIC_SKIP_AFTER": "0", - "SMT_FUZZER_AVOIDANCE_PROBE_INTERVAL": "0", - "SMT_FUZZER_AVOIDANCE_SKIP_PROBE_RATE": "0", - "SMT_FUZZER_SKIP_SHORT_IMAGE_ABORTS": "0", - "SMT_FUZZER_DISABLE_SCHEDULER_CRASH_PENALTY": "0", - } - normalized_policy = _normalize_optimized_policy(optimized_policy) - optimized_env = _optimized_policy_env(normalized_policy, enable_llvm_profiles) - - baseline_root = root / "baseline" - optimized_root = root / "optimized" - with _patched_environ(baseline_env): - baseline_summary = run_multitarget_monitor( - config_path=config, - work_root=baseline_root, - workers=workers, - cases_per_target=None, - duration_sec=duration_sec, - timeout_sec=timeout_sec, - max_run_gb=max_run_gb or None, - run_command=_run_command( - config, - baseline_root, - duration_sec, - workers, - timeout_sec, - max_run_gb, - variant="baseline", - ), - capture_stdout=False, - discovery_mode="coverage", - scheduler="round-robin", - runtime_skip=False, - crash_skip_after=999999, - prune_uninteresting=False, - summary_mode="concise", - ) - - with _patched_environ(optimized_env): - optimized_summary = run_multitarget_monitor( - config_path=config, - work_root=optimized_root, - workers=workers, - cases_per_target=None, - duration_sec=duration_sec, - timeout_sec=timeout_sec, - max_run_gb=max_run_gb or None, - run_command=_run_command( - config, - optimized_root, - duration_sec, - workers, - timeout_sec, - max_run_gb, - variant="optimized", - optimized_policy=normalized_policy, - ), - capture_stdout=False, - discovery_mode="coverage", - scheduler="novelty", - runtime_skip=normalized_policy == "avoidance", - crash_skip_after=1 if normalized_policy == "avoidance" else 999999, - generalized_skip=normalized_policy == "avoidance", - family_skip_after=32, - skip_probe_rate=0.01 if normalized_policy == "avoidance" else 0.0, - prune_uninteresting=False, - summary_mode="concise", - ) - - baseline_run_dir = Path(baseline_summary.work_dir) - optimized_run_dir = Path(optimized_summary.work_dir) - dedup_run(baseline_run_dir, output_json=baseline_run_dir / "dedup.json", output_md=baseline_run_dir / "dedup.md") - dedup_run(optimized_run_dir, output_json=optimized_run_dir / "dedup.json", output_md=optimized_run_dir / "dedup.md") - - baseline_coverage_json = "" - optimized_coverage_json = "" - if export_llvm_coverage: - baseline_coverage_json = _export_llvm_coverage(baseline_run_dir, objects, root / "coverage" / "baseline") - optimized_coverage_json = _export_llvm_coverage(optimized_run_dir, objects, root / "coverage" / "optimized") - - metrics_dir = root / "metrics" - metrics_dir.mkdir(parents=True, exist_ok=True) - baseline_metrics_path = metrics_dir / "baseline.json" - optimized_metrics_path = metrics_dir / "optimized.json" - baseline_metrics = summarize_run_metrics( - baseline_run_dir, - llvm_coverage_json=baseline_coverage_json or None, - ) - optimized_metrics = summarize_run_metrics( - optimized_run_dir, - llvm_coverage_json=optimized_coverage_json or None, - ) - write_run_metrics(baseline_metrics, baseline_metrics_path) - write_run_metrics(optimized_metrics, optimized_metrics_path) - - comparison = build_comparison_payload( - comparison_id=comparison_id, - config_path=config, - baseline=baseline_metrics, - optimized=optimized_metrics, - oss_fuzz_status=oss_status, - optimized_policy=normalized_policy, - ) - comparison_json = root / "comparison.json" - comparison_md = root / "comparison.md" - comparison_json.write_text(json.dumps(comparison, indent=2, sort_keys=True) + "\n", encoding="utf-8") - comparison_md.write_text(render_comparison_markdown(comparison), encoding="utf-8") - - return LocalComparisonResult( - comparison_id=comparison_id, - work_root=str(root), - baseline_run_dir=str(baseline_run_dir), - optimized_run_dir=str(optimized_run_dir), - baseline_metrics=str(baseline_metrics_path), - optimized_metrics=str(optimized_metrics_path), - comparison_json=str(comparison_json), - comparison_md=str(comparison_md), - baseline_coverage_json=baseline_coverage_json, - optimized_coverage_json=optimized_coverage_json, - oss_fuzz_status=oss_status, - ) - - -def build_comparison_payload( - *, - comparison_id: str, - config_path: str | Path, - baseline: dict[str, Any], - optimized: dict[str, Any], - oss_fuzz_status: OssFuzzStatus, - optimized_policy: str = "avoidance", -) -> dict[str, Any]: - normalized_policy = _normalize_optimized_policy(optimized_policy) - if normalized_policy == "avoidance": - optimized_label = "smt-semantic-feedback-with-crash-avoidance" - policy_note = ( - "The optimized run enables novelty scheduling, runtime crash-shape suppression, " - "semantic suppression, and deterministic skip probes." - ) - else: - optimized_label = "smt-semantic-feedback-without-crash-avoidance" - policy_note = ( - "The optimized run keeps novelty scheduling but disables runtime crash-shape " - "suppression, semantic suppression, generalized skip, deterministic skip probes, " - "and scheduler crash/repeat-crash penalties." - ) - return { - "comparison_id": comparison_id, - "config_path": str(config_path), - "baseline_label": "local-oss-fuzz-style-baseline", - "optimized_label": optimized_label, - "optimized_policy": normalized_policy, - "oss_fuzz_status": oss_fuzz_status.to_dict(), - "metrics": { - "baseline": _selected_metrics(baseline), - "optimized": _selected_metrics(optimized), - "delta": _metric_delta(_selected_metrics(baseline), _selected_metrics(optimized)), - }, - "interpretation": [ - "No private reproducer is added to either run.", - "Both local variants use the same target config and generated input families.", - policy_note, - "When Docker is unavailable this is not an official OSS-Fuzz execution; it is a local, fair baseline using the same metrics contract.", - ], - } - - -def render_comparison_markdown(payload: dict[str, Any]) -> str: - baseline = payload["metrics"]["baseline"] - optimized = payload["metrics"]["optimized"] - delta = payload["metrics"]["delta"] - oss = payload["oss_fuzz_status"] - rows = [ - ("cases", "cases"), - ("retained_cases", "retained"), - ("coverage_features", "features"), - ("features_per_min", "features/min"), - ("retained_density", "retained density"), - ("crashes", "crashes"), - ("unique_crashes", "unique crashes"), - ("repeat_crash_records", "repeat crash records"), - ("crash_density", "crash density"), - ("skipped", "skipped"), - ("llvm_functions_percent", "LLVM funcs %"), - ("llvm_lines_percent", "LLVM lines %"), - ("llvm_branches_percent", "LLVM branches %"), - ] - lines = [ - f"# Baseline Comparison {payload['comparison_id']}", - "", - "## OSS-Fuzz Availability", - "", - f"- Project exists: `{oss['project_exists']}`", - f"- Docker available: `{oss['docker_available']}`", - f"- Official helper usable locally: `{oss['official_available_locally']}`", - f"- Reason: `{oss['reason']}`", - "", - "Official commands when Docker is available:", - "", - ] - lines.extend(f"- `{command}`" for command in oss["helper_commands"]) - lines.extend( - [ - "", - "## Local Fair Comparison", - "", - "| Metric | Baseline | Optimized | Delta |", - "| --- | ---: | ---: | ---: |", - ] - ) - for key, label in rows: - lines.append( - f"| {label} | {_fmt_metric(baseline.get(key))} | " - f"{_fmt_metric(optimized.get(key))} | {_fmt_metric(delta.get(key))} |" - ) - lines.extend( - [ - "", - "## Notes", - "", - ] - ) - lines.extend(f"- {item}" for item in payload["interpretation"]) - lines.append("") - return "\n".join(lines) - - -def _selected_metrics(payload: dict[str, Any]) -> dict[str, Any]: - run = payload.get("run", {}) - derived = payload.get("derived", {}) - totals = payload.get("llvm_cov", {}).get("totals", {}) - return { - "run_dir": payload.get("run_dir", ""), - "elapsed_sec": run.get("elapsed_sec", 0), - "cases": run.get("cases", 0), - "retained_cases": run.get("retained_cases", 0), - "coverage_features": run.get("coverage_features", 0), - "crashes": run.get("crashes", 0), - "unique_crashes": run.get("unique_crashes", 0), - "timeouts": run.get("timeouts", 0), - "skipped": run.get("skipped", 0), - "features_per_min": derived.get("features_per_min", 0), - "retained_density": derived.get("retained_density", 0), - "crash_density": derived.get("crash_density", 0), - "repeat_crash_records": max(0, int(run.get("crashes", 0)) - int(run.get("unique_crashes", 0))), - "llvm_functions_count": _coverage_count(totals, "functions"), - "llvm_functions_percent": _coverage_percent(totals, "functions"), - "llvm_lines_count": _coverage_count(totals, "lines"), - "llvm_lines_percent": _coverage_percent(totals, "lines"), - "llvm_branches_count": _coverage_count(totals, "branches"), - "llvm_branches_percent": _coverage_percent(totals, "branches"), - } - - -def _metric_delta(baseline: dict[str, Any], optimized: dict[str, Any]) -> dict[str, Any]: - delta: dict[str, Any] = {} - for key, base_value in baseline.items(): - opt_value = optimized.get(key) - if isinstance(base_value, (int, float)) and isinstance(opt_value, (int, float)): - delta[key] = round(float(opt_value) - float(base_value), 6) - return delta - - -def _coverage_count(totals: dict[str, Any], key: str) -> int: - value = totals.get(key, {}) - if not isinstance(value, dict): - return 0 - return int(value.get("covered", 0) or value.get("count", 0) or 0) - - -def _coverage_percent(totals: dict[str, Any], key: str) -> float: - value = totals.get(key, {}) - if not isinstance(value, dict): - return 0.0 - return round(float(value.get("percent", 0.0) or 0.0), 3) - - -def _fmt_metric(value: Any) -> str: - if isinstance(value, float): - return f"{value:.3f}" - if value is None: - return "" - return str(value) - - -def _coverage_objects_from_config(config_path: Path) -> list[str]: - objects: list[str] = [] - for profile in load_profiles(config_path): - path = Path(profile.filter_binary) - if not path.is_absolute(): - path = Path.cwd() / path - if path.exists(): - resolved = str(path) - if resolved not in objects: - objects.append(resolved) - return objects - - -def _export_llvm_coverage(run_dir: Path, objects: list[str], out_dir: Path) -> str: - profiles = sorted(str(path) for path in run_dir.rglob("*.profraw") if path.is_file() and path.stat().st_size > 0) - if not profiles or not objects: - return "" - profdata = _which_llvm_tool("llvm-profdata") - cov = _which_llvm_tool("llvm-cov") - if not profdata or not cov: - return "" - out_dir.mkdir(parents=True, exist_ok=True) - profdata_path = out_dir / "coverage.profdata" - subprocess.run([profdata, "merge", "-sparse", *profiles, "-o", str(profdata_path)], check=True) - first, *rest = objects - object_args = [f"-object={path}" for path in rest] - coverage_json = out_dir / "coverage.json" - coverage_txt = out_dir / "coverage.txt" - exported = subprocess.run( - [cov, "export", first, f"-instr-profile={profdata_path}", *object_args], - check=True, - text=True, - stdout=subprocess.PIPE, - ) - coverage_json.write_text(exported.stdout, encoding="utf-8") - reported = subprocess.run( - [cov, "report", first, f"-instr-profile={profdata_path}", *object_args], - check=True, - text=True, - stdout=subprocess.PIPE, - ) - coverage_txt.write_text(reported.stdout, encoding="utf-8") - return str(coverage_json) - - -def _which_llvm_tool(base: str) -> str: - for candidate in (f"{base}-18", f"{base}-17", f"{base}-16", base): - path = shutil.which(candidate) - if path: - return path - return "" - - -def _extract_build_sh_source(dockerfile: Path) -> str: - if not dockerfile.exists(): - return "" - for line in dockerfile.read_text(encoding="utf-8", errors="replace").splitlines(): - stripped = line.strip() - if "oss_fuzz_build.sh" in stripped and "build.sh" in stripped: - return stripped - return "" - - -def _run_command( - config: Path, - work_root: Path, - duration_sec: int, - workers: int, - timeout_sec: int, - max_run_gb: float, - *, - variant: str, - optimized_policy: str = "avoidance", -) -> str: - args = [ - "PYTHONPATH=src", - "python3", - "-m", - "parser_fuzzers.cli", - "multitarget-monitor", - "--config", - str(config), - "--work-root", - str(work_root), - "--duration-sec", - str(duration_sec), - "--workers", - str(workers), - "--timeout-sec", - str(timeout_sec), - ] - if max_run_gb: - args.extend(["--max-run-gb", f"{max_run_gb:g}"]) - args.extend(["--discovery-mode", "coverage"]) - if variant == "optimized": - args.extend( - [ - "--scheduler", - "novelty", - ] - ) - if optimized_policy == "avoidance": - args.extend( - [ - "--runtime-skip", - "--crash-skip-after", - "1", - "--generalized-skip", - "--skip-probe-rate", - "0.01", - ] - ) - else: - args.extend(["--scheduler", "round-robin"]) - return " ".join(args) - - -def _normalize_optimized_policy(value: str) -> str: - normalized = value.strip().lower().replace("_", "-") - if normalized in {"avoidance", "with-avoidance", "crash-avoidance"}: - return "avoidance" - if normalized in {"no-crash-avoidance", "without-avoidance", "no-avoidance", "novelty-only"}: - return "no-crash-avoidance" - raise ValueError(f"unknown optimized policy: {value}") - - -def _optimized_policy_env(policy: str, enable_llvm_profiles: bool) -> dict[str, str]: - env = { - "SMT_FUZZER_ENABLE_LLVM_PROFILES": "1" if enable_llvm_profiles else "", - } - if policy == "avoidance": - env.update( - { - "SMT_FUZZER_HAZARD_SKIP_AFTER": "1", - "SMT_FUZZER_SEMANTIC_SKIP_AFTER": "1", - "SMT_FUZZER_AVOIDANCE_PROBE_INTERVAL": "32", - "SMT_FUZZER_AVOIDANCE_SKIP_PROBE_RATE": "0.06", - "SMT_FUZZER_SKIP_SHORT_IMAGE_ABORTS": "1", - "SMT_FUZZER_DISABLE_SCHEDULER_CRASH_PENALTY": "0", - } - ) - else: - env.update( - { - "SMT_FUZZER_HAZARD_SKIP_AFTER": "0", - "SMT_FUZZER_SEMANTIC_SKIP_AFTER": "0", - "SMT_FUZZER_AVOIDANCE_PROBE_INTERVAL": "0", - "SMT_FUZZER_AVOIDANCE_SKIP_PROBE_RATE": "0", - "SMT_FUZZER_SKIP_SHORT_IMAGE_ABORTS": "0", - "SMT_FUZZER_DISABLE_SCHEDULER_CRASH_PENALTY": "1", - } - ) - return env - - -@contextlib.contextmanager -def _patched_environ(updates: dict[str, str]) -> Iterator[None]: - old_values: dict[str, str | None] = {} - for key, value in updates.items(): - old_values[key] = os.environ.get(key) - if value == "": - os.environ.pop(key, None) - else: - os.environ[key] = value - try: - yield - finally: - for key, value in old_values.items(): - if value is None: - os.environ.pop(key, None) - else: - os.environ[key] = value diff --git a/parser-fuzzers/src/parser_fuzzers/metrics/loop_metrics.py b/parser-fuzzers/src/parser_fuzzers/metrics/loop_metrics.py deleted file mode 100644 index 1022c67..0000000 --- a/parser-fuzzers/src/parser_fuzzers/metrics/loop_metrics.py +++ /dev/null @@ -1,124 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - - -def summarize_loop_metrics(campaign_dir: str | Path) -> dict[str, Any]: - root = Path(campaign_dir) - manifest = _read_json(root / "loop_manifest.json") - template_metrics = _read_json(Path(manifest.get("template_run", "")) / "standard_metrics.json") - feedback_template_metrics = _read_json(Path(manifest.get("feedback_run", "")) / "standard_metrics.json") - afl_metrics = _read_json(root / "afl-standard-metrics.json") - feedback_profile = _read_json(root / "feedback-profile-build.json") - seed_export = _read_json(root / "seed-export.json") - afl_import = _summarize_afl_import(_read_json(root / "afl-import.json")) - - return { - "schema_version": "template-afl-loop-metrics-v1", - "campaign_dir": str(root), - "manifest": manifest, - "seed_export": _summarize_seed_export(seed_export), - "template_metrics": template_metrics, - "afl_metrics": afl_metrics, - "afl_import": afl_import, - "feedback_profile": feedback_profile, - "feedback_template_metrics": feedback_template_metrics, - "summary": _loop_summary(template_metrics, afl_metrics, feedback_template_metrics), - } - - -def write_loop_metrics(payload: dict[str, Any], output_path: str | Path) -> None: - path = Path(output_path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def write_standard_loop_metrics( - campaign_dir: str | Path, - *, - output_path: str | Path | None = None, -) -> dict[str, Any]: - root = Path(campaign_dir) - payload = summarize_loop_metrics(root) - write_loop_metrics(payload, output_path or root / "loop_standard_metrics.json") - return payload - - -def _read_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - data = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return {} - return data if isinstance(data, dict) else {} - - -def _standard(metrics: dict[str, Any]) -> dict[str, Any]: - standard = metrics.get("standard") - return standard if isinstance(standard, dict) else {} - - -def _loop_summary( - template_metrics: dict[str, Any], - afl_metrics: dict[str, Any], - feedback_template_metrics: dict[str, Any], -) -> dict[str, Any]: - template = _standard(template_metrics) - afl = _standard(afl_metrics) - feedback = _standard(feedback_template_metrics) - return { - "template_execs_done": _safe_int(template.get("execs_done")), - "template_features": _safe_int(template.get("coverage_features")), - "template_corpus_count": _safe_int(template.get("corpus_count")), - "afl_execs_done": _safe_int(afl.get("execs_done")), - "afl_edges_found": _safe_int(afl.get("coverage_features")), - "afl_corpus_count": _safe_int(afl.get("corpus_count")), - "afl_crashes": _safe_int(afl.get("crashes")), - "afl_hangs": _safe_int(afl.get("timeouts")), - "feedback_template_execs_done": _safe_int(feedback.get("execs_done")), - "feedback_template_features": _safe_int(feedback.get("coverage_features")), - "feedback_template_corpus_count": _safe_int(feedback.get("corpus_count")), - "feedback_feature_delta_vs_template": _safe_int(feedback.get("coverage_features")) - - _safe_int(template.get("coverage_features")), - "feedback_corpus_delta_vs_template": _safe_int(feedback.get("corpus_count")) - - _safe_int(template.get("corpus_count")), - } - - -def _summarize_seed_export(payload: dict[str, Any]) -> dict[str, Any]: - return { - "exported": _safe_int(payload.get("exported")), - "targets": payload.get("targets", []), - "extensions": payload.get("extensions", []), - "exported_by_target": payload.get("exported_by_target", {}), - "output_dir": payload.get("output_dir", ""), - } - - -def _summarize_afl_import(payload: dict[str, Any]) -> dict[str, Any]: - imported = payload.get("imported", []) - if not isinstance(imported, list): - imported = [] - source_counts: dict[str, int] = {} - for item in imported: - if not isinstance(item, dict): - continue - source = str(item.get("source", "unknown")) - source_counts[source] = source_counts.get(source, 0) + 1 - return { - "imported": len(imported), - "crashes_imported": _safe_int(payload.get("crashes_imported")), - "duplicates_skipped": _safe_int(payload.get("duplicates_skipped")), - "source_counts": dict(sorted(source_counts.items())), - "afl_instance_dir": payload.get("afl_instance_dir", ""), - } - - -def _safe_int(value: Any) -> int: - try: - return int(float(value)) - except (TypeError, ValueError): - return 0 diff --git a/parser-fuzzers/src/parser_fuzzers/metrics/run_metrics.py b/parser-fuzzers/src/parser_fuzzers/metrics/run_metrics.py deleted file mode 100644 index 3060401..0000000 --- a/parser-fuzzers/src/parser_fuzzers/metrics/run_metrics.py +++ /dev/null @@ -1,293 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - - -def summarize_run_metrics( - run_dir: str | Path, - *, - afl_output_dir: str | Path | None = None, - llvm_coverage_json: str | Path | None = None, -) -> dict[str, Any]: - root = Path(run_dir) - afl_stats = _afl_stats(Path(afl_output_dir)) if afl_output_dir else {} - summary = _read_json(root / "summary.concise.json") or _read_json(root / "summary.json") - if not summary and afl_stats.get("status") == "ok": - summary = _summary_from_afl_stats(afl_stats) - dedup = _read_json(root / "dedup.json") or _read_json(root / "crash_dedup.json") - timeline = _timeline_counts(root / "timeline.jsonl") - elapsed_sec = _safe_float(summary.get("elapsed_sec")) - cases = _safe_int(summary.get("cases")) - retained = _safe_int(summary.get("retained_cases")) - crashes = _safe_int(summary.get("crashes")) - features = _safe_int(summary.get("coverage_features")) - unique_summary = _safe_int(summary.get("unique_crashes")) - unique_dedup = _safe_int(dedup.get("unique_crashes")) - unique_effective = unique_dedup or unique_summary - run_dir_bytes = _safe_int(summary.get("run_dir_bytes")) - - payload: dict[str, Any] = { - "schema_version": "standard-run-metrics-v1", - "run_dir": str(root), - "run": { - "run_id": summary.get("run_id", ""), - "elapsed_sec": elapsed_sec, - "cases": cases, - "retained_cases": retained, - "coverage_features": features, - "crashes": crashes, - "unique_crashes": unique_summary, - "timeouts": _safe_int(summary.get("timeouts")), - "skipped": _safe_int(summary.get("skipped")), - "pruned_cases": _safe_int(summary.get("pruned_cases")), - "targets": _safe_int(summary.get("targets")), - "run_dir_bytes": run_dir_bytes, - "stop_reason": summary.get("stop_reason", ""), - }, - "derived": { - "crash_density": _ratio(crashes, cases), - "retained_density": _ratio(retained, cases), - "cases_per_sec": _per_sec(cases, elapsed_sec), - "retained_per_min": _per_min(retained, elapsed_sec), - "features_per_min": _per_min(features, elapsed_sec), - "features_per_hour": _per_hour(features, elapsed_sec), - "crashes_per_hour": _per_hour(crashes, elapsed_sec), - "timeline_records": timeline["records"], - "z3_structure_avoid_records": timeline["z3_structure_avoid_records"], - "new_crash_signature_records": timeline["new_crash_signature_records"], - }, - "dedup": { - "crash_records": _safe_int(dedup.get("crash_records")), - "unique_crash_signatures": _safe_int(dedup.get("unique_crashes")), - "clusters": _cluster_summary(dedup.get("clusters", [])), - }, - "standard": { - "elapsed_sec": elapsed_sec, - "elapsed_hours": round(elapsed_sec / 3600.0, 6) if elapsed_sec > 0 else 0.0, - "execs_done": cases, - "execs_per_sec": _per_sec(cases, elapsed_sec), - "corpus_count": retained, - "corpus_density": _ratio(retained, cases), - "coverage_features": features, - "coverage_features_per_min": _per_min(features, elapsed_sec), - "coverage_features_per_hour": _per_hour(features, elapsed_sec), - "crashes": crashes, - "unique_crashes": unique_effective, - "summary_unique_crashes": unique_summary, - "dedup_unique_crashes": unique_dedup, - "repeat_crashes_estimate": max(0, crashes - unique_effective), - "crash_density": _ratio(crashes, cases), - "crashes_per_hour": _per_hour(crashes, elapsed_sec), - "timeouts": _safe_int(summary.get("timeouts")), - "skipped": _safe_int(summary.get("skipped")), - "pruned_cases": _safe_int(summary.get("pruned_cases")), - "targets": _safe_int(summary.get("targets")), - "run_dir_bytes": run_dir_bytes, - "run_dir_gb": round(run_dir_bytes / (1024.0**3), 6) if run_dir_bytes else 0.0, - "stop_reason": str(summary.get("stop_reason", "")), - }, - "target_stats": _target_stats_summary(summary.get("target_stats", {})), - } - if afl_output_dir: - payload["afl"] = afl_stats - if llvm_coverage_json: - payload["llvm_cov"] = _llvm_coverage_totals(Path(llvm_coverage_json)) - return payload - - -def write_run_metrics(payload: dict[str, Any], output_path: str | Path) -> None: - path = Path(output_path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def write_standard_run_metrics( - run_dir: str | Path, - *, - output_path: str | Path | None = None, - afl_output_dir: str | Path | None = None, - llvm_coverage_json: str | Path | None = None, -) -> dict[str, Any]: - root = Path(run_dir) - payload = summarize_run_metrics( - root, - afl_output_dir=afl_output_dir, - llvm_coverage_json=llvm_coverage_json, - ) - write_run_metrics(payload, output_path or root / "standard_metrics.json") - return payload - - -def _read_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - return json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return {} - - -def _timeline_counts(path: Path) -> dict[str, int]: - counts = { - "records": 0, - "z3_structure_avoid_records": 0, - "new_crash_signature_records": 0, - } - if not path.exists(): - return counts - with path.open("r", encoding="utf-8", errors="replace") as handle: - for line in handle: - counts["records"] += 1 - if "z3-structure-avoid" in line: - counts["z3_structure_avoid_records"] += 1 - try: - record = json.loads(line) - except json.JSONDecodeError: - continue - if record.get("new_crash_signature") is True: - counts["new_crash_signature_records"] += 1 - return counts - - -def _cluster_summary(clusters: Any) -> list[dict[str, Any]]: - if not isinstance(clusters, list): - return [] - rows = [] - for item in clusters[:20]: - if not isinstance(item, dict): - continue - rows.append( - { - "target_id": item.get("target_id", ""), - "count": _safe_int(item.get("count")), - "signature": item.get("signature", ""), - "representative_work_dir": item.get("representative_work_dir", ""), - } - ) - return rows - - -def _target_stats_summary(raw_stats: Any) -> dict[str, dict[str, Any]]: - if not isinstance(raw_stats, dict): - return {} - summary: dict[str, dict[str, Any]] = {} - for target_id, stats in raw_stats.items(): - if not isinstance(stats, dict): - continue - completed = _safe_int(stats.get("completed")) - retained = _safe_int(stats.get("retained_cases")) - crashes = _safe_int(stats.get("crashes")) - summary[str(target_id)] = { - "completed": completed, - "retained_cases": retained, - "crashes": crashes, - "timeouts": _safe_int(stats.get("timeouts")), - "runtime_suppressed": _safe_int(stats.get("runtime_suppressed")), - "retained_density": _ratio(retained, completed), - "crash_density": _ratio(crashes, completed), - } - return dict(sorted(summary.items())) - - -def _afl_stats(output_dir: Path) -> dict[str, str]: - stats_path = _find_afl_stats(output_dir) - if not stats_path: - return {"status": "missing-fuzzer-stats", "output_dir": str(output_dir)} - stats: dict[str, str] = {"status": "ok", "fuzzer_stats": str(stats_path)} - for line in stats_path.read_text(encoding="utf-8", errors="replace").splitlines(): - if ":" not in line: - continue - key, value = line.split(":", 1) - stats[key.strip()] = value.strip() - return stats - - -def _summary_from_afl_stats(stats: dict[str, str]) -> dict[str, Any]: - return { - "run_id": Path(stats.get("fuzzer_stats", "")).parent.name, - "elapsed_sec": _safe_float(stats.get("run_time")), - "cases": _safe_int(stats.get("execs_done")), - "retained_cases": _safe_int(stats.get("corpus_count")), - "coverage_features": _safe_int(stats.get("edges_found")), - "crashes": _safe_int(stats.get("saved_crashes")), - "unique_crashes": _safe_int(stats.get("saved_crashes")), - "timeouts": _safe_int(stats.get("saved_hangs")), - "skipped": 0, - "pruned_cases": 0, - "targets": 1, - "run_dir_bytes": _run_dir_size_bytes(Path(stats.get("fuzzer_stats", ".")).parent), - "stop_reason": "afl++", - "target_stats": {}, - } - - -def _find_afl_stats(output_dir: Path) -> Path | None: - candidates = [output_dir / "default" / "fuzzer_stats", output_dir / "fuzzer_stats"] - candidates.extend(sorted(output_dir.glob("*/fuzzer_stats"))) - for path in candidates: - if path.exists(): - return path - return None - - -def _run_dir_size_bytes(root: Path) -> int: - total = 0 - if not root.exists(): - return 0 - for path in root.rglob("*"): - try: - if path.is_file(): - total += path.stat().st_size - except OSError: - continue - return total - - -def _llvm_coverage_totals(path: Path) -> dict[str, Any]: - payload = _read_json(path) - totals = payload.get("data", [{}])[0].get("totals", {}) if payload else {} - return { - "status": "ok" if totals else "missing-totals", - "coverage_json": str(path), - "totals": totals, - } - - -def _ratio(numerator: int, denominator: int) -> float: - if denominator <= 0: - return 0.0 - return round(numerator / denominator, 6) - - -def _per_min(value: int, elapsed_sec: float) -> float: - if elapsed_sec <= 0.0: - return 0.0 - return round(value * 60.0 / elapsed_sec, 3) - - -def _per_hour(value: int, elapsed_sec: float) -> float: - if elapsed_sec <= 0.0: - return 0.0 - return round(value * 3600.0 / elapsed_sec, 3) - - -def _per_sec(value: int, elapsed_sec: float) -> float: - if elapsed_sec <= 0.0: - return 0.0 - return round(value / elapsed_sec, 3) - - -def _safe_int(value: Any) -> int: - try: - return int(value) - except (TypeError, ValueError): - return 0 - - -def _safe_float(value: Any) -> float: - try: - return float(value) - except (TypeError, ValueError): - return 0.0 diff --git a/parser-fuzzers/src/parser_fuzzers/metrics/run_recovery.py b/parser-fuzzers/src/parser_fuzzers/metrics/run_recovery.py deleted file mode 100644 index a5d4039..0000000 --- a/parser-fuzzers/src/parser_fuzzers/metrics/run_recovery.py +++ /dev/null @@ -1,205 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -from parser_fuzzers.multitarget_runner import _run_dir_size_bytes - - -def recover_run_summary(run_dir: str | Path) -> dict[str, Any]: - root = Path(run_dir) - manifest = _read_json(root / "run_manifest.json") - timeline_path = root / "timeline.jsonl" - if not timeline_path.exists(): - raise FileNotFoundError(f"missing timeline.jsonl in {root}") - - target_stats = _initial_target_stats(manifest) - skip_counts: dict[str, int] = {} - oracle_counts: dict[str, int] = {} - cases = 0 - crashes = 0 - reached = 0 - valid_ppds = 0 - timeouts = 0 - skipped = 0 - retained_cases = 0 - coverage_features = 0 - unique_crashes = 0 - repeat_crashes = 0 - - for record in _iter_timeline(timeline_path): - target_id = str(record.get("target_id", "unknown")) - stats = target_stats.setdefault(target_id, _empty_target_stats()) - if record.get("skipped"): - skipped += 1 - reason = str(record.get("skip_reason") or "skipped") - skip_counts[reason] = skip_counts.get(reason, 0) + 1 - stats["skipped"] += 1 - if reason.startswith("runtime-known-crash-"): - stats["runtime_suppressed"] += 1 - continue - - cases += 1 - stats["submitted"] += 1 - stats["completed"] += 1 - oracle = str(record.get("oracle") or "none") - oracle_counts[oracle] = oracle_counts.get(oracle, 0) + 1 - if record.get("crashed"): - crashes += 1 - stats["crashes"] += 1 - if record.get("new_crash_signature") is True: - unique_crashes += 1 - stats["unique_crashes"] += 1 - elif record.get("new_crash_signature") is False: - repeat_crashes += 1 - stats["repeat_crashes"] += 1 - if record.get("timed_out"): - timeouts += 1 - stats["timeouts"] += 1 - if record.get("reached_expected_filter"): - reached += 1 - if record.get("cupstestppd_ok"): - valid_ppds += 1 - if record.get("retained_for_coverage"): - retained_cases += 1 - stats["retained_cases"] += 1 - new_feature_count = _safe_int(record.get("new_feature_count", 0)) - coverage_features += new_feature_count - stats["new_features"] += new_feature_count - - summary = { - "run_id": str(manifest.get("run_id") or root.name), - "work_dir": str(root), - "config_path": str(manifest.get("config_path") or ""), - "duration_budget_sec": _optional_int(manifest.get("duration_sec")), - "elapsed_sec": _estimate_elapsed_sec(root, timeline_path), - "workers": _safe_int(manifest.get("workers", 0)), - "timeout_sec": _safe_int(manifest.get("timeout_sec", 0)), - "max_run_bytes": _safe_int(manifest.get("max_run_bytes", 0)), - "run_dir_bytes": _run_dir_size_bytes(root), - "stop_reason": "recovered-missing-summary", - "targets": len(manifest.get("targets") or target_stats), - "cases": cases, - "crashes": crashes, - "reached": reached, - "valid_ppds": valid_ppds, - "timeouts": timeouts, - "skipped": skipped, - "pruned_cases": 0, - "skip_counts": dict(sorted(skip_counts.items())), - "scheduler": str(manifest.get("scheduler") or ""), - "min_target_share": float(manifest.get("min_target_share") or 0.0), - "max_target_share": float(manifest.get("max_target_share") or 1.0), - "runtime_skip_enabled": bool(manifest.get("runtime_skip")), - "auto_skip_state_enabled": bool(manifest.get("auto_skip_state")), - "auto_skip_search_root": str(manifest.get("auto_skip_search_root") or ""), - "runtime_suppressed_shapes": 0, - "seeded_runtime_suppressed_shapes": _safe_int(manifest.get("seeded_runtime_suppressed_shapes", 0)), - "runtime_suppressed_families": 0, - "seeded_runtime_suppressed_families": _safe_int(manifest.get("seeded_runtime_suppressed_families", 0)), - "generalized_skip_enabled": bool(manifest.get("generalized_skip")), - "family_skip_after": _safe_int(manifest.get("family_skip_after", 0)), - "skip_probe_rate": float(manifest.get("skip_probe_rate") or 0.0), - "skip_only_stop_after": _safe_int(manifest.get("skip_only_stop_after", 0)), - "stagnation_stop_after_sec": _safe_int(manifest.get("stagnation_stop_after_sec", 0)), - "seed_skip_state_path": str(manifest.get("seed_skip_state_path") or ""), - "target_stats": target_stats, - "retained_cases": retained_cases, - "coverage_features": coverage_features, - "unique_crashes": unique_crashes, - "repeat_crashes": repeat_crashes, - "oracle_counts": dict(sorted(oracle_counts.items())), - "recovered": True, - "summary_source": "timeline.jsonl", - } - _merge_discovery_state(root, summary) - - (root / "summary.concise.json").write_text( - json.dumps(summary, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - (root / "summary.json").write_text( - json.dumps( - { - **summary, - "summary_mode": "concise", - "results_omitted": True, - "results_source": "timeline.jsonl", - }, - indent=2, - sort_keys=True, - ) - + "\n", - encoding="utf-8", - ) - return summary - - -def _merge_discovery_state(root: Path, summary: dict[str, Any]) -> None: - state_path = root / "discovery_state.json" - if not state_path.exists(): - return - state = _read_json(state_path) - summary["runtime_suppressed_shapes"] = _safe_int(state.get("runtime_suppressed_shapes", 0)) - summary["runtime_suppressed_families"] = _safe_int(state.get("runtime_suppressed_families", 0)) - - -def _initial_target_stats(manifest: dict[str, Any]) -> dict[str, dict[str, int]]: - stats = {} - for item in manifest.get("targets") or []: - target_id = str(item.get("id") or "") - if target_id: - stats[target_id] = _empty_target_stats() - return stats - - -def _empty_target_stats() -> dict[str, int]: - return { - "submitted": 0, - "completed": 0, - "skipped": 0, - "retained_cases": 0, - "new_features": 0, - "crashes": 0, - "unique_crashes": 0, - "repeat_crashes": 0, - "timeouts": 0, - "runtime_suppressed": 0, - } - - -def _read_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - return json.loads(path.read_text(encoding="utf-8")) - - -def _iter_timeline(path: Path): - with path.open("r", encoding="utf-8") as handle: - for line in handle: - stripped = line.strip() - if stripped: - yield json.loads(stripped) - - -def _estimate_elapsed_sec(root: Path, timeline_path: Path) -> float: - start_path = root / "run_manifest.json" - try: - start = start_path.stat().st_mtime if start_path.exists() else root.stat().st_mtime - return round(max(0.0, timeline_path.stat().st_mtime - start), 3) - except OSError: - return 0.0 - - -def _optional_int(value: Any) -> int | None: - if value is None: - return None - return _safe_int(value) - - -def _safe_int(value: Any) -> int: - try: - return int(value) - except (TypeError, ValueError): - return 0 diff --git a/parser-fuzzers/src/parser_fuzzers/metrics/run_set_metrics.py b/parser-fuzzers/src/parser_fuzzers/metrics/run_set_metrics.py deleted file mode 100644 index 9a5838f..0000000 --- a/parser-fuzzers/src/parser_fuzzers/metrics/run_set_metrics.py +++ /dev/null @@ -1,258 +0,0 @@ -from __future__ import annotations - -import json -import time -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from parser_fuzzers.run_metrics import summarize_run_metrics - - -@dataclass(frozen=True) -class RunRow: - campaign: str - run_dir: str - run_id: str - elapsed_sec: float - cases: int - retained_cases: int - coverage_features: int - crashes: int - unique_crashes: int - repeat_crashes: int - timeouts: int - skipped: int - retained_density: float - crash_density: float - features_per_min: float - stop_reason: str - - def to_dict(self) -> dict[str, Any]: - return { - "campaign": self.campaign, - "run_dir": self.run_dir, - "run_id": self.run_id, - "elapsed_sec": self.elapsed_sec, - "cases": self.cases, - "retained_cases": self.retained_cases, - "coverage_features": self.coverage_features, - "crashes": self.crashes, - "unique_crashes": self.unique_crashes, - "repeat_crashes": self.repeat_crashes, - "timeouts": self.timeouts, - "skipped": self.skipped, - "retained_density": self.retained_density, - "crash_density": self.crash_density, - "features_per_min": self.features_per_min, - "stop_reason": self.stop_reason, - } - - -def summarize_run_set(roots: list[str | Path]) -> dict[str, Any]: - rows = [_row_from_metrics(campaign, run_dir) for campaign, run_dir in _find_run_dirs(roots)] - rows.sort(key=lambda row: (row.campaign, row.run_id, row.run_dir)) - campaigns: dict[str, list[RunRow]] = {} - for row in rows: - campaigns.setdefault(row.campaign, []).append(row) - return { - "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - "roots": [str(Path(root)) for root in roots], - "run_count": len(rows), - "aggregate": _aggregate_rows(rows), - "campaigns": {name: _aggregate_rows(items) for name, items in sorted(campaigns.items())}, - "runs": [row.to_dict() for row in rows], - "notes": [ - "coverage_features is summed from each run's within-run new feature count; it is not globally deduplicated across runs.", - "unique_crashes is summed from run summaries and may count the same crash signature again across separate runs.", - "Use LLVM profraw/llvm-cov or AFL++ edge data for globally comparable source/edge coverage.", - ], - } - - -def write_run_set_metrics(payload: dict[str, Any], output_json: str | Path, output_md: str | Path | None = None) -> None: - json_path = Path(output_json) - json_path.parent.mkdir(parents=True, exist_ok=True) - json_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - if output_md: - Path(output_md).write_text(render_run_set_markdown(payload), encoding="utf-8") - - -def render_run_set_markdown(payload: dict[str, Any]) -> str: - lines = [ - f"# Long-Run Metrics {payload.get('generated_at', '')}", - "", - "## Aggregate", - "", - _aggregate_table(payload.get("aggregate", {})), - "", - "## Campaigns", - "", - "| Campaign | Runs | Hours | Cases | Retained | Features sum | Features/hour | Crashes | Unique crashes sum | Crash density |", - "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", - ] - for name, stats in payload.get("campaigns", {}).items(): - lines.append( - f"| {name} | {stats.get('run_count', 0)} | {_fmt(stats.get('elapsed_hours', 0))} | " - f"{stats.get('cases', 0)} | {stats.get('retained_cases', 0)} | " - f"{stats.get('coverage_features_sum', 0)} | {_fmt(stats.get('features_per_hour', 0))} | " - f"{stats.get('crashes', 0)} | {stats.get('unique_crashes_sum', 0)} | " - f"{_fmt(stats.get('crash_density', 0))} |" - ) - lines.extend( - [ - "", - "## Top Feature Runs", - "", - "| Campaign | Run | Minutes | Cases | Features | Features/min | Crashes | Crash density |", - "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |", - ] - ) - top_runs = sorted(payload.get("runs", []), key=lambda row: row.get("coverage_features", 0), reverse=True)[:20] - for row in top_runs: - lines.append( - f"| {row.get('campaign', '')} | {row.get('run_id', '')} | " - f"{_fmt(float(row.get('elapsed_sec', 0)) / 60.0)} | {row.get('cases', 0)} | " - f"{row.get('coverage_features', 0)} | {_fmt(row.get('features_per_min', 0))} | " - f"{row.get('crashes', 0)} | {_fmt(row.get('crash_density', 0))} |" - ) - lines.extend(["", "## Notes", ""]) - lines.extend(f"- {note}" for note in payload.get("notes", [])) - lines.append("") - return "\n".join(lines) - - -def _find_run_dirs(roots: list[str | Path]) -> list[tuple[str, Path]]: - found: list[tuple[str, Path]] = [] - seen: set[Path] = set() - for raw_root in roots: - root = Path(raw_root) - campaign = root.name - if _is_run_dir(root): - resolved = root.resolve() - if resolved not in seen: - found.append((campaign, root)) - seen.add(resolved) - continue - if not root.exists(): - continue - for run_dir in sorted(path for path in root.iterdir() if path.is_dir()): - if not _is_run_dir(run_dir): - continue - resolved = run_dir.resolve() - if resolved in seen: - continue - found.append((campaign, run_dir)) - seen.add(resolved) - return found - - -def _is_run_dir(path: Path) -> bool: - return (path / "timeline.jsonl").exists() and ( - (path / "summary.concise.json").exists() - or (path / "summary.json").exists() - or (path / "run_manifest.json").exists() - ) - - -def _row_from_metrics(campaign: str, run_dir: Path) -> RunRow: - metrics = summarize_run_metrics(run_dir) - run = metrics.get("run", {}) - derived = metrics.get("derived", {}) - dedup = metrics.get("dedup", {}) - crashes = _int(run.get("crashes")) - unique = max(_int(run.get("unique_crashes")), _int(dedup.get("unique_crash_signatures"))) - return RunRow( - campaign=campaign, - run_dir=str(run_dir), - run_id=str(run.get("run_id") or run_dir.name), - elapsed_sec=_float(run.get("elapsed_sec")), - cases=_int(run.get("cases")), - retained_cases=_int(run.get("retained_cases")), - coverage_features=_int(run.get("coverage_features")), - crashes=crashes, - unique_crashes=unique, - repeat_crashes=max(0, crashes - unique), - timeouts=_int(run.get("timeouts")), - skipped=_int(run.get("skipped")), - retained_density=_float(derived.get("retained_density")), - crash_density=_float(derived.get("crash_density")), - features_per_min=_float(derived.get("features_per_min")), - stop_reason=str(run.get("stop_reason") or ""), - ) - - -def _aggregate_rows(rows: list[RunRow]) -> dict[str, Any]: - elapsed_sec = sum(row.elapsed_sec for row in rows) - cases = sum(row.cases for row in rows) - retained = sum(row.retained_cases for row in rows) - features = sum(row.coverage_features for row in rows) - crashes = sum(row.crashes for row in rows) - unique = sum(row.unique_crashes for row in rows) - return { - "run_count": len(rows), - "elapsed_sec": round(elapsed_sec, 3), - "elapsed_hours": round(elapsed_sec / 3600.0, 3), - "cases": cases, - "cases_per_sec": _rate(cases, elapsed_sec), - "retained_cases": retained, - "retained_density": _ratio(retained, cases), - "coverage_features_sum": features, - "features_per_hour": _rate(features, elapsed_sec / 3600.0), - "crashes": crashes, - "unique_crashes_sum": unique, - "repeat_crashes_estimate": max(0, crashes - unique), - "crash_density": _ratio(crashes, cases), - "timeouts": sum(row.timeouts for row in rows), - "skipped": sum(row.skipped for row in rows), - "max_run_coverage_features": max((row.coverage_features for row in rows), default=0), - "max_run_features_per_min": max((row.features_per_min for row in rows), default=0.0), - } - - -def _aggregate_table(stats: dict[str, Any]) -> str: - return "\n".join( - [ - f"- Runs: `{stats.get('run_count', 0)}`", - f"- Elapsed hours: `{_fmt(stats.get('elapsed_hours', 0))}`", - f"- Cases: `{stats.get('cases', 0)}`", - f"- Retained cases: `{stats.get('retained_cases', 0)}`", - f"- Coverage features sum: `{stats.get('coverage_features_sum', 0)}`", - f"- Features/hour: `{_fmt(stats.get('features_per_hour', 0))}`", - f"- Crashes: `{stats.get('crashes', 0)}`", - f"- Unique crashes sum: `{stats.get('unique_crashes_sum', 0)}`", - f"- Crash density: `{_fmt(stats.get('crash_density', 0))}`", - ] - ) - - -def _ratio(numerator: int, denominator: int) -> float: - if denominator <= 0: - return 0.0 - return round(numerator / denominator, 6) - - -def _rate(value: int, elapsed: float) -> float: - if elapsed <= 0.0: - return 0.0 - return round(value / elapsed, 3) - - -def _fmt(value: Any) -> str: - if isinstance(value, float): - return f"{value:.3f}" - return str(value) - - -def _int(value: Any) -> int: - try: - return int(value) - except (TypeError, ValueError): - return 0 - - -def _float(value: Any) -> float: - try: - return float(value) - except (TypeError, ValueError): - return 0.0 diff --git a/parser-fuzzers/src/parser_fuzzers/models.py b/parser-fuzzers/src/parser_fuzzers/models.py deleted file mode 100644 index 82b365d..0000000 --- a/parser-fuzzers/src/parser_fuzzers/models.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.core.models`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.core.models") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/multitarget_runner.py b/parser-fuzzers/src/parser_fuzzers/multitarget_runner.py deleted file mode 100644 index b648898..0000000 --- a/parser-fuzzers/src/parser_fuzzers/multitarget_runner.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.runner.multitarget_runner`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.runner.multitarget_runner") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/output_feedback.py b/parser-fuzzers/src/parser_fuzzers/output_feedback.py deleted file mode 100644 index 1fb1d58..0000000 --- a/parser-fuzzers/src/parser_fuzzers/output_feedback.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.feedback.output_feedback`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.feedback.output_feedback") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/patcher.py b/parser-fuzzers/src/parser_fuzzers/patcher.py deleted file mode 100644 index 2dd71e9..0000000 --- a/parser-fuzzers/src/parser_fuzzers/patcher.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.patcher`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.patcher") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/ppd_templates.py b/parser-fuzzers/src/parser_fuzzers/ppd_templates.py deleted file mode 100644 index 946b7f3..0000000 --- a/parser-fuzzers/src/parser_fuzzers/ppd_templates.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.ppd_templates`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.ppd_templates") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/run_metrics.py b/parser-fuzzers/src/parser_fuzzers/run_metrics.py deleted file mode 100644 index eff5d17..0000000 --- a/parser-fuzzers/src/parser_fuzzers/run_metrics.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.metrics.run_metrics`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.metrics.run_metrics") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/run_recovery.py b/parser-fuzzers/src/parser_fuzzers/run_recovery.py deleted file mode 100644 index 5cb2b1b..0000000 --- a/parser-fuzzers/src/parser_fuzzers/run_recovery.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.metrics.run_recovery`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.metrics.run_recovery") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/run_set_metrics.py b/parser-fuzzers/src/parser_fuzzers/run_set_metrics.py deleted file mode 100644 index 542e3cf..0000000 --- a/parser-fuzzers/src/parser_fuzzers/run_set_metrics.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.metrics.run_set_metrics`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.metrics.run_set_metrics") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/runner/__init__.py b/parser-fuzzers/src/parser_fuzzers/runner/__init__.py deleted file mode 100644 index b05b17c..0000000 --- a/parser-fuzzers/src/parser_fuzzers/runner/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Layer package for parser-fuzzers.""" diff --git a/parser-fuzzers/src/parser_fuzzers/runner/cli.py b/parser-fuzzers/src/parser_fuzzers/runner/cli.py deleted file mode 100644 index 7403a35..0000000 --- a/parser-fuzzers/src/parser_fuzzers/runner/cli.py +++ /dev/null @@ -1,839 +0,0 @@ -from __future__ import annotations - -import argparse -import json -from pathlib import Path - -from parser_fuzzers.afl import afl_build_env, build_afl_plan, print_afl_plan, run_afl_plan -from parser_fuzzers.afl_integration.seed_export import export_template_seeds -from parser_fuzzers.afl_integration.template_seed_generation import generate_template_seeds -from parser_fuzzers.arithmetic_explorer import run_arithmetic_explore -from parser_fuzzers.auto_expand import build_auto_expand_plan, write_auto_expand_plan -from parser_fuzzers.baseline_compare import inspect_oss_fuzz_cups_filters, run_local_baseline_comparison -from parser_fuzzers.crash_dedup import dedup_run -from parser_fuzzers.dynamic_constraints import build_dynamic_compare_profile, write_dynamic_compare_profile -from parser_fuzzers.experiment import estimate_cpu_hours, load_experiment_rows -from parser_fuzzers.hashing import sha256_file -from parser_fuzzers.loop_metrics import write_standard_loop_metrics -from parser_fuzzers.models import BranchEvent -from parser_fuzzers.multitarget_runner import run_multitarget_monitor -from parser_fuzzers.output_feedback import build_output_feedback_profile, write_output_feedback_profile -from parser_fuzzers.patcher import apply_solver_result, load_solver_result, write_solver_result -from parser_fuzzers.run_recovery import recover_run_summary -from parser_fuzzers.run_metrics import summarize_run_metrics, write_run_metrics -from parser_fuzzers.run_set_metrics import render_run_set_markdown, summarize_run_set, write_run_set_metrics -from parser_fuzzers.source_constraints import mine_source_constraints, write_source_constraint_profile -from parser_fuzzers.solver import MissingSolverError, condition_holds, read_event_value, solve_event -from parser_fuzzers.template_feedback import build_feedback_profile, write_feedback_profile -from parser_fuzzers.validation import validate_all - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="parser-fuzzers") - subparsers = parser.add_subparsers(dest="command", required=True) - - validate_parser = subparsers.add_parser("validate", help="validate bug metadata and configs") - validate_parser.add_argument("--bugs", default="bugs") - validate_parser.add_argument("--configs", default="configs") - validate_parser.add_argument( - "--allow-missing-local-artifacts", - action="store_true", - help=( - "demote missing private reproducer/report paths to warnings; useful for clone-only smoke runs " - "outside the original /data/pre-gsoc workspace" - ), - ) - - solve_parser = subparsers.add_parser("solve-event", help="solve one branch event and emit patches") - solve_parser.add_argument("--event", required=True) - solve_parser.add_argument("--input") - solve_parser.add_argument("--output") - solve_parser.add_argument("--allow-fallback", action="store_true") - - patch_parser = subparsers.add_parser("patch-input", help="apply solver patches to an input") - patch_parser.add_argument("--result", required=True) - patch_parser.add_argument("--input") - patch_parser.add_argument("--output-dir", default="work/corpus/smt") - - smoke_parser = subparsers.add_parser("smoke", help="run a synthetic solver-to-patch workflow") - smoke_parser.add_argument("--work-dir", default="work/smoke") - smoke_parser.add_argument("--strict-z3", action="store_true") - - plan_parser = subparsers.add_parser("plan-experiment", help="print experiment matrix and cost") - plan_parser.add_argument("--configs", default="configs") - plan_parser.add_argument("--targets", type=int, default=3) - plan_parser.add_argument("--trials", type=int, default=10) - plan_parser.add_argument("--hours", type=int, default=24) - - afl_prepare_parser = subparsers.add_parser("afl-prepare", help="prepare AFL++ corpus/dict and print command") - _add_afl_common_args(afl_prepare_parser) - afl_prepare_parser.add_argument("--json", action="store_true") - - afl_run_parser = subparsers.add_parser("afl-run", help="print or execute an AFL++ run") - _add_afl_common_args(afl_run_parser) - afl_run_parser.add_argument("--execute", action="store_true", help="actually launch afl-fuzz") - afl_run_parser.add_argument("--json", action="store_true", help="print JSON when not executing") - afl_run_parser.add_argument( - "--allow-non-instrumented", - action="store_true", - help="allow executing a non-AFL-instrumented binary; standard runs should not use this", - ) - - afl_env_parser = subparsers.add_parser("afl-build-env", help="print AFL++ compiler environment") - afl_env_parser.add_argument("--configs", default="configs") - afl_env_parser.add_argument("--json", action="store_true") - - multitarget_parser = subparsers.add_parser( - "multitarget-monitor", - help="run multi-target PPD+document monitoring harness", - ) - multitarget_parser.add_argument("--config", default="configs/parser_targets.yaml") - multitarget_parser.add_argument("--work-root", default="work/multitarget") - multitarget_parser.add_argument( - "--filter-root", - help="rewrite direct_filter filter_binary entries to this directory before running", - ) - multitarget_parser.add_argument("--workers", type=int, default=4) - multitarget_parser.add_argument("--cases-per-target", type=int) - multitarget_parser.add_argument("--duration-sec", type=int) - multitarget_parser.add_argument("--timeout-sec", type=int, default=15) - multitarget_parser.add_argument("--max-run-gb", type=float, default=0.0) - multitarget_parser.add_argument("--discard-stdout", action="store_true") - multitarget_parser.add_argument("--discovery-mode", choices=["crash", "coverage"], default="crash") - multitarget_parser.add_argument("--scheduler", choices=["round-robin", "novelty"], default="round-robin") - multitarget_parser.add_argument( - "--min-target-share", - type=float, - default=0.0, - help="minimum scheduled-attempt share per target during duration campaigns", - ) - multitarget_parser.add_argument( - "--max-target-share", - type=float, - default=1.0, - help="maximum scheduled-attempt share per target during duration campaigns; 0 disables the cap", - ) - multitarget_parser.add_argument("--runtime-skip", action="store_true") - multitarget_parser.add_argument("--crash-skip-after", type=int, default=1) - multitarget_parser.add_argument( - "--seed-skip-state", - help="preload runtime crash-shape suppression from a previous discovery_state.json", - ) - multitarget_parser.add_argument( - "--auto-skip-state", - action="store_true", - help="preload the newest useful discovery_state.json under --auto-skip-root when --seed-skip-state is absent", - ) - multitarget_parser.add_argument( - "--auto-skip-root", - default="work", - help="bounded search root for --auto-skip-state", - ) - multitarget_parser.add_argument( - "--generalized-skip", - action="store_true", - help="also suppress whole target/PPD/document families after repeated seeded/runtime crash shapes", - ) - multitarget_parser.add_argument("--family-skip-after", type=int, default=32) - multitarget_parser.add_argument( - "--skip-probe-rate", - type=float, - default=0.0, - help="deterministic fraction of runtime-suppressed cases to still execute", - ) - multitarget_parser.add_argument( - "--skip-only-stop-after", - type=int, - default=0, - help="stop a duration campaign after this many consecutive skipped cases without a submitted run; 0 disables", - ) - multitarget_parser.add_argument( - "--stagnation-stop-after-sec", - type=int, - default=0, - help="stop coverage campaigns after this many seconds without retained coverage or a new crash; 0 disables", - ) - multitarget_parser.add_argument( - "--summary-mode", - choices=["full", "concise"], - default="full", - help="write full per-result summary.json or concise metadata-only summary.json", - ) - multitarget_parser.add_argument("--prune-uninteresting", action="store_true") - - arithmetic_parser = subparsers.add_parser( - "arithmetic-explore", - help="run time-budgeted cross-input arithmetic boundary exploration", - ) - arithmetic_parser.add_argument("--work-dir", default="work/arithmetic-explore") - arithmetic_parser.add_argument("--duration-sec", type=int, default=600) - arithmetic_parser.add_argument("--workers", type=int, default=4) - arithmetic_parser.add_argument("--timeout-sec", type=int, default=5) - arithmetic_parser.add_argument("--filter-binary", default="/usr/lib/cups/filter/pwgtoraster") - - dedup_parser = subparsers.add_parser("dedup-crashes", help="deduplicate crashes from a run directory") - dedup_parser.add_argument("--run-dir", required=True) - dedup_parser.add_argument("--output-json") - dedup_parser.add_argument("--output-md") - dedup_parser.add_argument("--include-timeouts", action="store_true") - dedup_parser.add_argument("--keep-infra", action="store_true") - - feedback_parser = subparsers.add_parser( - "build-template-feedback", - help="build a structural template feedback profile from retained/crashing run cases", - ) - feedback_parser.add_argument("--run-dir", required=True) - feedback_parser.add_argument("--output", required=True) - feedback_parser.add_argument("--max-cases-per-kind", type=int, default=128) - - output_feedback_parser = subparsers.add_parser( - "build-output-feedback", - help="build an output-structure feedback profile from run timeline semantic shapes", - ) - output_feedback_parser.add_argument("--run-dir", required=True) - output_feedback_parser.add_argument("--output", required=True) - - export_seeds_parser = subparsers.add_parser( - "export-template-seeds", - help="export retained template-generated documents into an AFL++ seed directory", - ) - export_seeds_parser.add_argument("--run-dir", required=True) - export_seeds_parser.add_argument("--output-dir", required=True) - export_seeds_parser.add_argument("--target-id", action="append", default=[]) - export_seeds_parser.add_argument("--extension", action="append", default=[]) - export_seeds_parser.add_argument("--limit", type=int, default=0) - export_seeds_parser.add_argument("--include-crashes", action="store_true") - export_seeds_parser.add_argument("--include-ppd", action="store_true") - - generate_seeds_parser = subparsers.add_parser( - "generate-template-seeds", - help="generate structured template documents directly into an AFL++ seed directory", - ) - generate_seeds_parser.add_argument("--document-kind", required=True) - generate_seeds_parser.add_argument("--output-dir", required=True) - generate_seeds_parser.add_argument("--count", type=int, default=64) - generate_seeds_parser.add_argument("--target-id", default="") - generate_seeds_parser.add_argument("--start-index", type=int, default=0) - generate_seeds_parser.add_argument("--extension", action="append", default=[]) - - recover_parser = subparsers.add_parser( - "recover-run-summary", - help="recover concise run summaries from timeline.jsonl when a campaign did not finish cleanly", - ) - recover_parser.add_argument("--run-dir", required=True) - - metrics_parser = subparsers.add_parser( - "summarize-run-metrics", - help="summarize run, dedup, AFL++, and optional LLVM coverage metrics as JSON", - ) - metrics_parser.add_argument("--run-dir", required=True) - metrics_parser.add_argument("--afl-output-dir") - metrics_parser.add_argument("--llvm-coverage-json") - metrics_parser.add_argument("--output") - - loop_metrics_parser = subparsers.add_parser( - "summarize-loop-metrics", - help="summarize a template -> AFL++ -> feedback-template campaign as JSON", - ) - loop_metrics_parser.add_argument("--campaign-dir", required=True) - loop_metrics_parser.add_argument("--output") - - run_set_metrics_parser = subparsers.add_parser( - "summarize-run-set", - help="summarize multiple campaign directories into aggregate metrics", - ) - run_set_metrics_parser.add_argument("--root", action="append", required=True) - run_set_metrics_parser.add_argument("--output", required=True) - run_set_metrics_parser.add_argument("--output-md") - - oss_status_parser = subparsers.add_parser( - "oss-fuzz-status", - help="inspect local OSS-Fuzz cups-filters availability", - ) - oss_status_parser.add_argument("--oss-fuzz-dir", default="/data/pre-gsoc/oss-fuzz") - - compare_parser = subparsers.add_parser( - "compare-baseline-metrics", - help="run a local OSS-Fuzz-style baseline and an optimized semantic run, then compare metrics", - ) - compare_parser.add_argument("--config", default="work/parser_targets_cold_semantic_llvm.yaml") - compare_parser.add_argument("--work-root", default="work/baseline-comparison") - compare_parser.add_argument("--oss-fuzz-dir", default="/data/pre-gsoc/oss-fuzz") - compare_parser.add_argument("--duration-sec", type=int, default=60) - compare_parser.add_argument("--workers", type=int, default=4) - compare_parser.add_argument("--timeout-sec", type=int, default=5) - compare_parser.add_argument("--max-run-gb", type=float, default=10.0) - compare_parser.add_argument("--enable-llvm-profiles", action="store_true") - compare_parser.add_argument("--export-llvm-coverage", action="store_true") - compare_parser.add_argument( - "--optimized-policy", - choices=["avoidance", "no-crash-avoidance"], - default="avoidance", - help="choose whether the optimized comparison run uses crash-avoidance suppression", - ) - - auto_expand_parser = subparsers.add_parser( - "auto-expand", - help="build a frontier feedback profile and next-run expansion plan from previous campaigns", - ) - auto_expand_parser.add_argument("--search-root", default="work") - auto_expand_parser.add_argument("--output-profile", default="work/template-feedback/auto-expand-feedback.json") - auto_expand_parser.add_argument("--plan-output", default="") - auto_expand_parser.add_argument("--max-runs", type=int, default=8) - auto_expand_parser.add_argument("--max-cases-per-kind", type=int, default=160) - auto_expand_parser.add_argument("--stale-window", type=int, default=5000) - auto_expand_parser.add_argument("--duration-sec", type=int, default=1200) - auto_expand_parser.add_argument("--workers", type=int, default=10) - auto_expand_parser.add_argument("--timeout-sec", type=int, default=5) - auto_expand_parser.add_argument("--max-run-gb", type=float, default=10.0) - auto_expand_parser.add_argument("--skip-probe-rate", type=float, default=0.01) - - source_constraints_parser = subparsers.add_parser( - "mine-source-constraints", - help="mine source-code comparison and field hints for SMT/template biasing", - ) - source_constraints_parser.add_argument( - "--source-dir", - action="append", - required=True, - help="C/C++ source directory or file to scan; can be repeated", - ) - source_constraints_parser.add_argument("--output", required=True) - source_constraints_parser.add_argument("--max-records", type=int, default=2000) - - dynamic_constraints_parser = subparsers.add_parser( - "summarize-dynamic-constraints", - help="summarize per-case dynamic compare traces from a run directory", - ) - dynamic_constraints_parser.add_argument("--run-dir", required=True) - dynamic_constraints_parser.add_argument("--output", required=True) - dynamic_constraints_parser.add_argument("--max-records", type=int, default=2000) - - args = parser.parse_args(argv) - if args.command == "validate": - return _cmd_validate(args) - if args.command == "solve-event": - return _cmd_solve_event(args) - if args.command == "patch-input": - return _cmd_patch_input(args) - if args.command == "smoke": - return _cmd_smoke(args) - if args.command == "plan-experiment": - return _cmd_plan_experiment(args) - if args.command == "afl-prepare": - return _cmd_afl_prepare(args) - if args.command == "afl-run": - return _cmd_afl_run(args) - if args.command == "afl-build-env": - return _cmd_afl_build_env(args) - if args.command == "multitarget-monitor": - return _cmd_multitarget_monitor(args) - if args.command == "arithmetic-explore": - return _cmd_arithmetic_explore(args) - if args.command == "dedup-crashes": - return _cmd_dedup_crashes(args) - if args.command == "build-template-feedback": - return _cmd_build_template_feedback(args) - if args.command == "build-output-feedback": - return _cmd_build_output_feedback(args) - if args.command == "export-template-seeds": - return _cmd_export_template_seeds(args) - if args.command == "generate-template-seeds": - return _cmd_generate_template_seeds(args) - if args.command == "recover-run-summary": - return _cmd_recover_run_summary(args) - if args.command == "summarize-run-metrics": - return _cmd_summarize_run_metrics(args) - if args.command == "summarize-loop-metrics": - return _cmd_summarize_loop_metrics(args) - if args.command == "summarize-run-set": - return _cmd_summarize_run_set(args) - if args.command == "oss-fuzz-status": - return _cmd_oss_fuzz_status(args) - if args.command == "compare-baseline-metrics": - return _cmd_compare_baseline_metrics(args) - if args.command == "auto-expand": - return _cmd_auto_expand(args) - if args.command == "mine-source-constraints": - return _cmd_mine_source_constraints(args) - if args.command == "summarize-dynamic-constraints": - return _cmd_summarize_dynamic_constraints(args) - parser.error(f"unknown command: {args.command}") - return 2 - - -def _cmd_validate(args: argparse.Namespace) -> int: - issues = validate_all( - args.bugs, - args.configs, - require_local_artifacts=not args.allow_missing_local_artifacts, - ) - for issue in issues: - print(f"{issue.level.upper()}: {issue.path}: {issue.message}") - error_count = sum(1 for issue in issues if issue.level == "error") - warning_count = sum(1 for issue in issues if issue.level == "warning") - print(json.dumps({"errors": error_count, "warnings": warning_count}, sort_keys=True)) - return 1 if error_count else 0 - - -def _cmd_solve_event(args: argparse.Namespace) -> int: - event = _load_event(args.event) - input_path = Path(args.input or event.input_path) - input_bytes = input_path.read_bytes() - _verify_hash(event, input_path) - try: - result = solve_event(event, input_bytes, allow_fallback=args.allow_fallback) - except MissingSolverError as exc: - print(str(exc)) - return 2 - if args.output: - write_solver_result(result, args.output) - else: - print(json.dumps(result.to_dict(), indent=2, sort_keys=True)) - return 0 if result.status in {"sat", "already_satisfied"} else 1 - - -def _cmd_patch_input(args: argparse.Namespace) -> int: - result = load_solver_result(args.result) - input_path = Path(args.input or result.event.input_path) - output = apply_solver_result(result, input_path, args.output_dir) - print(output) - return 0 - - -def _cmd_smoke(args: argparse.Namespace) -> int: - work_dir = Path(args.work_dir) - work_dir.mkdir(parents=True, exist_ok=True) - input_path = work_dir / "input.bin" - event_path = work_dir / "event.json" - result_path = work_dir / "result.json" - output_dir = work_dir / "corpus" - - input_path.write_bytes(b"\x00SMT-FUZZER-SMOKE\n") - event = BranchEvent( - target_id="synthetic_eq_u8", - input_path=str(input_path), - input_sha256=sha256_file(input_path), - offset=0, - width=1, - endianness="little", - signed=False, - op="eq", - rhs=0x41, - description="first byte must become ASCII A", - ) - event_path.write_text(json.dumps(event.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") - - try: - result = solve_event(event, input_path.read_bytes(), allow_fallback=not args.strict_z3) - except MissingSolverError as exc: - print(str(exc)) - return 2 - write_solver_result(result, result_path) - output_path = apply_solver_result(result, input_path, output_dir) - patched = output_path.read_bytes() - patched_value = read_event_value(event, patched) - ok = condition_holds(event, patched_value) - print( - json.dumps( - { - "ok": ok, - "event": str(event_path), - "result": str(result_path), - "patched_input": str(output_path), - "reason": result.reason, - }, - indent=2, - sort_keys=True, - ) - ) - return 0 if ok else 1 - - -def _cmd_plan_experiment(args: argparse.Namespace) -> int: - rows = load_experiment_rows(Path(args.configs) / "experiment.yaml") - cpu_hours = estimate_cpu_hours(len(rows), args.targets, args.trials, args.hours) - print("ID Name Dict CmpLog Grammar SMT") - for row in rows: - print( - f"{row.id:<3} {row.name:<24} " - f"{_yes(row.dictionary):<5} {_yes(row.cmplog):<7} {_yes(row.grammar):<8} {_yes(row.smt):<3}" - ) - print() - print( - json.dumps( - { - "configs": len(rows), - "targets": args.targets, - "trials": args.trials, - "hours_per_trial": args.hours, - "cpu_hours": cpu_hours, - }, - sort_keys=True, - ) - ) - return 0 - - -def _cmd_afl_prepare(args: argparse.Namespace) -> int: - plan = _build_afl_plan_from_args(args) - print_afl_plan(plan, as_json=args.json) - return 0 - - -def _cmd_afl_run(args: argparse.Namespace) -> int: - plan = _build_afl_plan_from_args(args) - if args.execute: - return run_afl_plan(plan, execute=True, require_instrumented=not args.allow_non_instrumented) - print_afl_plan(plan, as_json=args.json) - return 0 - - -def _cmd_afl_build_env(args: argparse.Namespace) -> int: - env = afl_build_env(args.configs) - if args.json: - print(json.dumps(env, indent=2, sort_keys=True)) - else: - for key, value in env.items(): - print(f"export {key}={_shell_quote(value)}") - return 0 - - -def _cmd_multitarget_monitor(args: argparse.Namespace) -> int: - run_command = ( - "PYTHONPATH=src python3 -m parser_fuzzers.cli multitarget-monitor " - f"--config {_shell_quote(args.config)} " - f"--work-root {_shell_quote(args.work_root)} " - f"--workers {args.workers} " - f"--timeout-sec {args.timeout_sec}" - ) - if args.filter_root: - run_command += f" --filter-root {_shell_quote(args.filter_root)}" - if args.cases_per_target is not None: - run_command += f" --cases-per-target {args.cases_per_target}" - if args.duration_sec is not None: - run_command += f" --duration-sec {args.duration_sec}" - if args.max_run_gb: - run_command += f" --max-run-gb {args.max_run_gb:g}" - if args.discard_stdout: - run_command += " --discard-stdout" - if args.discovery_mode != "crash": - run_command += f" --discovery-mode {args.discovery_mode}" - if args.scheduler != "round-robin": - run_command += f" --scheduler {args.scheduler}" - if args.min_target_share: - run_command += f" --min-target-share {args.min_target_share:g}" - if args.max_target_share != 1.0: - run_command += f" --max-target-share {args.max_target_share:g}" - if args.runtime_skip: - run_command += " --runtime-skip" - if args.crash_skip_after != 1: - run_command += f" --crash-skip-after {args.crash_skip_after}" - if args.seed_skip_state: - run_command += f" --seed-skip-state {_shell_quote(args.seed_skip_state)}" - if args.auto_skip_state: - run_command += " --auto-skip-state" - if args.auto_skip_root != "work": - run_command += f" --auto-skip-root {_shell_quote(args.auto_skip_root)}" - if args.generalized_skip: - run_command += " --generalized-skip" - if args.family_skip_after != 32: - run_command += f" --family-skip-after {args.family_skip_after}" - if args.skip_probe_rate: - run_command += f" --skip-probe-rate {args.skip_probe_rate:g}" - if args.skip_only_stop_after: - run_command += f" --skip-only-stop-after {args.skip_only_stop_after}" - if args.stagnation_stop_after_sec: - run_command += f" --stagnation-stop-after-sec {args.stagnation_stop_after_sec}" - if args.summary_mode != "full": - run_command += f" --summary-mode {args.summary_mode}" - if args.prune_uninteresting: - run_command += " --prune-uninteresting" - summary = run_multitarget_monitor( - config_path=args.config, - work_root=args.work_root, - workers=args.workers, - cases_per_target=args.cases_per_target, - duration_sec=args.duration_sec, - timeout_sec=args.timeout_sec, - max_run_gb=args.max_run_gb or None, - run_command=run_command, - capture_stdout=not args.discard_stdout, - filter_root=args.filter_root, - discovery_mode=args.discovery_mode, - scheduler=args.scheduler, - min_target_share=args.min_target_share, - max_target_share=args.max_target_share, - runtime_skip=args.runtime_skip, - crash_skip_after=args.crash_skip_after, - prune_uninteresting=args.prune_uninteresting, - seed_skip_state_path=args.seed_skip_state, - auto_skip_state=args.auto_skip_state, - auto_skip_search_root=args.auto_skip_root, - generalized_skip=args.generalized_skip, - family_skip_after=args.family_skip_after, - skip_probe_rate=args.skip_probe_rate, - skip_only_stop_after=args.skip_only_stop_after, - stagnation_stop_after_sec=args.stagnation_stop_after_sec, - summary_mode=args.summary_mode, - ) - if args.duration_sec is not None: - print(json.dumps(summary.concise_dict(), indent=2, sort_keys=True)) - else: - print(json.dumps(summary.to_dict(), indent=2, sort_keys=True)) - return 0 - - -def _cmd_arithmetic_explore(args: argparse.Namespace) -> int: - summary = run_arithmetic_explore( - work_dir=args.work_dir, - duration_sec=args.duration_sec, - workers=args.workers, - timeout_sec=args.timeout_sec, - filter_binary=args.filter_binary, - ) - print(json.dumps(summary.concise_dict(), indent=2, sort_keys=True)) - return 0 - - -def _cmd_dedup_crashes(args: argparse.Namespace) -> int: - summary = dedup_run( - args.run_dir, - output_json=args.output_json, - output_md=args.output_md, - include_timeouts=args.include_timeouts, - exclude_infra=not args.keep_infra, - ) - print( - json.dumps( - { - "run_dir": summary.run_dir, - "crash_records": summary.crash_records, - "infra_excluded_records": summary.infra_excluded_records, - "unique_crashes": summary.unique_crashes, - "clusters": [ - { - "target_id": cluster.target_id, - "count": cluster.count, - "signature": cluster.signature, - "representative_work_dir": cluster.representative_work_dir, - } - for cluster in summary.clusters - ], - }, - indent=2, - sort_keys=True, - ) - ) - return 0 - - -def _cmd_build_template_feedback(args: argparse.Namespace) -> int: - profile = build_feedback_profile( - args.run_dir, - max_cases_per_kind=args.max_cases_per_kind, - ) - write_feedback_profile(profile, args.output) - print( - json.dumps( - { - "run_dir": args.run_dir, - "output": args.output, - "cups_seeds": len(profile.cups), - "pwg_seeds": len(profile.pwg), - "image_seeds": len(profile.images), - }, - indent=2, - sort_keys=True, - ) - ) - return 0 - - -def _cmd_build_output_feedback(args: argparse.Namespace) -> int: - profile = build_output_feedback_profile(args.run_dir) - write_output_feedback_profile(profile, args.output) - print( - json.dumps( - { - "run_dir": args.run_dir, - "output": args.output, - "formats": profile.format_counts, - "structures": len(profile.structure_counts), - "objectives": len(profile.objective_counts), - }, - indent=2, - sort_keys=True, - ) - ) - return 0 - - -def _cmd_export_template_seeds(args: argparse.Namespace) -> int: - summary = export_template_seeds( - run_dir=args.run_dir, - output_dir=args.output_dir, - target_ids=args.target_id, - extensions=args.extension, - limit=args.limit, - include_crashes=args.include_crashes, - include_ppd=args.include_ppd, - ) - print(json.dumps(summary.to_dict(), indent=2, sort_keys=True)) - return 0 if summary.exported else 1 - - -def _cmd_generate_template_seeds(args: argparse.Namespace) -> int: - summary = generate_template_seeds( - document_kind=args.document_kind, - output_dir=args.output_dir, - count=args.count, - target_id=args.target_id, - start_index=args.start_index, - extensions=args.extension, - ) - print(json.dumps(summary.to_dict(), indent=2, sort_keys=True)) - return 0 if summary.generated else 1 - - -def _cmd_recover_run_summary(args: argparse.Namespace) -> int: - summary = recover_run_summary(args.run_dir) - print(json.dumps(summary, indent=2, sort_keys=True)) - return 0 - - -def _cmd_summarize_run_metrics(args: argparse.Namespace) -> int: - payload = summarize_run_metrics( - args.run_dir, - afl_output_dir=args.afl_output_dir, - llvm_coverage_json=args.llvm_coverage_json, - ) - if args.output: - write_run_metrics(payload, args.output) - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - - -def _cmd_summarize_loop_metrics(args: argparse.Namespace) -> int: - payload = write_standard_loop_metrics(args.campaign_dir, output_path=args.output) - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - - -def _cmd_summarize_run_set(args: argparse.Namespace) -> int: - payload = summarize_run_set(args.root) - write_run_set_metrics(payload, args.output, args.output_md) - if args.output_md: - print(render_run_set_markdown(payload)) - else: - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - - -def _cmd_oss_fuzz_status(args: argparse.Namespace) -> int: - status = inspect_oss_fuzz_cups_filters(args.oss_fuzz_dir) - print(json.dumps(status.to_dict(), indent=2, sort_keys=True)) - return 0 if status.project_exists else 1 - - -def _cmd_compare_baseline_metrics(args: argparse.Namespace) -> int: - result = run_local_baseline_comparison( - config_path=args.config, - work_root=args.work_root, - oss_fuzz_dir=args.oss_fuzz_dir, - duration_sec=args.duration_sec, - workers=args.workers, - timeout_sec=args.timeout_sec, - max_run_gb=args.max_run_gb, - enable_llvm_profiles=args.enable_llvm_profiles, - export_llvm_coverage=args.export_llvm_coverage, - optimized_policy=args.optimized_policy, - ) - print(json.dumps(result.to_dict(), indent=2, sort_keys=True)) - return 0 - - -def _cmd_auto_expand(args: argparse.Namespace) -> int: - plan = build_auto_expand_plan( - search_root=args.search_root, - output_profile=args.output_profile, - max_runs=args.max_runs, - max_cases_per_kind=args.max_cases_per_kind, - stale_window=args.stale_window, - duration_sec=args.duration_sec, - workers=args.workers, - timeout_sec=args.timeout_sec, - max_run_gb=args.max_run_gb, - skip_probe_rate=args.skip_probe_rate, - ) - if args.plan_output: - write_auto_expand_plan(plan, args.plan_output) - print(json.dumps(plan.to_dict(), indent=2, sort_keys=True)) - return 0 - - -def _cmd_mine_source_constraints(args: argparse.Namespace) -> int: - profile = mine_source_constraints(args.source_dir, max_records=args.max_records) - write_source_constraint_profile(profile, args.output) - print(json.dumps(profile["summary"], indent=2, sort_keys=True)) - return 0 - - -def _cmd_summarize_dynamic_constraints(args: argparse.Namespace) -> int: - profile = build_dynamic_compare_profile(args.run_dir, max_records=args.max_records) - write_dynamic_compare_profile(profile, args.output) - print(json.dumps(profile["summary"], indent=2, sort_keys=True)) - return 0 - - -def _build_afl_plan_from_args(args: argparse.Namespace): - return build_afl_plan( - root=args.root, - configs_dir=args.configs, - target_id=args.target, - config_ref=args.config, - binary=args.binary, - input_dir=args.input_dir, - output_dir=args.output_dir, - timeout_ms=args.timeout_ms, - memory_mb=args.memory_mb, - duration_sec=args.duration_sec, - ) - - -def _load_event(path: str | Path) -> BranchEvent: - with Path(path).open("r", encoding="utf-8") as handle: - return BranchEvent.from_dict(json.load(handle)) - - -def _verify_hash(event: BranchEvent, input_path: Path) -> None: - actual = sha256_file(input_path) - if actual != event.input_sha256: - raise SystemExit(f"input hash mismatch: expected {event.input_sha256}, got {actual}") - - -def _yes(value: bool) -> str: - return "yes" if value else "no" - - -def _add_afl_common_args(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--root", default=".") - parser.add_argument("--configs", default="configs") - parser.add_argument("--target", required=True) - parser.add_argument("--config", required=True, help="A0-A4 id or config name") - parser.add_argument("--binary", required=True, help="AFL-instrumented target binary") - parser.add_argument("--input-dir", help="use this seed directory instead of preparing one from configs/afl.yaml") - parser.add_argument("--output-dir", help="write AFL++ output here instead of configs/afl.yaml default") - parser.add_argument("--duration-sec", type=int, help="add AFL++ -V duration seconds") - parser.add_argument("--timeout-ms", type=int, help="override AFL++ -t timeout in milliseconds") - parser.add_argument("--memory-mb", help="override AFL++ -m memory limit; use 'none' when appropriate") - - -def _shell_quote(value: str) -> str: - if value.replace("_", "").replace("-", "").replace("/", "").replace(".", "").replace(",", "").isalnum(): - return value - return "'" + value.replace("'", "'\"'\"'") + "'" - - -if __name__ == "__main__": # pragma: no cover - raise SystemExit(main()) diff --git a/parser-fuzzers/src/parser_fuzzers/runner/document_harness.py b/parser-fuzzers/src/parser_fuzzers/runner/document_harness.py deleted file mode 100644 index 3b80b29..0000000 --- a/parser-fuzzers/src/parser_fuzzers/runner/document_harness.py +++ /dev/null @@ -1,1032 +0,0 @@ -from __future__ import annotations - -import zlib -import struct -from dataclasses import dataclass - -from parser_fuzzers.image_templates import image_feedback_instance -from parser_fuzzers.structured_templates import ( - cups_feedback_instance, - cups_structural_instance, - pwg_feedback_instance, - pwg_structural_instance, -) -from parser_fuzzers.template_synth import ( - synthesize_cups_raster_slots, - synthesize_image_slots, - synthesize_pwg_raster_slots, -) - - -HEADER_SIZE = 1796 -SYNC_CUPS_RASTER_V3 = b"3SaR" -SYNC_PWG_RASTER = b"2SaR" - -OFF_MEDIA_CLASS = 0 -OFF_MEDIA_TYPE = 128 -OFF_OUTPUT_TYPE = 192 -OFF_HW_RESOLUTION = 276 -OFF_IMAGING_BBOX = 284 -OFF_MARGINS = 312 -OFF_PAGE_SIZE = 352 -OFF_CUPS_WIDTH = 372 -OFF_CUPS_HEIGHT = 376 -OFF_CUPS_BITS_PER_COLOR = 384 -OFF_CUPS_BITS_PER_PIXEL = 388 -OFF_CUPS_BYTES_PER_LINE = 392 -OFF_CUPS_COLOR_ORDER = 396 -OFF_CUPS_COLOR_SPACE = 400 -OFF_CUPS_COMPRESSION = 404 -OFF_CUPS_ROW_COUNT = 408 -OFF_CUPS_NUM_COLORS = 420 -OFF_CUPS_PAGE_SIZE = 428 -OFF_CUPS_IMAGING_BBOX = 436 -OFF_CUPS_PAGE_SIZE_NAME = 1732 - -CUPS_CSPACE_RGB = 1 -CUPS_CSPACE_K = 3 -CUPS_CSPACE_CMYK = 6 -CUPS_CSPACE_SW = 18 - -RASTER_BOUNDARY_CASES = [ - (8, 1, 0, 1), - (16, 1, 0, 3), - (9, 1, 1, 3), - (11, 1, 10, 3), - (14, 1, 10, 3), - (5, 6, 10, 3), - (31, 1, 0, 1), - (64, 2, 0, 3), -] - -RASTER_GENERAL_CASES = [ - (1, 1, 0, 1), - (8, 1, 0, 1), - (16, 1, 0, 1), - (32, 2, 0, 1), - (64, 4, 0, 1), - (127, 1, 0, 1), - (255, 2, 0, 1), - (8, 1, 0, 3), - (16, 2, 0, 3), - (31, 3, 0, 3), - (64, 1, 0, 3), - (128, 4, 0, 3), -] - -RASTER_COVERAGE_CASES = [ - # width, height, compression, num_colors, color_space, color_order, bits_per_pixel, pages, x_res, y_res - (1, 1, 0, 1, CUPS_CSPACE_K, 0, 8, 1, 300, 300), - (8, 1, 0, 1, CUPS_CSPACE_SW, 0, 8, 2, 300, 300), - (16, 2, 0, 1, CUPS_CSPACE_K, 0, 8, 1, 600, 300), - (32, 1, 0, 3, CUPS_CSPACE_RGB, 0, 24, 1, 300, 600), - (48, 3, 0, 3, CUPS_CSPACE_RGB, 0, 24, 2, 203, 203), - (63, 1, 0, 4, CUPS_CSPACE_CMYK, 0, 32, 1, 360, 360), - (65, 2, 0, 1, CUPS_CSPACE_SW, 0, 8, 3, 720, 360), - (127, 1, 0, 3, CUPS_CSPACE_RGB, 0, 24, 1, 1200, 600), - (255, 2, 0, 1, CUPS_CSPACE_K, 0, 8, 1, 75, 75), - (17, 4, 0, 3, CUPS_CSPACE_RGB, 0, 24, 2, 100, 100), - (33, 1, 0, 1, CUPS_CSPACE_SW, 0, 8, 1, 150, 300), - (96, 2, 0, 4, CUPS_CSPACE_CMYK, 0, 32, 1, 600, 1200), - (12, 1, 1, 3, CUPS_CSPACE_RGB, 0, 24, 1, 300, 300), - (24, 2, 10, 3, CUPS_CSPACE_RGB, 0, 24, 1, 300, 300), -] - -PWG_BOUNDARY_CASES = [ - (8, 1, 8, 300), - (16, 1, 1, 600), - (3, 2, 8, 1200), - (11, 1, 24, 65535), - (4, 4, 16, 65536), - (7, 2, 16, 2147483647), - (13, 1, 8, 4294967295), - (5, 6, 16, 2147483648), -] - -PWG_GENERAL_CASES = [ - (1, 1, 8, 72), - (8, 1, 8, 150), - (16, 2, 8, 203), - (32, 4, 8, 300), - (64, 1, 8, 600), - (127, 2, 8, 1200), - (255, 1, 8, 2400), - (8, 1, 16, 300), - (16, 2, 16, 600), - (31, 3, 16, 1200), - (64, 1, 16, 65535), - (128, 2, 16, 65536), - (8, 1, 24, 300), - (16, 2, 24, 600), - (31, 1, 32, 1200), -] - -PWG_COVERAGE_CASES = [ - # width, height, bits_per_pixel, y_res, pages - (1, 1, 8, 72, 1), - (8, 2, 8, 150, 2), - (16, 1, 16, 203, 1), - (32, 3, 8, 300, 2), - (64, 1, 16, 600, 1), - (127, 2, 8, 1200, 1), - (255, 1, 24, 2400, 1), - (31, 3, 32, 1200, 2), - (65, 4, 16, 65535, 1), - (129, 1, 8, 32768, 1), -] - -PDF_COVERAGE_CASES = [ - # media box, body text - ((0, 0, 200, 200), "SMT PDF smoke"), - ((0, 0, 612, 792), "Letter page"), - ((0, 0, 144, 144), "Small page"), - ((0, 0, 1008, 612), "Wide page"), -] -PDF_SEMANTIC_PERIOD = 32 - -IMAGE_COVERAGE_CASES = [ - # format, width, height, channels - ("png_rgb", 4, 4, 3), - ("png_gray", 9, 2, 1), - ("ppm", 1, 1, 3), - ("ppm", 8, 2, 3), - ("pgm", 16, 1, 1), - ("pbm", 17, 3, 1), -] - -TEXT_COVERAGE_CASES = [ - b"SMT text parser smoke\n", - b"Header: value\n\nBody line 1\nBody line 2\n", - b"\tIndented\tcolumns\t12345\n", - b"A" * 256 + b"\n", - b"%%Title: looks-like-postscript-but-text\nplain\n", -] - -POSTSCRIPT_COVERAGE_CASES = [ - b"%!PS-Adobe-3.0\n%%Pages: 1\nshowpage\n", - b"%!PS-Adobe-3.0\n%%BoundingBox: 0 0 144 144\n/newpath { } def\nshowpage\n", - b"%!PS\n72 72 moveto (SMT PostScript) show\nshowpage\n", - b"%!PS-Adobe-3.0\n%%Pages: 2\n%%Page: 1 1\nshowpage\n%%Page: 2 2\nshowpage\n", -] - -COMMAND_COVERAGE_CASES = [ - b"#CUPS-COMMAND\nReportLevels\n", - b"#CUPS-COMMAND\nClean\n", - b"#CUPS-COMMAND\nPrintSelfTestPage\n", - b"#CUPS-COMMAND\nPrintAlignmentPage 1\n", - b"#CUPS-COMMAND\nSetAlignment 0 0\nUnknownCommand\n", -] - -TEXT_SEMANTIC_CASES = [ - b"\xef\xbb\xbfTitle\tColumn\tValue\r\nOne\tTwo\tThree\r\n", - b"Line 001\r\nLine 002\r\n\fLine after form feed\r\n", - b"Header: value\nContinuation: " + b"x" * 96 + b"\n\nBody\n", - b"\x1b%-12345X@PJL INFO STATUS\r\nPlain text after printer control\r\n", - b"Column A Column B Column C\n" + b"1234567890 " * 24 + b"\n", - b"Backspace demo: ABC\b\bXY\nOverprint-like line\r\n", - b"\tIndented\tcolumns\twith\ttabs\n" + b" " * 64 + b"right edge\n", - b"%%BeginFeature: *InputSlot Tray1\nplain text payload\n%%EndFeature\n", - b"Page 1\n\fPage 2\n\fPage 3 with trailing bytes\0\0\n", - b"UTF8-ish bytes: \xc2\xa9 \xe2\x82\xac \xf0\x9f\x98\x80\n", -] - -POSTSCRIPT_SEMANTIC_CASES = [ - ( - b"%!PS-Adobe-3.0\n%%Pages: 1\n" - b"<< /PageSize [144 144] /ImagingBBox null >> setpagedevice\n" - b"/Courier findfont 10 scalefont setfont\n24 72 moveto (setpagedevice path) show\nshowpage\n" - ), - ( - b"%!PS-Adobe-3.0\n%%BoundingBox: 0 0 64 64\n" - b"gsave 32 32 translate 30 rotate 1 0 0 setrgbcolor\n" - b"0 0 moveto 24 0 lineto 12 24 lineto closepath fill\ngrestore\nshowpage\n" - ), - ( - b"%!PS-Adobe-3.0\n%%Pages: 2\n%%Page: 1 1\n" - b"/F /Helvetica findfont 12 scalefont def F setfont\n24 24 moveto (page one) show\nshowpage\n" - b"%%Page: 2 2\n90 rotate 24 -96 moveto (rotated page two) show\nshowpage\n" - ), - ( - b"%!PS-Adobe-3.0\n%%BoundingBox: 0 0 16 16\n/picstr 2 string def\n" - b"2 2 1 [2 0 0 -2 0 2] {<80C04020>} image\nshowpage\n" - ), - ( - b"%!PS\n/userdict 12 dict dup begin /x 42 def /paint { x 2 mul 24 moveto (dict) show } bind def end def\n" - b"userdict begin /Helvetica findfont 9 scalefont setfont paint end showpage\n" - ), - ( - b"%!PS-Adobe-3.0\n%%BeginResource: procset smt 1 0\n" - b"/box { newpath 0 0 moveto 50 0 lineto 50 50 lineto 0 50 lineto closepath stroke } bind def\n" - b"%%EndResource\n10 10 translate box showpage\n" - ), - ( - b"%!PS-Adobe-3.0\n%%LanguageLevel: 2\n" - b"<< /Policies << /PageSize 3 >> /PageSize [612 792] >> setpagedevice\n" - b"/Times-Roman findfont 14 scalefont setfont 72 720 moveto (policy page) show showpage\n" - ), - ( - b"%!PS-Adobe-3.0\n%%BoundingBox: 0 0 128 128\n" - b"/DeviceGray setcolorspace 0.5 setgray 16 16 96 64 rectfill\n" - b"/DeviceRGB setcolorspace 1 0 0 setrgbcolor 24 24 32 32 rectstroke\nshowpage\n" - ), -] - -COMMAND_SEMANTIC_CASES = [ - b"#CUPS-COMMAND\nReportLevels\nReportStatus\n", - b"#CUPS-COMMAND\nClean all\nClean print-heads\n", - b"#CUPS-COMMAND\nPrintSelfTestPage\nPrintAlignmentPage 1\n", - b"#CUPS-COMMAND\nSetAlignment 0 0\nSetAlignment 1 -1\n", - b"#CUPS-COMMAND\nAutoConfigure\nReportConfig\nReportLevels\n", - b"#CUPS-COMMAND\nReportStatus\nUnknownCommand key=value count=3\n", - b"#CUPS-COMMAND\r\nClean\r\nPrintSelfTestPage\r\n", - b"#CUPS-COMMAND\nSetAlignment 2147483647 -2147483648\nReportStatus\n", - b"#CUPS-COMMAND\n# comment line\nReportLevels\n\nReportConfig\n", - b"#CUPS-COMMAND\nNoOp\nClean\nReportStatus\nPrintAlignmentPage 99\n", -] - - -@dataclass(frozen=True) -class DocumentCase: - kind: str - data: bytes - mime: str - description: str - extension: str = ".bin" - - -def make_document(kind: str, case_index: int, target_id: str = "") -> DocumentCase: - if kind == "text": - return DocumentCase( - kind=kind, - data=b"SMT multi-target text job\n", - mime="text/plain", - description="minimal text job", - extension=".txt", - ) - if kind == "text_coverage_sweep": - data = TEXT_COVERAGE_CASES[case_index % len(TEXT_COVERAGE_CASES)] - return DocumentCase( - kind=kind, - data=data, - mime="text/plain", - description="coverage-oriented text input", - extension=".txt", - ) - if kind == "text_semantic_sweep": - data = TEXT_SEMANTIC_CASES[case_index % len(TEXT_SEMANTIC_CASES)] - return DocumentCase( - kind=kind, - data=data, - mime="text/plain", - description="semantic text input", - extension=".txt", - ) - if kind == "postscript": - return DocumentCase( - kind=kind, - data=b"%!PS-Adobe-3.0\n%%Pages: 1\nshowpage\n", - mime="application/postscript", - description="minimal PostScript job", - extension=".ps", - ) - if kind == "postscript_coverage_sweep": - data = POSTSCRIPT_COVERAGE_CASES[case_index % len(POSTSCRIPT_COVERAGE_CASES)] - return DocumentCase( - kind=kind, - data=data, - mime="application/postscript", - description="coverage-oriented PostScript input", - extension=".ps", - ) - if kind == "postscript_semantic_sweep": - data = POSTSCRIPT_SEMANTIC_CASES[case_index % len(POSTSCRIPT_SEMANTIC_CASES)] - return DocumentCase( - kind=kind, - data=data, - mime="application/postscript", - description="semantic PostScript input", - extension=".ps", - ) - if kind == "pdf_coverage_sweep": - media_box, text = PDF_COVERAGE_CASES[case_index % len(PDF_COVERAGE_CASES)] - return DocumentCase( - kind=kind, - data=make_pdf(media_box=media_box, text=text), - mime="application/pdf", - description="coverage-oriented PDF input", - extension=".pdf", - ) - if kind == "pdf_semantic_sweep": - return DocumentCase( - kind=kind, - data=make_pdf_semantic(case_index), - mime="application/pdf", - description="semantic PDF input", - extension=".pdf", - ) - if kind == "image_coverage_sweep": - slots = synthesize_image_slots(case_index) - extension = ".png" if slots.image_format.startswith("png") else { - "ppm": ".ppm", - "pgm": ".pgm", - "pbm": ".pbm", - }[slots.image_format] - return DocumentCase( - kind=kind, - data=make_image( - image_format=slots.image_format, - width=slots.width, - height=slots.height, - channels=slots.channels, - ), - mime="image/png" if slots.image_format.startswith("png") else "image/x-portable-anymap", - description=f"SMT-filled coverage {slots.image_format} image", - extension=extension, - ) - if kind == "image_feedback_sweep": - instance = image_feedback_instance(case_index, target_id=target_id) - extension = ".png" if instance.image_format.startswith("png") else { - "ppm": ".ppm", - "pgm": ".pgm", - "pbm": ".pbm", - }[instance.image_format] - return DocumentCase( - kind=kind, - data=make_image( - image_format=instance.image_format, - width=instance.width, - height=instance.height, - channels=instance.channels, - maxval=instance.maxval, - payload_delta=instance.payload_delta, - comment_style=instance.comment_style, - png_interlace=instance.png_interlace, - ), - mime="image/png" if instance.image_format.startswith("png") else "image/x-portable-anymap", - description=f"feedback-driven image sweep via {instance.objective}/{instance.solved_by}", - extension=extension, - ) - if kind == "command_coverage_sweep": - data = COMMAND_COVERAGE_CASES[case_index % len(COMMAND_COVERAGE_CASES)] - return DocumentCase( - kind=kind, - data=data, - mime="application/vnd.cups-command", - description="coverage-oriented CUPS command input", - extension=".cmd", - ) - if kind == "command_semantic_sweep": - data = COMMAND_SEMANTIC_CASES[case_index % len(COMMAND_SEMANTIC_CASES)] - return DocumentCase( - kind=kind, - data=data, - mime="application/vnd.cups-command", - description="semantic CUPS command input", - extension=".cmd", - ) - if kind == "cups_raster_basic": - return DocumentCase( - kind=kind, - data=make_cups_raster(width=16, height=1, compression=0, num_colors=1), - mime="application/vnd.cups-raster", - description="minimal CUPS Raster page", - extension=".ras", - ) - if kind == "cups_raster_mode10": - return DocumentCase( - kind=kind, - data=make_cups_raster(width=11 + case_index, height=1, compression=10, num_colors=3), - mime="application/vnd.cups-raster", - description="Mode 10 RGB CUPS Raster page", - extension=".ras", - ) - if kind == "cups_raster_boundary_sweep": - width, height, compression, num_colors = RASTER_BOUNDARY_CASES[case_index % len(RASTER_BOUNDARY_CASES)] - return DocumentCase( - kind=kind, - data=make_cups_raster(width=width, height=height, compression=compression, num_colors=num_colors), - mime="application/vnd.cups-raster", - description="generic CUPS Raster boundary sweep", - extension=".ras", - ) - if kind == "cups_raster_general_sweep": - width, height, compression, num_colors = RASTER_GENERAL_CASES[case_index % len(RASTER_GENERAL_CASES)] - return DocumentCase( - kind=kind, - data=make_cups_raster(width=width, height=height, compression=compression, num_colors=num_colors), - mime="application/vnd.cups-raster", - description="general valid CUPS Raster sweep", - extension=".ras", - ) - if kind == "cups_raster_coverage_sweep": - slots = synthesize_cups_raster_slots(case_index) - return DocumentCase( - kind=kind, - data=make_cups_raster( - width=slots.width, - height=slots.height, - compression=slots.compression, - num_colors=slots.num_colors, - color_space=slots.color_space, - color_order=slots.color_order, - bits_per_pixel=slots.bits_per_pixel, - pages=slots.pages, - x_res=slots.x_res, - y_res=slots.y_res, - ), - mime="application/vnd.cups-raster", - description="SMT-filled coverage CUPS Raster sweep", - extension=".ras", - ) - if kind == "cups_raster_structural_sweep": - instance = cups_structural_instance(case_index) - return DocumentCase( - kind=kind, - data=make_cups_raster( - width=instance.get("width"), - height=instance.get("height"), - compression=instance.get("compression"), - num_colors=instance.get("num_colors"), - color_space=instance.get("color_space"), - color_order=instance.get("color_order"), - bits_per_pixel=instance.get("bits_per_pixel"), - pages=instance.get("pages"), - x_res=instance.get("x_res"), - y_res=instance.get("y_res"), - bytes_per_line=instance.get("bytes_per_line"), - row_count=instance.get("row_count"), - payload_rows=instance.get("payload_rows"), - ), - mime="application/vnd.cups-raster", - description=f"structural CUPS Raster sweep via {instance.objective}", - extension=".ras", - ) - if kind == "cups_raster_feedback_sweep": - instance = cups_feedback_instance(case_index) - return DocumentCase( - kind=kind, - data=make_cups_raster( - width=instance.get("width"), - height=instance.get("height"), - compression=instance.get("compression"), - num_colors=instance.get("num_colors"), - color_space=instance.get("color_space"), - color_order=instance.get("color_order"), - bits_per_pixel=instance.get("bits_per_pixel"), - pages=instance.get("pages"), - x_res=instance.get("x_res"), - y_res=instance.get("y_res"), - bytes_per_line=instance.get("bytes_per_line"), - row_count=instance.get("row_count"), - payload_rows=instance.get("payload_rows"), - ), - mime="application/vnd.cups-raster", - description=f"feedback-driven CUPS Raster sweep via {instance.objective}", - extension=".ras", - ) - if kind == "pwg_raster_resolution_stress": - stress = case_index > 0 - return DocumentCase( - kind=kind, - data=make_pwg_raster_resolution_stress(stress=stress), - mime="application/vnd.cups-pwg", - description="resolution stress PWG Raster" if stress else "benign small PWG Raster", - extension=".pwg", - ) - if kind == "pwg_raster_boundary_sweep": - width, height, bits_per_pixel, y_res = PWG_BOUNDARY_CASES[case_index % len(PWG_BOUNDARY_CASES)] - return DocumentCase( - kind=kind, - data=make_pwg_raster(width=width, height=height, bits_per_pixel=bits_per_pixel, y_res=y_res), - mime="application/vnd.cups-pwg", - description="generic PWG Raster boundary sweep", - extension=".pwg", - ) - if kind == "pwg_raster_general_sweep": - width, height, bits_per_pixel, y_res = PWG_GENERAL_CASES[case_index % len(PWG_GENERAL_CASES)] - return DocumentCase( - kind=kind, - data=make_pwg_raster(width=width, height=height, bits_per_pixel=bits_per_pixel, x_res=y_res, y_res=y_res), - mime="application/vnd.cups-pwg", - description="general valid PWG Raster sweep", - extension=".pwg", - ) - if kind == "pwg_raster_coverage_sweep": - slots = synthesize_pwg_raster_slots(case_index) - return DocumentCase( - kind=kind, - data=make_pwg_raster( - width=slots.width, - height=slots.height, - bits_per_pixel=slots.bits_per_pixel, - x_res=slots.x_res, - y_res=slots.y_res, - pages=slots.pages, - ), - mime="application/vnd.cups-pwg", - description="SMT-filled coverage PWG Raster sweep", - extension=".pwg", - ) - if kind == "pwg_raster_structural_sweep": - instance = pwg_structural_instance(case_index) - return DocumentCase( - kind=kind, - data=make_pwg_raster( - width=instance.get("width"), - height=instance.get("height"), - bits_per_pixel=instance.get("bits_per_pixel"), - x_res=instance.get("x_res"), - y_res=instance.get("y_res"), - pages=instance.get("pages"), - bytes_per_line=instance.get("bytes_per_line"), - row_count=instance.get("row_count"), - payload_rows=instance.get("payload_rows"), - ), - mime="application/vnd.cups-pwg", - description=f"structural PWG Raster sweep via {instance.objective}", - extension=".pwg", - ) - if kind == "pwg_raster_feedback_sweep": - instance = pwg_feedback_instance(case_index) - return DocumentCase( - kind=kind, - data=make_pwg_raster( - width=instance.get("width"), - height=instance.get("height"), - bits_per_pixel=instance.get("bits_per_pixel"), - x_res=instance.get("x_res"), - y_res=instance.get("y_res"), - pages=instance.get("pages"), - bytes_per_line=instance.get("bytes_per_line"), - row_count=instance.get("row_count"), - payload_rows=instance.get("payload_rows"), - ), - mime="application/vnd.cups-pwg", - description=f"feedback-driven PWG Raster sweep via {instance.objective}", - extension=".pwg", - ) - raise ValueError(f"unknown document kind: {kind}") - - -def make_pdf(*, media_box: tuple[int, int, int, int], text: str) -> bytes: - escaped = _escape_pdf_text(text) - x0, y0, x1, y1 = media_box - stream = f"BT /F1 12 Tf 24 24 Td ({escaped}) Tj ET\n".encode("ascii") - objects = [ - b"<< /Type /Catalog /Pages 2 0 R >>", - b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - ( - f"<< /Type /Page /Parent 2 0 R /MediaBox [{x0} {y0} {x1} {y1}] " - "/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>" - ).encode("ascii"), - b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", - b"<< /Length " + str(len(stream)).encode("ascii") + b" >>\nstream\n" + stream + b"endstream", - ] - output = bytearray(b"%PDF-1.4\n%\xE2\xE3\xCF\xD3\n") - offsets = [0] - for index, obj in enumerate(objects, start=1): - offsets.append(len(output)) - output.extend(f"{index} 0 obj\n".encode("ascii")) - output.extend(obj) - output.extend(b"\nendobj\n") - xref_offset = len(output) - output.extend(f"xref\n0 {len(objects) + 1}\n".encode("ascii")) - output.extend(b"0000000000 65535 f \n") - for offset in offsets[1:]: - output.extend(f"{offset:010d} 00000 n \n".encode("ascii")) - output.extend( - ( - f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" - f"startxref\n{xref_offset}\n%%EOF\n" - ).encode("ascii") - ) - return bytes(output) - - -def make_pdf_semantic(case_index: int) -> bytes: - media_boxes = [ - (0, 0, 200, 200), - (0, 0, 612, 792), - (0, 0, 144, 144), - (0, 0, 1008, 612), - ] - media_box = media_boxes[(case_index // 8) % len(media_boxes)] - variant = case_index % 8 - text = f"semantic pdf case {case_index}" - x0, y0, x1, y1 = media_box - - if variant == 0: - return make_pdf(media_box=media_box, text=text) - - if variant == 1: - page_one = _pdf_text_stream(f"{text} page one", x=24, y=48, size=12) - page_two = _pdf_text_stream(f"{text} rotated page two", x=24, y=48, size=10) - objects = [ - b"<< /Type /Catalog /Pages 2 0 R /PageMode /UseNone >>", - b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>", - ( - f"<< /Type /Page /Parent 2 0 R /MediaBox [{x0} {y0} {x1} {y1}] " - "/Resources << /Font << /F1 5 0 R >> >> /Contents 6 0 R >>" - ).encode("ascii"), - ( - f"<< /Type /Page /Parent 2 0 R /Rotate 90 /MediaBox [{x0} {y0} {x1} {y1}] " - "/Resources << /Font << /F1 5 0 R >> >> /Contents 7 0 R >>" - ).encode("ascii"), - b"<< /Type /Font /Subtype /Type1 /BaseFont /Courier >>", - _pdf_stream(page_one), - _pdf_stream(page_two), - ] - return _build_pdf(objects) - - if variant == 2: - stream = ( - b"q 0.8 0.8 0.8 rg 18 18 96 48 re f Q\n" - + _pdf_text_stream(f"{text} flate stream", x=24, y=34, size=9) - ) - compressed = zlib.compress(stream) - objects = [ - b"<< /Type /Catalog /Pages 2 0 R >>", - b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - ( - f"<< /Type /Page /Parent 2 0 R /MediaBox [{x0} {y0} {x1} {y1}] " - "/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>" - ).encode("ascii"), - b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", - _pdf_stream(compressed, extra=b"/Filter /FlateDecode"), - ] - return _build_pdf(objects) - - if variant == 3: - image_pixels = bytes([255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 0]) - content = b"q 32 0 0 32 24 24 cm /Im1 Do Q\n" - objects = [ - b"<< /Type /Catalog /Pages 2 0 R >>", - b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - ( - f"<< /Type /Page /Parent 2 0 R /MediaBox [{x0} {y0} {x1} {y1}] " - "/Resources << /XObject << /Im1 5 0 R >> >> /Contents 4 0 R >>" - ).encode("ascii"), - _pdf_stream(content), - ( - b"<< /Type /XObject /Subtype /Image /Width 2 /Height 2 " - b"/ColorSpace /DeviceRGB /BitsPerComponent 8 /Length " - + str(len(image_pixels)).encode("ascii") - + b" >>\nstream\n" - + image_pixels - + b"\nendstream" - ), - ] - return _build_pdf(objects) - - if variant == 4: - content = _pdf_text_stream(f"{text} annotation page", x=24, y=48, size=11) - objects = [ - b"<< /Type /Catalog /Pages 2 0 R /OpenAction [3 0 R /Fit] >>", - b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - ( - f"<< /Type /Page /Parent 2 0 R /MediaBox [{x0} {y0} {x1} {y1}] " - "/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R /Annots [6 0 R] >>" - ).encode("ascii"), - b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Oblique >>", - _pdf_stream(content), - b"<< /Type /Annot /Subtype /Link /Rect [20 20 100 60] /Border [0 0 0] /Dest [3 0 R /Fit] >>", - ] - return _build_pdf(objects) - - if variant == 5: - form_stream = ( - b"q 1 0 0 rg 0 0 36 18 re f Q\n" - b"BT /F1 8 Tf 2 6 Td (form) Tj ET\n" - ) - content = b"q 1 0 0 1 24 24 cm /Fm1 Do Q\n" - objects = [ - b"<< /Type /Catalog /Pages 2 0 R >>", - b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - ( - f"<< /Type /Page /Parent 2 0 R /MediaBox [{x0} {y0} {x1} {y1}] " - "/Resources << /Font << /F1 4 0 R >> /XObject << /Fm1 6 0 R >> >> /Contents 5 0 R >>" - ).encode("ascii"), - b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", - _pdf_stream(content), - ( - b"<< /Type /XObject /Subtype /Form /BBox [0 0 40 20] " - b"/Resources << /Font << /F1 4 0 R >> >> /Length " - + str(len(form_stream)).encode("ascii") - + b" >>\nstream\n" - + form_stream - + b"endstream" - ), - ] - return _build_pdf(objects) - - if variant == 6: - path_stream = ( - b"q 0 0 1 RG 2 w 20 20 m 60 20 l 60 60 l 20 60 l h S Q\n" - b"q 0.2 0.6 0.1 rg 72 24 36 18 re f Q\n" - + _pdf_text_stream(f"{text} paths", x=20, y=90, size=8) - ) - objects = [ - b"<< /Type /Catalog /Pages 2 0 R >>", - b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - ( - f"<< /Type /Page /Parent 2 0 R /MediaBox [{x0} {y0} {x1} {y1}] " - "/Resources << /Font << /F1 4 0 R >> /ProcSet [/PDF /Text] >> /Contents 5 0 R >>" - ).encode("ascii"), - b"<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman >>", - _pdf_stream(path_stream), - ] - return _build_pdf(objects) - - metadata = ( - f"<< /Title ({_escape_pdf_text(text)}) /Producer (parser-fuzzers) " - "/Creator (semantic sweep) >>" - ).encode("ascii") - content = _pdf_text_stream(f"{text} info object", x=24, y=48, size=12) - objects = [ - b"<< /Type /Catalog /Pages 2 0 R /ViewerPreferences << /FitWindow true >> >>", - b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - ( - f"<< /Type /Page /Parent 2 0 R /MediaBox [{x0} {y0} {x1} {y1}] " - "/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>" - ).encode("ascii"), - b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>", - _pdf_stream(content), - metadata, - ] - return _build_pdf(objects, info_obj=6) - - -def _escape_pdf_text(text: str) -> str: - return text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") - - -def _pdf_text_stream(text: str, *, x: int, y: int, size: int) -> bytes: - escaped = _escape_pdf_text(text) - return f"BT /F1 {size} Tf {x} {y} Td ({escaped}) Tj ET\n".encode("ascii") - - -def _pdf_stream(data: bytes, *, extra: bytes = b"") -> bytes: - if extra: - return b"<< /Length " + str(len(data)).encode("ascii") + b" " + extra + b" >>\nstream\n" + data + b"endstream" - return b"<< /Length " + str(len(data)).encode("ascii") + b" >>\nstream\n" + data + b"endstream" - - -def _build_pdf(objects: list[bytes], *, info_obj: int | None = None) -> bytes: - output = bytearray(b"%PDF-1.4\n%\xE2\xE3\xCF\xD3\n") - offsets = [0] - for index, obj in enumerate(objects, start=1): - offsets.append(len(output)) - output.extend(f"{index} 0 obj\n".encode("ascii")) - output.extend(obj) - output.extend(b"\nendobj\n") - xref_offset = len(output) - output.extend(f"xref\n0 {len(objects) + 1}\n".encode("ascii")) - output.extend(b"0000000000 65535 f \n") - for offset in offsets[1:]: - output.extend(f"{offset:010d} 00000 n \n".encode("ascii")) - trailer = f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R" - if info_obj is not None: - trailer += f" /Info {info_obj} 0 R" - trailer += f" >>\nstartxref\n{xref_offset}\n%%EOF\n" - output.extend(trailer.encode("ascii")) - return bytes(output) - - -def make_image( - *, - image_format: str, - width: int, - height: int, - channels: int, - maxval: int = 255, - payload_delta: int = 0, - comment_style: int = 0, - png_interlace: int = 0, -) -> bytes: - if image_format == "ppm": - header = _pnm_header("P6", width, height, maxval, comment_style) - pixels = _sized_payload(width * height * 3 * _sample_bytes(maxval), payload_delta, 17) - return header + pixels - if image_format == "pgm": - header = _pnm_header("P5", width, height, maxval, comment_style) - pixels = _sized_payload(width * height * _sample_bytes(maxval), payload_delta, 31) - return header + pixels - if image_format == "pbm": - row_bytes = (width + 7) // 8 - header = _pnm_header("P4", width, height, 1, comment_style) - pixels = _sized_payload(row_bytes * height, payload_delta, 0x5A) - return header + pixels - if image_format == "png_rgb": - return make_png( - width=width, - height=height, - color_type=2, - channels=channels, - payload_delta=payload_delta, - interlace=png_interlace, - ) - if image_format == "png_gray": - return make_png( - width=width, - height=height, - color_type=0, - channels=channels, - payload_delta=payload_delta, - interlace=png_interlace, - ) - if image_format == "png_rgba": - return make_png( - width=width, - height=height, - color_type=6, - channels=channels, - payload_delta=payload_delta, - interlace=png_interlace, - ) - raise ValueError(f"unknown image format: {image_format}") - - -def make_png( - *, - width: int, - height: int, - color_type: int, - channels: int, - payload_delta: int = 0, - interlace: int = 0, -) -> bytes: - bit_depth = 8 - raw = bytearray() - for y in range(height): - raw.append(0) # filter type: None - for x in range(width): - if channels == 1: - raw.append((x * 23 + y * 17) & 0xFF) - else: - raw.extend(((x * 31) & 0xFF, (y * 47) & 0xFF, ((x + y) * 19) & 0xFF)) - if channels == 4: - raw.append(((x * 11 + y * 13) & 0xFF) or 1) - raw = bytearray(_sized_payload(len(raw), payload_delta, 23, seed=bytes(raw))) - ihdr = struct.pack(">IIBBBBB", width, height, bit_depth, color_type, 0, 0, 1 if interlace else 0) - compressed = zlib.compress(bytes(raw)) - return ( - b"\x89PNG\r\n\x1a\n" - + _png_chunk(b"IHDR", ihdr) - + _png_chunk(b"IDAT", compressed) - + _png_chunk(b"IEND", b"") - ) - - -def _pnm_header(magic: str, width: int, height: int, maxval: int, comment_style: int) -> bytes: - if magic == "P4": - if comment_style == 1: - return f"{magic}\n# smt image feedback\n{width} {height}\n".encode("ascii") - if comment_style == 2: - return f"{magic}\r\n{width}\t{height}\r\n".encode("ascii") - if comment_style == 3: - return f"{magic} # inline\n{width} {height}\n".encode("ascii") - return f"{magic}\n{width} {height}\n".encode("ascii") - bounded_maxval = max(1, min(65535, maxval)) - if comment_style == 1: - return f"{magic}\n# smt image feedback\n{width} {height}\n{bounded_maxval}\n".encode("ascii") - if comment_style == 2: - return f"{magic}\r\n{width}\t{height}\r\n{bounded_maxval}\r\n".encode("ascii") - if comment_style == 3: - return f"{magic} # inline\n{width} {height}\n{bounded_maxval}\n".encode("ascii") - return f"{magic}\n{width} {height}\n{bounded_maxval}\n".encode("ascii") - - -def _sample_bytes(maxval: int) -> int: - return 2 if maxval > 255 else 1 - - -def _sized_payload(size: int, delta: int, salt: int, seed: bytes | None = None) -> bytes: - target_size = max(0, size + delta) - if seed is not None: - if target_size <= len(seed): - return seed[:target_size] - extra = bytes(((index * salt + 3) & 0xFF) for index in range(target_size - len(seed))) - return seed + extra - return bytes(((index * salt + 1) & 0xFF) for index in range(target_size)) - - -def _png_chunk(chunk_type: bytes, payload: bytes) -> bytes: - crc = zlib.crc32(chunk_type + payload) & 0xFFFFFFFF - return struct.pack(">I", len(payload)) + chunk_type + payload + struct.pack(">I", crc) - - -def make_cups_raster( - *, - width: int, - height: int, - compression: int, - num_colors: int, - color_space: int | None = None, - color_order: int = 0, - bits_per_pixel: int | None = None, - pages: int = 1, - x_res: int = 300, - y_res: int = 300, - bytes_per_line: int | None = None, - row_count: int | None = None, - payload_rows: int | None = None, -) -> bytes: - bits_per_color = 8 - bits_per_pixel = bits_per_pixel or max(1, bits_per_color * num_colors) - raw_bpl = (width * bits_per_pixel + 7) // 8 - cups_bytes_per_line = max(1, bytes_per_line if bytes_per_line is not None else ((raw_bpl + 7) // 8) * 8) - raster_rows = max(1, payload_rows if payload_rows is not None else height) - header = bytearray(HEADER_SIZE) - _write_cstr(header, OFF_MEDIA_CLASS, "PwgRaster", 64) - _write_cstr(header, OFF_CUPS_PAGE_SIZE_NAME, "Letter", 64) - _pack_u32(header, OFF_HW_RESOLUTION, x_res) - _pack_u32(header, OFF_HW_RESOLUTION + 4, y_res) - _pack_page_geometry(header) - _pack_u32(header, OFF_CUPS_WIDTH, width) - _pack_u32(header, OFF_CUPS_HEIGHT, height) - _pack_u32(header, OFF_CUPS_BITS_PER_COLOR, bits_per_color) - _pack_u32(header, OFF_CUPS_BITS_PER_PIXEL, bits_per_pixel) - _pack_u32(header, OFF_CUPS_BYTES_PER_LINE, cups_bytes_per_line) - _pack_u32(header, OFF_CUPS_COLOR_ORDER, color_order) - _pack_u32(header, OFF_CUPS_COLOR_SPACE, color_space if color_space is not None else (CUPS_CSPACE_RGB if num_colors == 3 else CUPS_CSPACE_K)) - _pack_u32(header, OFF_CUPS_COMPRESSION, compression) - _pack_u32(header, OFF_CUPS_ROW_COUNT, row_count if row_count is not None else height) - _pack_u32(header, OFF_CUPS_NUM_COLORS, num_colors) - pixel = bytes((i & 0xFF) or 1 for i in range(cups_bytes_per_line)) - page = bytes(header) + pixel * raster_rows - return SYNC_CUPS_RASTER_V3 + page * max(1, pages) - - -def make_pwg_raster_resolution_stress(*, stress: bool) -> bytes: - width = 5 if stress else 8 - height = 6 if stress else 1 - bits_per_pixel = 16 - y_res = 2147483648 if stress else 300 - return make_pwg_raster(width=width, height=height, bits_per_pixel=bits_per_pixel, x_res=300, y_res=y_res) - - -def make_pwg_raster( - *, - width: int, - height: int, - bits_per_pixel: int, - x_res: int = 300, - y_res: int, - pages: int = 1, - bytes_per_line: int | None = None, - row_count: int | None = None, - payload_rows: int | None = None, -) -> bytes: - cups_bytes_per_line = max(1, bytes_per_line if bytes_per_line is not None else (width * bits_per_pixel + 7) // 8) - raster_rows = max(1, payload_rows if payload_rows is not None else height) - header = bytearray(HEADER_SIZE) - _write_cstr(header, OFF_MEDIA_CLASS, "PwgRaster", 64) - _write_cstr(header, OFF_MEDIA_TYPE, "PLAIN", 64) - _write_cstr(header, OFF_OUTPUT_TYPE, "Automatic", 64) - _write_cstr(header, OFF_CUPS_PAGE_SIZE_NAME, "Letter", 64) - _pack_u32(header, OFF_HW_RESOLUTION, x_res) - _pack_u32(header, OFF_HW_RESOLUTION + 4, y_res) - _pack_page_geometry(header) - _pack_u32(header, OFF_CUPS_WIDTH, width) - _pack_u32(header, OFF_CUPS_HEIGHT, height) - _pack_u32(header, OFF_CUPS_BITS_PER_COLOR, 8) - _pack_u32(header, OFF_CUPS_BITS_PER_PIXEL, bits_per_pixel) - _pack_u32(header, OFF_CUPS_BYTES_PER_LINE, cups_bytes_per_line) - _pack_u32(header, OFF_CUPS_COLOR_ORDER, 0) - _pack_u32(header, OFF_CUPS_COLOR_SPACE, CUPS_CSPACE_SW) - _pack_u32(header, OFF_CUPS_COMPRESSION, 0) - _pack_u32(header, OFF_CUPS_ROW_COUNT, row_count if row_count is not None else height) - _pack_u32(header, OFF_CUPS_NUM_COLORS, 1) - row = bytes([0xFF if y_res >= 2147483648 else 0x80]) * cups_bytes_per_line - page = bytes(header) + row * raster_rows - return SYNC_PWG_RASTER + page * max(1, pages) - - -def _pack_page_geometry(buffer: bytearray) -> None: - _pack_u32(buffer, OFF_IMAGING_BBOX, 18) - _pack_u32(buffer, OFF_IMAGING_BBOX + 4, 36) - _pack_u32(buffer, OFF_IMAGING_BBOX + 8, 594) - _pack_u32(buffer, OFF_IMAGING_BBOX + 12, 756) - _pack_u32(buffer, OFF_MARGINS, 18) - _pack_u32(buffer, OFF_MARGINS + 4, 36) - _pack_u32(buffer, OFF_PAGE_SIZE, 612) - _pack_u32(buffer, OFF_PAGE_SIZE + 4, 792) - _pack_float(buffer, OFF_CUPS_PAGE_SIZE, 612.0) - _pack_float(buffer, OFF_CUPS_PAGE_SIZE + 4, 792.0) - _pack_float(buffer, OFF_CUPS_IMAGING_BBOX, 18.0) - _pack_float(buffer, OFF_CUPS_IMAGING_BBOX + 4, 36.0) - _pack_float(buffer, OFF_CUPS_IMAGING_BBOX + 8, 594.0) - _pack_float(buffer, OFF_CUPS_IMAGING_BBOX + 12, 756.0) - - -def _pack_u32(buffer: bytearray, offset: int, value: int) -> None: - struct.pack_into(" None: - struct.pack_into(" None: - encoded = value.encode("ascii")[: size - 1] - buffer[offset : offset + len(encoded)] = encoded diff --git a/parser-fuzzers/src/parser_fuzzers/runner/multitarget_runner.py b/parser-fuzzers/src/parser_fuzzers/runner/multitarget_runner.py deleted file mode 100644 index 8aaba11..0000000 --- a/parser-fuzzers/src/parser_fuzzers/runner/multitarget_runner.py +++ /dev/null @@ -1,2655 +0,0 @@ -from __future__ import annotations - -import concurrent.futures -import hashlib -import json -import math -import os -import re -import shutil -import shlex -import struct -import subprocess -import time -from dataclasses import asdict, dataclass, field -from functools import lru_cache -from pathlib import Path -from typing import Any - -import yaml - -from parser_fuzzers.crash_dedup import compute_crash_signature -from parser_fuzzers.semantic_shapes import ( - build_planned_shape, - build_result_shape_bundle, - compact_shape_record, - semantic_runtime_key, - shape_feature_tokens, -) -from parser_fuzzers.image_templates import IMAGE_FEEDBACK_PERIOD, image_feedback_instance -from parser_fuzzers.document_harness import ( - COMMAND_COVERAGE_CASES, - COMMAND_SEMANTIC_CASES, - OFF_CUPS_BITS_PER_PIXEL, - OFF_CUPS_BYTES_PER_LINE, - OFF_CUPS_COLOR_ORDER, - OFF_CUPS_COLOR_SPACE, - OFF_CUPS_COMPRESSION, - OFF_CUPS_HEIGHT, - OFF_CUPS_ROW_COUNT, - OFF_CUPS_WIDTH, - OFF_HW_RESOLUTION, - PDF_COVERAGE_CASES, - PDF_SEMANTIC_PERIOD, - POSTSCRIPT_COVERAGE_CASES, - POSTSCRIPT_SEMANTIC_CASES, - PWG_BOUNDARY_CASES, - PWG_COVERAGE_CASES, - PWG_GENERAL_CASES, - RASTER_BOUNDARY_CASES, - RASTER_COVERAGE_CASES, - RASTER_GENERAL_CASES, - TEXT_COVERAGE_CASES, - TEXT_SEMANTIC_CASES, - IMAGE_COVERAGE_CASES, - make_document, -) -from parser_fuzzers.ppd_templates import ( - COVERAGE_RESOLUTIONS, - FILTER_COVERAGE_PPDS, - GENERAL_RESOLUTIONS, - GENERAL_STRING_VALUES, - GENERIC_RESOLUTIONS, - GENERIC_STRING_VALUES, - PAGE_SIZES, - make_ppd, -) -from parser_fuzzers.structured_templates import ( - CUPS_FEEDBACK_PERIOD, - CUPS_STRUCTURAL_PERIOD, - PWG_FEEDBACK_PERIOD, - PWG_STRUCTURAL_PERIOD, -) -from parser_fuzzers.template_synth import ( - CUPS_RASTER_SYNTH_PERIOD, - IMAGE_SYNTH_PERIOD, - PPD_SYNTH_PERIOD, - PWG_RASTER_SYNTH_PERIOD, -) - - -RASTERTOESCPX_DOTROWSTEP_ZERO_MODS = {3, 4, 6, 8, 9, 11} -COVERAGE_OPTION_PERIOD = math.lcm(4, 8, 12, 15, len(PAGE_SIZES)) -DOCUMENT_PERIODS = { - "text": 1, - "text_coverage_sweep": len(TEXT_COVERAGE_CASES), - "text_semantic_sweep": len(TEXT_SEMANTIC_CASES), - "postscript": 1, - "postscript_coverage_sweep": len(POSTSCRIPT_COVERAGE_CASES), - "postscript_semantic_sweep": len(POSTSCRIPT_SEMANTIC_CASES), - "pdf_coverage_sweep": len(PDF_COVERAGE_CASES), - "pdf_semantic_sweep": PDF_SEMANTIC_PERIOD, - "image_coverage_sweep": IMAGE_SYNTH_PERIOD, - "image_feedback_sweep": IMAGE_FEEDBACK_PERIOD, - "command_coverage_sweep": len(COMMAND_COVERAGE_CASES), - "command_semantic_sweep": len(COMMAND_SEMANTIC_CASES), - "cups_raster_basic": 1, - "cups_raster_mode10": COVERAGE_OPTION_PERIOD, - "cups_raster_boundary_sweep": len(RASTER_BOUNDARY_CASES), - "cups_raster_general_sweep": len(RASTER_GENERAL_CASES), - "cups_raster_coverage_sweep": CUPS_RASTER_SYNTH_PERIOD, - "cups_raster_structural_sweep": CUPS_STRUCTURAL_PERIOD, - "cups_raster_feedback_sweep": CUPS_FEEDBACK_PERIOD, - "pwg_raster_resolution_stress": 2, - "pwg_raster_boundary_sweep": len(PWG_BOUNDARY_CASES), - "pwg_raster_general_sweep": len(PWG_GENERAL_CASES), - "pwg_raster_coverage_sweep": PWG_RASTER_SYNTH_PERIOD, - "pwg_raster_structural_sweep": PWG_STRUCTURAL_PERIOD, - "pwg_raster_feedback_sweep": PWG_FEEDBACK_PERIOD, -} -PPD_PERIODS = { - "rastertopclx": len(GENERAL_STRING_VALUES), - "rastertopclx_string_sweep": len(GENERIC_STRING_VALUES), - "rastertopclx_general_strings": len(GENERAL_STRING_VALUES), - "rastertopclx_plain": 1, - "rastertoescpx_single_pagesize": 1, - "rastertoescpx_size_sweep": 1, - "raster_coverage_options": PPD_SYNTH_PERIOD, - "rastertops_plain": 1, - "rastertopwg_plain": 1, - "pwgtopdf_plain": 1, - "pwgtopdf_coverage_options": PPD_SYNTH_PERIOD, - "pwgtoraster_1dpi": 1, - "pwg_resolution_sweep": len(GENERIC_RESOLUTIONS), - "pwg_resolution_general": len(GENERAL_RESOLUTIONS), - "pwg_resolution_coverage": PPD_SYNTH_PERIOD, -} -for _filter_ppd_kind in FILTER_COVERAGE_PPDS: - PPD_PERIODS[_filter_ppd_kind] = PPD_SYNTH_PERIOD - -JOB_OPTION_PERIOD = PPD_SYNTH_PERIOD -JOB_COLOR_MODELS = ["Gray", "RGB", "CMYK", "Black"] -JOB_PRINT_QUALITIES = ["Draft", "Normal", "High", "Photo"] -JOB_MEDIA_TYPES = ["Plain", "Glossy", "Transparency", "Envelope"] -JOB_DUPLEX_MODES = ["None", "DuplexNoTumble", "DuplexTumble"] -JOB_SCALING_VALUES = ["100", "25", "50", "150", "200"] -JOB_NATURAL_SCALING_VALUES = ["100", "25", "50", "150", "200"] -JOB_ORIENTATIONS = ["3", "4", "5", "6"] -JOB_PRINT_SCALING = ["auto", "fit", "fill", "none"] - - -@dataclass -class TargetDiscoveryStats: - submitted: int = 0 - completed: int = 0 - skipped: int = 0 - retained_cases: int = 0 - new_features: int = 0 - crashes: int = 0 - unique_crashes: int = 0 - repeat_crashes: int = 0 - timeouts: int = 0 - runtime_suppressed: int = 0 - - -@dataclass -class RunCounters: - cases: int = 0 - crashes: int = 0 - reached: int = 0 - valid_ppds: int = 0 - timeouts: int = 0 - oracle_counts: dict[str, int] = field(default_factory=dict) - - -@dataclass -class DiscoveryState: - seen_features: set[str] = field(default_factory=set) - seen_crash_signatures: set[str] = field(default_factory=set) - crash_shape_counts: dict[str, int] = field(default_factory=dict) - crash_hazard_counts: dict[str, int] = field(default_factory=dict) - crash_family_counts: dict[str, int] = field(default_factory=dict) - semantic_crash_counts: dict[str, int] = field(default_factory=dict) - suppressed_case_shapes: dict[str, str] = field(default_factory=dict) - suppressed_case_hazards: dict[str, str] = field(default_factory=dict) - suppressed_case_families: dict[str, str] = field(default_factory=dict) - suppressed_semantic_shapes: dict[str, str] = field(default_factory=dict) - target_stats: dict[str, TargetDiscoveryStats] = field(default_factory=dict) - scheduler_credit: dict[str, float] = field(default_factory=dict) - scheduler_cursor: int = 0 - runtime_skip_enabled: bool = False - crash_skip_after: int = 1 - hazard_skip_after: int = 0 - semantic_skip_after: int = 0 - generalized_skip_enabled: bool = False - family_skip_after: int = 32 - retained_cases: int = 0 - unique_crashes: int = 0 - repeat_crashes: int = 0 - completed_cases: int = 0 - - -@dataclass(frozen=True) -class TargetProfile: - id: str - description: str - ppd_kind: str - document_kind: str - executor: str - input_mime: str - output_mime: str - expected_filters: list[str] - cases: int - oracle: str - filter_binary: str - - -@dataclass(frozen=True) -class CaseResult: - target_id: str - case_id: int - target_description: str - ppd_kind: str - document_kind: str - document_description: str - work_dir: str - ppd_path: str - document_path: str - command_path: str - command_line: str - job_options: str - env_overrides: dict[str, str] - compare_trace_path: str - stdout_path: str - stderr_path: str - meta_path: str - cupstestppd_ok: bool - filters: list[str] - reached_expected_filter: bool - returncode: int | None - timed_out: bool - crashed: bool - oracle: str - duration_ms: float - stderr_tail: list[str] - - -@dataclass(frozen=True) -class MultiTargetSummary: - run_id: str - work_dir: str - config_path: str - duration_budget_sec: int | None - elapsed_sec: float - workers: int - timeout_sec: int - max_run_bytes: int - run_dir_bytes: int - stop_reason: str - targets: int - cases: int - crashes: int - reached: int - valid_ppds: int - timeouts: int - skipped: int - pruned_cases: int - skip_counts: dict[str, int] - scheduler: str - min_target_share: float - max_target_share: float - runtime_skip_enabled: bool - auto_skip_state_enabled: bool - auto_skip_search_root: str - runtime_suppressed_shapes: int - seeded_runtime_suppressed_shapes: int - runtime_suppressed_hazards: int - seeded_runtime_suppressed_hazards: int - runtime_suppressed_families: int - seeded_runtime_suppressed_families: int - runtime_suppressed_semantic_shapes: int - seeded_runtime_suppressed_semantic_shapes: int - generalized_skip_enabled: bool - family_skip_after: int - hazard_skip_after: int - semantic_skip_after: int - skip_probe_rate: float - skip_only_stop_after: int - stagnation_stop_after_sec: int - template_cycle_epochs: int - llvm_profile_enabled: bool - llvm_profile_files: int - seed_skip_state_path: str - target_stats: dict[str, dict[str, int]] - retained_cases: int - coverage_features: int - unique_crashes: int - repeat_crashes: int - oracle_counts: dict[str, int] - results: list[CaseResult] - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - def concise_dict(self) -> dict[str, Any]: - return { - "run_id": self.run_id, - "work_dir": self.work_dir, - "config_path": self.config_path, - "duration_budget_sec": self.duration_budget_sec, - "elapsed_sec": self.elapsed_sec, - "workers": self.workers, - "timeout_sec": self.timeout_sec, - "max_run_bytes": self.max_run_bytes, - "run_dir_bytes": self.run_dir_bytes, - "stop_reason": self.stop_reason, - "targets": self.targets, - "cases": self.cases, - "crashes": self.crashes, - "reached": self.reached, - "valid_ppds": self.valid_ppds, - "timeouts": self.timeouts, - "skipped": self.skipped, - "pruned_cases": self.pruned_cases, - "skip_counts": self.skip_counts, - "scheduler": self.scheduler, - "min_target_share": self.min_target_share, - "max_target_share": self.max_target_share, - "runtime_skip_enabled": self.runtime_skip_enabled, - "auto_skip_state_enabled": self.auto_skip_state_enabled, - "auto_skip_search_root": self.auto_skip_search_root, - "runtime_suppressed_shapes": self.runtime_suppressed_shapes, - "seeded_runtime_suppressed_shapes": self.seeded_runtime_suppressed_shapes, - "runtime_suppressed_hazards": self.runtime_suppressed_hazards, - "seeded_runtime_suppressed_hazards": self.seeded_runtime_suppressed_hazards, - "runtime_suppressed_families": self.runtime_suppressed_families, - "seeded_runtime_suppressed_families": self.seeded_runtime_suppressed_families, - "runtime_suppressed_semantic_shapes": self.runtime_suppressed_semantic_shapes, - "seeded_runtime_suppressed_semantic_shapes": self.seeded_runtime_suppressed_semantic_shapes, - "generalized_skip_enabled": self.generalized_skip_enabled, - "family_skip_after": self.family_skip_after, - "hazard_skip_after": self.hazard_skip_after, - "semantic_skip_after": self.semantic_skip_after, - "skip_probe_rate": self.skip_probe_rate, - "skip_only_stop_after": self.skip_only_stop_after, - "stagnation_stop_after_sec": self.stagnation_stop_after_sec, - "template_cycle_epochs": self.template_cycle_epochs, - "llvm_profile_enabled": self.llvm_profile_enabled, - "llvm_profile_files": self.llvm_profile_files, - "seed_skip_state_path": self.seed_skip_state_path, - "target_stats": self.target_stats, - "retained_cases": self.retained_cases, - "coverage_features": self.coverage_features, - "unique_crashes": self.unique_crashes, - "repeat_crashes": self.repeat_crashes, - "oracle_counts": self.oracle_counts, - } - - -def run_multitarget_monitor( - *, - config_path: str | Path, - work_root: str | Path, - workers: int, - cases_per_target: int | None, - duration_sec: int | None = None, - timeout_sec: int = 15, - max_run_gb: float | None = None, - run_command: str = "", - capture_stdout: bool = True, - discovery_mode: str = "crash", - scheduler: str = "round-robin", - min_target_share: float = 0.0, - max_target_share: float = 1.0, - runtime_skip: bool = False, - crash_skip_after: int = 1, - prune_uninteresting: bool = False, - seed_skip_state_path: str | Path | None = None, - auto_skip_state: bool = False, - auto_skip_search_root: str | Path | None = None, - generalized_skip: bool = False, - family_skip_after: int = 32, - skip_probe_rate: float = 0.0, - skip_only_stop_after: int = 0, - stagnation_stop_after_sec: int = 0, - summary_mode: str = "full", - filter_root: str | Path | None = None, -) -> MultiTargetSummary: - profiles = load_profiles(config_path, filter_root=filter_root) - run_id = time.strftime("%Y%m%d-%H%M%S") - root = Path(work_root) / run_id - root.mkdir(parents=True, exist_ok=True) - resolved_seed_skip_state_path = Path(seed_skip_state_path) if seed_skip_state_path else None - resolved_auto_skip_search_root = Path(auto_skip_search_root) if auto_skip_search_root else Path(work_root).parent - if auto_skip_state and resolved_seed_skip_state_path is None: - if _legacy_skip_state_enabled(): - resolved_seed_skip_state_path = find_latest_runtime_skip_state(resolved_auto_skip_search_root) - else: - resolved_seed_skip_state_path = find_latest_semantic_skip_state( - resolved_auto_skip_search_root, - [profile.id for profile in profiles], - ) - started = time.monotonic() - max_run_bytes = int(max_run_gb * 1024 * 1024 * 1024) if max_run_gb and max_run_gb > 0 else 0 - normalized_skip_probe_rate = max(0.0, min(1.0, skip_probe_rate)) - normalized_skip_only_stop_after = max(0, skip_only_stop_after) - normalized_stagnation_stop_after_sec = max(0, stagnation_stop_after_sec) - template_cycle_epochs = _template_cycle_epochs() - llvm_profile_enabled = _llvm_profiles_enabled() - hazard_skip_after = _hazard_skip_after() - semantic_skip_after = _semantic_skip_after() - normalized_min_target_share = _normalize_min_target_share(min_target_share, len(profiles)) - normalized_max_target_share = _normalize_max_target_share( - max_target_share, - len(profiles), - normalized_min_target_share, - ) - normalized_summary_mode = "concise" if summary_mode == "concise" else "full" - stop_reason = "duration" if duration_sec is not None else "case-budget" - last_size_check = 0.0 - last_run_dir_bytes = 0 - results: list[CaseResult] = [] - keep_results = normalized_summary_mode == "full" - counters = RunCounters() - skipped = 0 - pruned_cases = 0 - skip_counts: dict[str, int] = {} - consecutive_skips_without_submission = 0 - last_novelty_time = started - discovery_state = DiscoveryState( - runtime_skip_enabled=runtime_skip, - crash_skip_after=max(1, crash_skip_after), - hazard_skip_after=hazard_skip_after, - semantic_skip_after=semantic_skip_after, - generalized_skip_enabled=generalized_skip, - family_skip_after=max(1, family_skip_after), - ) - _initialize_target_stats(discovery_state, profiles) - seeded_runtime_suppressed_shapes = 0 - seeded_runtime_suppressed_hazards = 0 - seeded_runtime_suppressed_families = 0 - seeded_runtime_suppressed_semantic_shapes = 0 - if resolved_seed_skip_state_path: - load_runtime_skip_state( - discovery_state, - resolved_seed_skip_state_path, - generalized_skip=generalized_skip, - family_skip_after=max(1, family_skip_after), - ) - seeded_runtime_suppressed_shapes = len(discovery_state.suppressed_case_shapes) - seeded_runtime_suppressed_hazards = len(discovery_state.suppressed_case_hazards) - seeded_runtime_suppressed_families = len(discovery_state.suppressed_case_families) - seeded_runtime_suppressed_semantic_shapes = len(discovery_state.suppressed_semantic_shapes) - manifest = { - "run_id": run_id, - "work_dir": str(root), - "config_path": str(config_path), - "filter_root": str(filter_root) if filter_root else "", - "workers": workers, - "cases_per_target": cases_per_target, - "duration_sec": duration_sec, - "timeout_sec": timeout_sec, - "max_run_gb": max_run_gb or 0, - "max_run_bytes": max_run_bytes, - "run_command": run_command, - "capture_stdout": capture_stdout, - "discovery_mode": discovery_mode, - "scheduler": scheduler, - "min_target_share": normalized_min_target_share, - "max_target_share": normalized_max_target_share, - "runtime_skip": runtime_skip, - "crash_skip_after": max(1, crash_skip_after), - "auto_skip_state": auto_skip_state, - "auto_skip_search_root": str(resolved_auto_skip_search_root), - "generalized_skip": generalized_skip, - "family_skip_after": max(1, family_skip_after), - "hazard_skip_after": hazard_skip_after, - "semantic_skip_after": semantic_skip_after, - "skip_probe_rate": normalized_skip_probe_rate, - "skip_only_stop_after": normalized_skip_only_stop_after, - "stagnation_stop_after_sec": normalized_stagnation_stop_after_sec, - "template_cycle_epochs": template_cycle_epochs, - "llvm_profile_enabled": llvm_profile_enabled, - "summary_mode": normalized_summary_mode, - "seed_skip_state_path": str(resolved_seed_skip_state_path) if resolved_seed_skip_state_path else "", - "requested_seed_skip_state_path": str(seed_skip_state_path) if seed_skip_state_path else "", - "seeded_runtime_suppressed_shapes": seeded_runtime_suppressed_shapes, - "seeded_runtime_suppressed_hazards": seeded_runtime_suppressed_hazards, - "seeded_runtime_suppressed_families": seeded_runtime_suppressed_families, - "seeded_runtime_suppressed_semantic_shapes": seeded_runtime_suppressed_semantic_shapes, - "prune_uninteresting": prune_uninteresting, - "guidance_policy": ( - "general-format-aware; no private reproducer seeds; no issue-specific payload dictionary; " - "valid PPD/raster templates with broad parser boundary values" - ), - "targets": [asdict(profile) for profile in profiles], - } - (root / "run_manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") - timeline_path = root / "timeline.jsonl" - commands_path = root / "commands.txt" - (root / "run.log").write_text( - "\n".join( - [ - f"run_id={run_id}", - f"config_path={config_path}", - f"filter_root={filter_root or ''}", - f"workers={workers}", - f"cases_per_target={cases_per_target}", - f"duration_sec={duration_sec}", - f"timeout_sec={timeout_sec}", - f"max_run_gb={max_run_gb or 0}", - f"capture_stdout={capture_stdout}", - f"discovery_mode={discovery_mode}", - f"scheduler={scheduler}", - f"min_target_share={normalized_min_target_share:g}", - f"max_target_share={normalized_max_target_share:g}", - f"runtime_skip={runtime_skip}", - f"crash_skip_after={max(1, crash_skip_after)}", - f"auto_skip_state={auto_skip_state}", - f"auto_skip_search_root={resolved_auto_skip_search_root}", - f"generalized_skip={generalized_skip}", - f"family_skip_after={max(1, family_skip_after)}", - f"hazard_skip_after={hazard_skip_after}", - f"semantic_skip_after={semantic_skip_after}", - f"skip_probe_rate={normalized_skip_probe_rate:g}", - f"skip_only_stop_after={normalized_skip_only_stop_after}", - f"stagnation_stop_after_sec={normalized_stagnation_stop_after_sec}", - f"template_cycle_epochs={template_cycle_epochs}", - f"llvm_profile_enabled={llvm_profile_enabled}", - f"summary_mode={normalized_summary_mode}", - f"seed_skip_state_path={resolved_seed_skip_state_path or ''}", - f"prune_uninteresting={prune_uninteresting}", - f"run_command={run_command}", - "guidance=general-format-aware", - "", - ] - ), - encoding="utf-8", - ) - if discovery_mode == "coverage" and duration_sec is None and not _skip_warm_template_cache(): - _warm_template_synth_cache(profiles) - - with timeline_path.open("a", encoding="utf-8") as timeline, commands_path.open("a", encoding="utf-8") as commands: - if duration_sec is None: - jobs: list[tuple[TargetProfile, int, Path]] = [] - for profile in profiles: - count = cases_per_target if cases_per_target is not None else profile.cases - for case_id in range(count): - if _run_dir_limit_reached(root, max_run_bytes): - stop_reason = "max-run-gb" - break - case_dir = root / profile.id / f"case-{case_id:04d}" - skip_reason = _combined_skip_reason( - profile, - case_id, - discovery_mode, - discovery_state, - skip_probe_rate=normalized_skip_probe_rate, - ) - if skip_reason: - skipped += 1 - skip_counts[skip_reason] = skip_counts.get(skip_reason, 0) + 1 - _record_skip(discovery_state, profile, skip_reason) - _write_skip_record(profile, case_id, case_dir, skip_reason, timeline, commands) - consecutive_skips_without_submission += 1 - if ( - normalized_skip_only_stop_after - and consecutive_skips_without_submission >= normalized_skip_only_stop_after - ): - stop_reason = "skip-only" - break - continue - consecutive_skips_without_submission = 0 - jobs.append((profile, case_id, case_dir)) - if stop_reason in {"max-run-gb", "skip-only"}: - break - with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: - futures = [ - executor.submit(run_case, profile, case_id, case_dir, timeout_sec, capture_stdout) - for profile, case_id, case_dir in jobs - ] - for profile, _, _ in jobs: - _record_submission(discovery_state, profile) - for future in concurrent.futures.as_completed(futures): - result = future.result() - if keep_results: - results.append(result) - _record_result_summary(counters, result) - extra = _process_discovery_result(result, root, discovery_mode, discovery_state) - _write_run_records(result, timeline, commands, extra) - if prune_uninteresting and _prune_case_artifacts(result, extra): - pruned_cases += 1 - else: - deadline = started + duration_sec - next_case_ids = {profile.id: 0 for profile in profiles} - inflight_counts = {profile.id: 0 for profile in profiles} - with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: - futures: dict[concurrent.futures.Future[CaseResult], str] = {} - while time.monotonic() < deadline or futures: - skip_streak = 0 - while time.monotonic() < deadline and len(futures) < workers: - if stop_reason in {"max-run-gb", "skip-only", "coverage-stagnation"}: - break - now = time.monotonic() - if _coverage_stagnated( - discovery_mode, - normalized_stagnation_stop_after_sec, - discovery_state, - last_novelty_time, - now, - ): - stop_reason = "coverage-stagnation" - break - if max_run_bytes and now - last_size_check >= 5.0: - last_run_dir_bytes = _run_dir_size_bytes(root) - last_size_check = now - if last_run_dir_bytes >= max_run_bytes: - stop_reason = "max-run-gb" - break - profile = _choose_next_profile( - profiles, - discovery_state, - inflight_counts, - scheduler, - min_target_share=normalized_min_target_share, - max_target_share=normalized_max_target_share, - ) - case_id = next_case_ids[profile.id] - next_case_ids[profile.id] += 1 - case_dir = root / profile.id / f"case-{case_id:04d}" - skip_reason = _combined_skip_reason( - profile, - case_id, - discovery_mode, - discovery_state, - skip_probe_rate=normalized_skip_probe_rate, - ) - if skip_reason: - skipped += 1 - skip_counts[skip_reason] = skip_counts.get(skip_reason, 0) + 1 - _record_skip(discovery_state, profile, skip_reason) - _write_skip_record(profile, case_id, case_dir, skip_reason, timeline, commands) - consecutive_skips_without_submission += 1 - if ( - normalized_skip_only_stop_after - and consecutive_skips_without_submission >= normalized_skip_only_stop_after - ): - stop_reason = "skip-only" - break - skip_streak += 1 - if skip_streak >= max(100, workers * len(profiles) * 4): - break - continue - skip_streak = 0 - consecutive_skips_without_submission = 0 - _record_submission(discovery_state, profile) - inflight_counts[profile.id] += 1 - future = executor.submit(run_case, profile, case_id, case_dir, timeout_sec, capture_stdout) - futures[future] = profile.id - if stop_reason in {"max-run-gb", "skip-only", "coverage-stagnation"} and not futures: - break - if not futures: - if time.monotonic() < deadline: - time.sleep(0.05) - continue - break - done, remaining = concurrent.futures.wait( - set(futures), - timeout=max(0.1, min(1.0, deadline - time.monotonic())), - return_when=concurrent.futures.FIRST_COMPLETED, - ) - remaining_map = {future: futures[future] for future in remaining} - for future in done: - target_id = futures.get(future, "") - if target_id: - inflight_counts[target_id] = max(0, inflight_counts[target_id] - 1) - result = future.result() - if keep_results: - results.append(result) - _record_result_summary(counters, result) - extra = _process_discovery_result(result, root, discovery_mode, discovery_state) - if _is_novel_discovery(extra): - last_novelty_time = time.monotonic() - _write_run_records(result, timeline, commands, extra) - if prune_uninteresting and _prune_case_artifacts(result, extra): - pruned_cases += 1 - futures = remaining_map - - run_dir_bytes = _run_dir_size_bytes(root) - llvm_profile_files = _count_llvm_profile_files(root) - if keep_results: - results.sort(key=lambda item: (item.target_id, item.case_id)) - summary = MultiTargetSummary( - run_id=run_id, - work_dir=str(root), - config_path=str(config_path), - duration_budget_sec=duration_sec, - elapsed_sec=round(time.monotonic() - started, 3), - workers=workers, - timeout_sec=timeout_sec, - max_run_bytes=max_run_bytes, - run_dir_bytes=run_dir_bytes, - stop_reason=stop_reason, - targets=len(profiles), - cases=counters.cases, - crashes=counters.crashes, - reached=counters.reached, - valid_ppds=counters.valid_ppds, - timeouts=counters.timeouts, - skipped=skipped, - pruned_cases=pruned_cases, - skip_counts=dict(sorted(skip_counts.items())), - scheduler=scheduler, - min_target_share=normalized_min_target_share, - max_target_share=normalized_max_target_share, - runtime_skip_enabled=runtime_skip, - auto_skip_state_enabled=auto_skip_state, - auto_skip_search_root=str(resolved_auto_skip_search_root), - runtime_suppressed_shapes=len(discovery_state.suppressed_case_shapes), - seeded_runtime_suppressed_shapes=seeded_runtime_suppressed_shapes, - runtime_suppressed_hazards=len(discovery_state.suppressed_case_hazards), - seeded_runtime_suppressed_hazards=seeded_runtime_suppressed_hazards, - runtime_suppressed_families=len(discovery_state.suppressed_case_families), - seeded_runtime_suppressed_families=seeded_runtime_suppressed_families, - runtime_suppressed_semantic_shapes=len(discovery_state.suppressed_semantic_shapes), - seeded_runtime_suppressed_semantic_shapes=seeded_runtime_suppressed_semantic_shapes, - generalized_skip_enabled=generalized_skip, - family_skip_after=max(1, family_skip_after), - hazard_skip_after=hazard_skip_after, - semantic_skip_after=semantic_skip_after, - skip_probe_rate=normalized_skip_probe_rate, - skip_only_stop_after=normalized_skip_only_stop_after, - stagnation_stop_after_sec=normalized_stagnation_stop_after_sec, - template_cycle_epochs=template_cycle_epochs, - llvm_profile_enabled=llvm_profile_enabled, - llvm_profile_files=llvm_profile_files, - seed_skip_state_path=str(resolved_seed_skip_state_path) if resolved_seed_skip_state_path else "", - target_stats=_target_stats_summary(discovery_state), - retained_cases=discovery_state.retained_cases, - coverage_features=len(discovery_state.seen_features), - unique_crashes=discovery_state.unique_crashes, - repeat_crashes=discovery_state.repeat_crashes, - oracle_counts=dict(sorted(counters.oracle_counts.items())), - results=results, - ) - concise_summary = summary.concise_dict() - (root / "summary.concise.json").write_text( - json.dumps(concise_summary, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - if normalized_summary_mode == "full": - summary_payload = summary.to_dict() - else: - summary_payload = { - **concise_summary, - "summary_mode": "concise", - "results_omitted": True, - "results_source": "timeline.jsonl", - } - (root / "summary.json").write_text( - json.dumps(summary_payload, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - _write_discovery_state(root, discovery_state) - _write_standard_metrics(root) - return summary - - -def run_case( - profile: TargetProfile, - case_id: int, - case_dir: Path, - timeout_sec: int = 15, - capture_stdout: bool = True, -) -> CaseResult: - case_dir.mkdir(parents=True, exist_ok=True) - ppd_path = case_dir / "candidate.ppd" - command_path = case_dir / "command.txt" - stdout_path = case_dir / "stdout.bin" - stderr_path = case_dir / "stderr.txt" - meta_path = case_dir / "meta.json" - - ppd_path.write_text(make_ppd(profile.ppd_kind, case_id), encoding="utf-8") - document = make_document(profile.document_kind, case_id, target_id=profile.id) - document_path = case_dir / f"document{document.extension}" - document_path.write_bytes(document.data) - - cupstestppd_ok = run_cupstestppd(ppd_path, case_dir / "cupstestppd.txt") - filters = list_filters(profile, ppd_path, document_path, case_dir / "list_filters.stderr") - reached = all(expected in filters for expected in profile.expected_filters) - - job_options = build_job_options(profile, case_id) - command = build_command(profile, ppd_path, document_path, job_options=job_options) - command_line = format_command(profile, ppd_path, command, case_dir) - env_overrides = build_env_overrides(profile, ppd_path, case_dir) - compare_trace_path = env_overrides.get("SMT_FUZZER_COMPARE_TRACE", "") - command_path.write_text(command_line + "\n", encoding="utf-8") - started = time.perf_counter() - returncode: int | None = None - timed_out = False - stderr_text = "" - try: - if capture_stdout: - stdout_handle = stdout_path.open("wb") - else: - stdout_path.write_text("stdout discarded for this campaign; replay command.txt to reproduce output.\n", encoding="utf-8") - stdout_handle = Path(os.devnull).open("wb") - with stdout_handle as stdout: - completed = subprocess.run( - command, - stdout=stdout, - stderr=subprocess.PIPE, - text=True, - timeout=timeout_sec, - check=False, - env=build_env(profile, ppd_path, case_dir), - ) - returncode = completed.returncode - stderr_text = completed.stderr - except subprocess.TimeoutExpired as exc: - timed_out = True - stderr_text = (exc.stderr or "") if isinstance(exc.stderr, str) else "" - duration_ms = round((time.perf_counter() - started) * 1000.0, 3) - stderr_path.write_text(stderr_text, encoding="utf-8") - crashed, oracle = classify(profile, stderr_text, returncode, timed_out) - - result = CaseResult( - target_id=profile.id, - case_id=case_id, - target_description=profile.description, - ppd_kind=profile.ppd_kind, - document_kind=profile.document_kind, - document_description=document.description, - work_dir=str(case_dir), - ppd_path=str(ppd_path), - document_path=str(document_path), - command_path=str(command_path), - command_line=command_line, - job_options=job_options, - env_overrides=env_overrides, - compare_trace_path=compare_trace_path, - stdout_path=str(stdout_path), - stderr_path=str(stderr_path), - meta_path=str(meta_path), - cupstestppd_ok=cupstestppd_ok, - filters=filters, - reached_expected_filter=reached, - returncode=returncode, - timed_out=timed_out, - crashed=crashed, - oracle=oracle, - duration_ms=duration_ms, - stderr_tail=_tail_lines(stderr_text, 12), - ) - meta_path.write_text(json.dumps(asdict(result), indent=2, sort_keys=True) + "\n", encoding="utf-8") - return result - - -def _write_standard_metrics(root: Path) -> None: - try: - from parser_fuzzers.run_metrics import write_standard_run_metrics - - write_standard_run_metrics(root) - except Exception as exc: # pragma: no cover - metrics must not break fuzz runs - (root / "standard_metrics.error.txt").write_text(f"{type(exc).__name__}: {exc}\n", encoding="utf-8") - - -def load_profiles(config_path: str | Path, *, filter_root: str | Path | None = None) -> list[TargetProfile]: - data = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) or {} - resolved_filter_root = Path(filter_root) if filter_root else None - profiles = [] - for item in data.get("targets", []): - executor = str(item["executor"]) - filter_binary = str(item.get("filter_binary", "")) - if resolved_filter_root is not None and executor == "direct_filter": - filter_binary = remap_filter_binary(filter_binary, resolved_filter_root) - profiles.append( - TargetProfile( - id=str(item["id"]), - description=str(item.get("description", "")), - ppd_kind=str(item["ppd_kind"]), - document_kind=str(item["document_kind"]), - executor=executor, - input_mime=str(item.get("input_mime", "")), - output_mime=str(item.get("output_mime", "printer/foo")), - expected_filters=[str(value) for value in item.get("expected_filters", [])], - cases=int(item.get("cases", 1)), - oracle=str(item.get("oracle", "crash_or_signal")), - filter_binary=filter_binary, - ) - ) - return profiles - - -def remap_filter_binary(filter_binary: str, filter_root: str | Path) -> str: - name = Path(filter_binary).name if filter_binary else "" - if not name: - return filter_binary - return str(Path(filter_root) / name) - - -def case_shape_key(profile: TargetProfile, case_id: int) -> str: - ppd_period = PPD_PERIODS.get(profile.ppd_kind, 1) - document_period = DOCUMENT_PERIODS.get(profile.document_kind, 1) - document_slot = _shape_slot(case_id, document_period) - document_epoch = _shape_epoch_suffix(profile, case_id, document_period) - return "|".join( - [ - f"target:{profile.id}", - f"ppd:{profile.ppd_kind}:{_shape_slot(case_id, ppd_period)}", - f"doc:{profile.document_kind}:{document_slot}{document_epoch}", - f"options:{_shape_slot(case_id, JOB_OPTION_PERIOD) if build_job_options(profile, case_id) else 'off'}", - ] - ) - - -def case_family_key(profile: TargetProfile) -> str: - return "|".join( - [ - f"target:{_target_family(profile.id)}", - f"ppd:{profile.ppd_kind}", - f"doc:{profile.document_kind}", - ] - ) - - -def case_hazard_key(profile: TargetProfile, case_id: int) -> str: - if profile.document_kind != "image_feedback_sweep": - return "" - image = image_feedback_instance(case_id, target_id=profile.id) - if image.payload_delta < 0: - payload = "short" - elif image.payload_delta > 0: - payload = "extra" - else: - payload = "exact" - return "|".join( - [ - f"target:{profile.id}", - f"ppd:{profile.ppd_kind}", - f"doc:{profile.document_kind}", - f"fmt:{image.image_format}", - f"objective:{image.objective}", - f"payload:{payload}", - f"interlace:{image.png_interlace}", - ] - ) - - -def planned_semantic_runtime_key(profile: TargetProfile, case_id: int) -> str: - semantic_hash = planned_semantic_input_hash( - profile.id, - profile.executor, - profile.ppd_kind, - profile.document_kind, - profile.input_mime, - profile.output_mime, - tuple(profile.expected_filters), - _semantic_case_slot(profile, case_id), - ) - return semantic_runtime_key(profile.id, semantic_hash) - - -@lru_cache(maxsize=65536) -def planned_semantic_input_hash( - target_id: str, - executor: str, - ppd_kind: str, - document_kind: str, - input_mime: str, - output_mime: str, - expected_filters: tuple[str, ...], - semantic_case_id: int, -) -> str: - document = make_document(document_kind, semantic_case_id, target_id=target_id) - profile = TargetProfile( - id=target_id, - description="", - ppd_kind=ppd_kind, - document_kind=document_kind, - executor=executor, - input_mime=input_mime, - output_mime=output_mime, - expected_filters=list(expected_filters), - cases=0, - oracle="", - filter_binary="", - ) - job_options = build_job_options(profile, semantic_case_id) - shape = build_planned_shape( - target_id=target_id, - ppd_kind=ppd_kind, - document_kind=document_kind, - input_mime=input_mime, - output_mime=output_mime, - expected_filters=list(expected_filters), - ppd_text=make_ppd(ppd_kind, semantic_case_id), - document_data=document.data, - job_options=job_options, - ) - return str(shape["semantic_input_hash"]) - - -def _semantic_case_slot(profile: TargetProfile, case_id: int) -> int: - ppd_period = max(1, PPD_PERIODS.get(profile.ppd_kind, 1)) - document_period = max(1, DOCUMENT_PERIODS.get(profile.document_kind, 1)) - if profile.document_kind == "image_feedback_sweep": - document_period *= _template_cycle_epochs() - option_period = JOB_OPTION_PERIOD if build_job_options(profile, case_id) else 1 - period = math.lcm(ppd_period, document_period, option_period) - return case_id % max(1, period) - - -def runtime_skip_reason(profile: TargetProfile, case_id: int, state: DiscoveryState) -> str: - if not state.runtime_skip_enabled: - return "" - semantic_reason = semantic_runtime_skip_reason(profile, case_id, state) - if semantic_reason: - return semantic_reason - signature = state.suppressed_case_shapes.get(case_shape_key(profile, case_id)) - if signature: - return f"runtime-known-crash-shape:{_short_signature_label(signature)}" - hazard = case_hazard_key(profile, case_id) - if hazard: - signature = state.suppressed_case_hazards.get(hazard) - if signature: - return f"runtime-known-crash-hazard:{_short_signature_label(signature)}" - if state.generalized_skip_enabled: - family_signature = state.suppressed_case_families.get(case_family_key(profile)) - if family_signature: - return f"runtime-known-crash-family:{_short_signature_label(family_signature)}" - return "" - - -def semantic_runtime_skip_reason(profile: TargetProfile, case_id: int, state: DiscoveryState) -> str: - if state.semantic_skip_after <= 0: - return "" - semantic_key = planned_semantic_runtime_key(profile, case_id) - signature = state.suppressed_semantic_shapes.get(semantic_key) - if signature: - return f"runtime-known-crash-semantic-shape:{_short_signature_label(signature)}" - return "" - - -def load_runtime_skip_state( - state: DiscoveryState, - state_path: str | Path, - *, - generalized_skip: bool = False, - family_skip_after: int = 32, -) -> int: - payload = json.loads(Path(state_path).read_text(encoding="utf-8")) - loaded = 0 - if _legacy_skip_state_enabled(): - for item in payload.get("suppressed_case_shapes", []): - if not isinstance(item, dict): - continue - shape = str(item.get("shape") or "") - signature = str(item.get("signature") or "") - if not shape or not signature: - continue - if shape not in state.suppressed_case_shapes: - loaded += 1 - state.suppressed_case_shapes.setdefault(shape, signature) - state.seen_crash_signatures.add(signature) - if generalized_skip: - _record_family_suppression_candidate( - state, - _shape_family_key(shape), - signature, - max(1, family_skip_after), - ) - for item in payload.get("suppressed_case_hazards", []): - if not isinstance(item, dict): - continue - hazard = str(item.get("hazard") or "") - signature = str(item.get("signature") or "") - if not hazard or not signature: - continue - state.suppressed_case_hazards.setdefault(hazard, signature) - state.seen_crash_signatures.add(signature) - if generalized_skip: - for item in payload.get("suppressed_case_families", []): - if not isinstance(item, dict): - continue - family = str(item.get("family") or "") - signature = str(item.get("signature") or "") - if not family or not signature: - continue - state.suppressed_case_families.setdefault(family, signature) - state.seen_crash_signatures.add(signature) - for item in payload.get("suppressed_semantic_shapes", []): - if not isinstance(item, dict): - continue - semantic = str(item.get("semantic") or item.get("shape") or "") - signature = str(item.get("signature") or "") - if not semantic or not signature: - continue - if semantic not in state.suppressed_semantic_shapes: - loaded += 1 - state.suppressed_semantic_shapes.setdefault(semantic, signature) - state.seen_crash_signatures.add(signature) - return loaded - - -def find_latest_runtime_skip_state(search_root: str | Path) -> Path | None: - root = Path(search_root) - if not root.exists(): - return None - candidates: list[tuple[float, Path]] = [] - for state_path in _iter_runtime_skip_state_candidates(root): - if _runtime_skip_state_has_suppression(state_path): - try: - candidates.append((state_path.stat().st_mtime, state_path)) - except OSError: - continue - if not candidates: - return None - return max(candidates, key=lambda item: (item[0], str(item[1])))[1] - - -def find_latest_semantic_skip_state(search_root: str | Path, target_ids: list[str]) -> Path | None: - root = Path(search_root) - if not root.exists(): - return None - target_prefixes = tuple(f"target:{target_id}|" for target_id in target_ids) - candidates: list[tuple[float, Path]] = [] - for state_path in _iter_runtime_skip_state_candidates(root): - if not _semantic_skip_state_matches_targets(state_path, target_prefixes): - continue - try: - candidates.append((state_path.stat().st_mtime, state_path)) - except OSError: - continue - if not candidates: - return None - return max(candidates, key=lambda item: (item[0], str(item[1])))[1] - - -def _semantic_skip_state_matches_targets(state_path: Path, target_prefixes: tuple[str, ...]) -> bool: - try: - payload = json.loads(state_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return False - for item in payload.get("suppressed_semantic_shapes", []): - if not isinstance(item, dict): - continue - semantic = str(item.get("semantic") or item.get("shape") or "") - if not target_prefixes or semantic.startswith(target_prefixes): - return True - return False - - -def _iter_runtime_skip_state_candidates(root: Path) -> list[Path]: - candidates: list[Path] = [] - direct = root / "discovery_state.json" - if direct.exists(): - candidates.append(direct) - try: - first_level = [path for path in root.iterdir() if path.is_dir()] - except OSError: - return candidates - for path in first_level: - state = path / "discovery_state.json" - if state.exists(): - candidates.append(state) - try: - second_level = [child for child in path.iterdir() if child.is_dir()] - except OSError: - continue - for child in second_level: - state = child / "discovery_state.json" - if state.exists(): - candidates.append(state) - return candidates - - -def _runtime_skip_state_has_suppression(state_path: Path) -> bool: - try: - payload = json.loads(state_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return False - if not _legacy_skip_state_enabled(): - return bool(payload.get("suppressed_semantic_shapes")) - return bool( - payload.get("suppressed_case_shapes") - or payload.get("suppressed_case_hazards") - or payload.get("suppressed_case_families") - or payload.get("suppressed_semantic_shapes") - ) - - -def record_runtime_crash_suppression( - state: DiscoveryState, - profile: TargetProfile, - case_id: int, - signature: str, -) -> bool: - shape = case_shape_key(profile, case_id) - count_key = f"{shape}|signature:{signature}" - count = state.crash_shape_counts.get(count_key, 0) + 1 - state.crash_shape_counts[count_key] = count - if count >= state.crash_skip_after: - state.suppressed_case_shapes.setdefault(shape, signature) - _record_hazard_suppression_candidate(state, profile, case_id, signature) - if state.generalized_skip_enabled: - _record_family_suppression_candidate( - state, - case_family_key(profile), - signature, - state.family_skip_after, - ) - return True - if state.generalized_skip_enabled: - _record_family_suppression_candidate( - state, - case_family_key(profile), - signature, - state.family_skip_after, - ) - _record_hazard_suppression_candidate(state, profile, case_id, signature) - return False - - -def record_semantic_crash_suppression( - state: DiscoveryState, - result: CaseResult, - shape_bundle: dict[str, Any], - signature: str, - *, - retained: bool, -) -> bool: - if state.semantic_skip_after <= 0 or retained: - return False - semantic_hash = str(shape_bundle.get("semantic_input_hash") or "") - failure_hash = str(shape_bundle.get("failure_shape_hash") or "") - if not semantic_hash or not failure_hash: - return False - semantic_key = semantic_runtime_key(result.target_id, semantic_hash) - count_key = f"{semantic_key}|failure:{failure_hash}|signature:{signature}" - count = state.semantic_crash_counts.get(count_key, 0) + 1 - state.semantic_crash_counts[count_key] = count - if count >= state.semantic_skip_after: - state.suppressed_semantic_shapes.setdefault(semantic_key, signature) - return True - return False - - -def _record_family_suppression_candidate( - state: DiscoveryState, - family: str, - signature: str, - threshold: int, -) -> bool: - count_key = f"{family}|signature:{signature}" - count = state.crash_family_counts.get(count_key, 0) + 1 - state.crash_family_counts[count_key] = count - if count >= threshold: - state.suppressed_case_families.setdefault(family, signature) - return True - return False - - -def _record_hazard_suppression_candidate( - state: DiscoveryState, - profile: TargetProfile, - case_id: int, - signature: str, -) -> bool: - if state.hazard_skip_after <= 0: - return False - hazard = case_hazard_key(profile, case_id) - if not hazard: - return False - count_key = f"{hazard}|signature:{signature}" - count = state.crash_hazard_counts.get(count_key, 0) + 1 - state.crash_hazard_counts[count_key] = count - if count >= state.hazard_skip_after: - state.suppressed_case_hazards.setdefault(hazard, signature) - return True - return False - - -def _shape_family_key(shape: str) -> str: - target = "" - ppd = "" - doc = "" - for part in shape.split("|"): - if part.startswith("target:"): - target = _target_family(part.removeprefix("target:")) - elif part.startswith("ppd:"): - ppd = part.removeprefix("ppd:").split(":", 1)[0] - elif part.startswith("doc:"): - doc = part.removeprefix("doc:").split(":", 1)[0] - return "|".join([f"target:{target}", f"ppd:{ppd}", f"doc:{doc}"]) - - -def _target_family(target_id: str) -> str: - for suffix in ("_coverage", "_general", "_explore", "_structural", "_feedback"): - if target_id.endswith(suffix): - return target_id.removesuffix(suffix) - return target_id - - -def _shape_slot(case_id: int, period: int) -> str: - if period <= 0: - return str(case_id) - return str(case_id % period) - - -def _shape_epoch_suffix(profile: TargetProfile, case_id: int, period: int) -> str: - if profile.document_kind != "image_feedback_sweep" or period <= 0: - return "" - epochs = _template_cycle_epochs() - if epochs <= 1: - return "" - return f":epoch{(case_id // period) % epochs}" - - -def _combined_skip_reason( - profile: TargetProfile, - case_id: int, - discovery_mode: str, - state: DiscoveryState, - *, - skip_probe_rate: float = 0.0, -) -> str: - if discovery_mode != "coverage": - return "" - static_reason = coverage_skip_reason(profile, case_id) - if static_reason: - return static_reason - runtime_reason = runtime_skip_reason(profile, case_id, state) - if runtime_reason and _should_force_avoidance_probe(profile, case_id, state): - return "" - if runtime_reason and _should_probe_runtime_skip(profile, case_id, runtime_reason, skip_probe_rate): - return "" - return runtime_reason - - -def _should_force_avoidance_probe(profile: TargetProfile, case_id: int, state: DiscoveryState) -> bool: - rate = _avoidance_skip_probe_rate() - if rate <= 0.0: - return False - if _target_seeded_suppression_pressure(state, profile.id) <= 0.0: - return False - if _case_has_avoidable_image_hazard(profile, case_id, state): - return True - if rate >= 1.0: - return True - key = f"avoidance-probe|{profile.id}|{profile.ppd_kind}|{profile.document_kind}|{case_id}".encode("utf-8") - value = int.from_bytes(hashlib.sha256(key).digest()[:8], "big") / float(1 << 64) - return value < rate - - -def _case_has_avoidable_image_hazard(profile: TargetProfile, case_id: int, state: DiscoveryState) -> bool: - hazard = case_hazard_key(profile, case_id) - if not hazard: - return False - if hazard in state.suppressed_case_hazards: - return True - target_id = _key_field(hazard, "target") - image_format = _key_field(hazard, "fmt") - payload = _key_field(hazard, "payload") - interlace = _key_field(hazard, "interlace") - if not target_id or not image_format or not payload: - return False - for suppressed_hazard in state.suppressed_case_hazards: - if not _same_or_derived_target(_key_field(suppressed_hazard, "target"), target_id): - continue - if _key_field(suppressed_hazard, "fmt") != image_format: - continue - if _key_field(suppressed_hazard, "payload") != payload: - continue - suppressed_interlace = _key_field(suppressed_hazard, "interlace") - if suppressed_interlace and interlace and suppressed_interlace != interlace: - continue - return True - return False - - -def _should_probe_runtime_skip( - profile: TargetProfile, - case_id: int, - skip_reason: str, - skip_probe_rate: float, -) -> bool: - if skip_probe_rate <= 0.0: - return False - if skip_probe_rate >= 1.0: - return True - key = f"{profile.id}|{profile.ppd_kind}|{profile.document_kind}|{case_id}|{skip_reason}".encode("utf-8") - value = int.from_bytes(hashlib.sha256(key).digest()[:8], "big") / float(1 << 64) - return value < skip_probe_rate - - -def _initialize_target_stats(state: DiscoveryState, profiles: list[TargetProfile]) -> None: - for profile in profiles: - _target_stats(state, profile.id) - state.scheduler_credit.setdefault(profile.id, 0.0) - - -def _target_stats(state: DiscoveryState, target_id: str) -> TargetDiscoveryStats: - if target_id not in state.target_stats: - state.target_stats[target_id] = TargetDiscoveryStats() - return state.target_stats[target_id] - - -def _record_submission(state: DiscoveryState, profile: TargetProfile) -> None: - _target_stats(state, profile.id).submitted += 1 - - -def _record_skip(state: DiscoveryState, profile: TargetProfile, skip_reason: str) -> None: - stats = _target_stats(state, profile.id) - stats.skipped += 1 - if skip_reason.startswith("runtime-known-crash-"): - stats.runtime_suppressed += 1 - - -def _record_result_summary(counters: RunCounters, result: CaseResult) -> None: - counters.cases += 1 - if result.crashed: - counters.crashes += 1 - if result.reached_expected_filter: - counters.reached += 1 - if result.cupstestppd_ok: - counters.valid_ppds += 1 - if result.timed_out: - counters.timeouts += 1 - oracle = result.oracle or "none" - counters.oracle_counts[oracle] = counters.oracle_counts.get(oracle, 0) + 1 - - -def _is_novel_discovery(extra: dict[str, Any] | None) -> bool: - if not extra: - return False - return bool(extra.get("retained_for_coverage") or extra.get("new_crash_signature") is True) - - -def _coverage_stagnated( - discovery_mode: str, - stagnation_stop_after_sec: int, - state: DiscoveryState, - last_novelty_time: float, - now: float, -) -> bool: - if discovery_mode != "coverage" or stagnation_stop_after_sec <= 0: - return False - if state.completed_cases < 100: - return False - return now - last_novelty_time >= stagnation_stop_after_sec - - -def _choose_next_profile( - profiles: list[TargetProfile], - state: DiscoveryState, - inflight_counts: dict[str, int], - scheduler: str, - *, - min_target_share: float = 0.0, - max_target_share: float = 1.0, -) -> TargetProfile: - if not profiles: - raise ValueError("no target profiles configured") - normalized_min_share = _normalize_min_target_share(min_target_share, len(profiles)) - normalized_max_share = _normalize_max_target_share(max_target_share, len(profiles), normalized_min_share) - floor_profile = _target_budget_floor_candidate(profiles, state, normalized_min_share) - if floor_profile is not None: - return floor_profile - probe_profile = _avoidance_probe_candidate(profiles, state, inflight_counts, normalized_max_share) - if probe_profile is not None: - return probe_profile - if scheduler == "round-robin": - eligible_profiles = _target_budget_eligible_profiles(profiles, state, normalized_max_share) - for _ in profiles: - profile = profiles[state.scheduler_cursor % len(profiles)] - state.scheduler_cursor += 1 - if profile in eligible_profiles: - return profile - return profiles[0] - if scheduler != "novelty": - raise ValueError(f"unknown scheduler: {scheduler}") - - eligible_profiles = _target_budget_eligible_profiles(profiles, state, normalized_max_share) - scores = {profile.id: _target_scheduler_score(state, profile.id) for profile in profiles} - total_score = sum(scores[profile.id] for profile in eligible_profiles) or 1.0 - for profile in profiles: - state.scheduler_credit[profile.id] = state.scheduler_credit.get(profile.id, 0.0) + scores[profile.id] - - def rank(profile: TargetProfile) -> tuple[float, int, str]: - stats = _target_stats(state, profile.id) - adjusted_credit = state.scheduler_credit.get(profile.id, 0.0) / (1 + inflight_counts.get(profile.id, 0)) - return (adjusted_credit, -stats.submitted, profile.id) - - chosen = max(eligible_profiles, key=rank) - state.scheduler_credit[chosen.id] = state.scheduler_credit.get(chosen.id, 0.0) - total_score - return chosen - - -def _normalize_min_target_share(value: float, target_count: int) -> float: - if target_count <= 0: - return 0.0 - return max(0.0, min(float(value), 1.0 / target_count)) - - -def _normalize_max_target_share(value: float, target_count: int, min_target_share: float = 0.0) -> float: - if target_count <= 0: - return 1.0 - if value <= 0.0: - return 1.0 - lower_bound = max(min_target_share, 1.0 / target_count) - return max(lower_bound, min(float(value), 1.0)) - - -def _target_budget_attempts(state: DiscoveryState, target_id: str) -> int: - stats = _target_stats(state, target_id) - return stats.submitted + stats.skipped - - -def _target_budget_floor_candidate( - profiles: list[TargetProfile], - state: DiscoveryState, - min_target_share: float, -) -> TargetProfile | None: - if min_target_share <= 0.0: - return None - total_attempts = sum(_target_budget_attempts(state, profile.id) for profile in profiles) - if total_attempts <= 0: - return None - - def rank(profile: TargetProfile) -> tuple[float, int, float, str]: - attempts = _target_budget_attempts(state, profile.id) - share = attempts / total_attempts - deficit = min_target_share - share - return (deficit, -attempts, _target_scheduler_score(state, profile.id), profile.id) - - under_budget = [ - profile - for profile in profiles - if (_target_budget_attempts(state, profile.id) / total_attempts) < min_target_share - ] - if not under_budget: - return None - return max(under_budget, key=rank) - - -def _target_budget_eligible_profiles( - profiles: list[TargetProfile], - state: DiscoveryState, - max_target_share: float, -) -> list[TargetProfile]: - if max_target_share >= 1.0 or len(profiles) <= 1: - return profiles - total_attempts = sum(_target_budget_attempts(state, profile.id) for profile in profiles) - if total_attempts <= 0: - return profiles - min_attempts = min(_target_budget_attempts(state, profile.id) for profile in profiles) - eligible = [] - for profile in profiles: - attempts = _target_budget_attempts(state, profile.id) - projected_share = (attempts + 1) / (total_attempts + 1) - if projected_share <= max_target_share or attempts == min_attempts: - eligible.append(profile) - return eligible or profiles - - -def _target_scheduler_score(state: DiscoveryState, target_id: str) -> float: - stats = _target_stats(state, target_id) - if stats.completed == 0: - score = 3.0 - else: - novelty_rate = stats.retained_cases / max(1, stats.completed) - crash_rate = stats.crashes / max(1, stats.completed) - repeat_crash_rate = stats.repeat_crashes / max(1, stats.crashes) - score = 1.0 + (8.0 * novelty_rate) + min(4.0, 0.05 * stats.new_features) - crash_penalty_disabled = _scheduler_crash_penalty_disabled() - if stats.crashes and not crash_penalty_disabled: - score *= 1.0 / (1.0 + 12.0 * crash_rate) - if stats.repeat_crashes and not crash_penalty_disabled: - score *= 1.0 / (1.0 + 4.0 * repeat_crash_rate) - if ( - stats.completed >= 100 - and stats.crashes > max(20, stats.retained_cases * 10) - and not crash_penalty_disabled - ): - score *= 0.25 - if stats.timeouts: - score *= 1.0 / (1.0 + min(4.0, 0.1 * stats.timeouts)) - if stats.runtime_suppressed: - score *= 1.0 / (1.0 + min(12.0, stats.runtime_suppressed / 20.0)) - suppression_pressure = _target_seeded_suppression_pressure(state, target_id) - if suppression_pressure: - score *= 1.0 / (1.0 + min(_avoidance_scheduler_penalty_cap(), suppression_pressure)) - if stats.completed == 0 and stats.skipped >= 100: - score *= 0.1 - return max(0.1, min(10.0, score)) - - -def _avoidance_probe_candidate( - profiles: list[TargetProfile], - state: DiscoveryState, - inflight_counts: dict[str, int], - max_target_share: float, -) -> TargetProfile | None: - interval = _avoidance_probe_interval() - if interval <= 0: - return None - total_attempts = sum(_target_budget_attempts(state, profile.id) for profile in profiles) - if total_attempts <= 0 or total_attempts % interval != 0: - return None - eligible_profiles = _target_budget_eligible_profiles(profiles, state, max_target_share) - candidates = [ - profile - for profile in eligible_profiles - if _target_seeded_suppression_pressure(state, profile.id) > 0.0 - ] - if not candidates: - return None - - def rank(profile: TargetProfile) -> tuple[int, int, float, str]: - stats = _target_stats(state, profile.id) - return ( - inflight_counts.get(profile.id, 0), - stats.submitted, - -_target_seeded_suppression_pressure(state, profile.id), - profile.id, - ) - - return min(candidates, key=rank) - - -def _target_seeded_suppression_pressure(state: DiscoveryState, target_id: str) -> float: - hazard_pressure = 0.0 - for hazard in state.suppressed_case_hazards: - hazard_target = _key_field(hazard, "target") - if hazard_target and _same_or_derived_target(hazard_target, target_id): - hazard_pressure += 0.35 - family_pressure = 0.0 - target_family = _target_family(target_id) - for family in state.suppressed_case_families: - family_target = _key_field(family, "target") - if family_target and ( - _same_or_derived_target(family_target, target_id) - or _same_or_derived_target(family_target, target_family) - ): - family_pressure += 0.75 - semantic_pressure = 0.0 - prefix = f"{target_id}|" - family_prefix = f"{target_family}|" - for semantic_key in state.suppressed_semantic_shapes: - if semantic_key.startswith(prefix) or semantic_key.startswith(family_prefix): - semantic_pressure += 0.10 - return hazard_pressure + family_pressure + min(4.0, semantic_pressure) - - -def _key_field(key: str, field_name: str) -> str: - prefix = f"{field_name}:" - for part in key.split("|"): - if part.startswith(prefix): - return part.removeprefix(prefix) - return "" - - -def _same_or_derived_target(left: str, right: str) -> bool: - if left == right: - return True - return left.startswith(f"{right}_") or right.startswith(f"{left}_") - - -def _short_signature_label(signature: str) -> str: - label = signature - if label.startswith("SUMMARY: AddressSanitizer: "): - label = label.removeprefix("SUMMARY: AddressSanitizer: ") - label = label.replace(" ", "-").replace("/", "_") - return label[:96] - - -def _target_stats_summary(state: DiscoveryState) -> dict[str, dict[str, int]]: - return { - target_id: asdict(stats) - for target_id, stats in sorted(state.target_stats.items()) - } - - -def _write_discovery_state(root: Path, state: DiscoveryState) -> None: - payload = { - "runtime_skip_enabled": state.runtime_skip_enabled, - "crash_skip_after": state.crash_skip_after, - "hazard_skip_after": state.hazard_skip_after, - "semantic_skip_after": state.semantic_skip_after, - "generalized_skip_enabled": state.generalized_skip_enabled, - "family_skip_after": state.family_skip_after, - "runtime_suppressed_shapes": len(state.suppressed_case_shapes), - "runtime_suppressed_hazards": len(state.suppressed_case_hazards), - "runtime_suppressed_families": len(state.suppressed_case_families), - "runtime_suppressed_semantic_shapes": len(state.suppressed_semantic_shapes), - "suppressed_case_shapes": [ - {"shape": shape, "signature": signature} - for shape, signature in sorted(state.suppressed_case_shapes.items()) - ], - "suppressed_semantic_shapes": [ - {"semantic": semantic, "signature": signature} - for semantic, signature in sorted(state.suppressed_semantic_shapes.items()) - ], - "suppressed_case_hazards": [ - {"hazard": hazard, "signature": signature} - for hazard, signature in sorted(state.suppressed_case_hazards.items()) - ], - "suppressed_case_families": [ - {"family": family, "signature": signature} - for family, signature in sorted(state.suppressed_case_families.items()) - ], - "target_stats": _target_stats_summary(state), - "scheduler_credit": { - target_id: round(credit, 6) - for target_id, credit in sorted(state.scheduler_credit.items()) - }, - } - (root / "discovery_state.json").write_text( - json.dumps(payload, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - - -def _warm_template_synth_cache(profiles: list[TargetProfile]) -> None: - warmed_ppd: set[tuple[str, int]] = set() - warmed_document: set[tuple[str, int]] = set() - for profile in profiles: - ppd_period = PPD_PERIODS.get(profile.ppd_kind, 1) - for case_id in range(ppd_period): - key = (profile.ppd_kind, case_id) - if key not in warmed_ppd: - make_ppd(profile.ppd_kind, case_id) - warmed_ppd.add(key) - document_period = DOCUMENT_PERIODS.get(profile.document_kind, 1) - for case_id in range(document_period): - key = (profile.id, profile.document_kind, case_id) - if key not in warmed_document: - make_document(profile.document_kind, case_id, target_id=profile.id) - warmed_document.add(key) - - -def _prune_case_artifacts(result: CaseResult, extra: dict[str, Any] | None) -> bool: - if result.crashed or result.timed_out: - return False - if extra and extra.get("retained_for_coverage"): - return False - path = Path(result.work_dir) - if not path.exists(): - return False - shutil.rmtree(path) - return True - - -def _run_dir_limit_reached(root: Path, max_run_bytes: int) -> bool: - return bool(max_run_bytes and _run_dir_size_bytes(root) >= max_run_bytes) - - -def _run_dir_size_bytes(root: Path) -> int: - total = 0 - if not root.exists(): - return total - for dirpath, _, filenames in os.walk(root): - for filename in filenames: - path = Path(dirpath) / filename - try: - total += path.stat().st_size - except OSError: - continue - return total - - -def coverage_skip_reason(profile: TargetProfile, case_id: int) -> str: - if _skip_short_image_aborts_enabled() and profile.document_kind == "image_feedback_sweep": - image = image_feedback_instance(case_id) - if image.objective == "short_payload" and image.image_format.startswith("png"): - return "low-value-short-png-libpng-abort" - if "rastertoescpx" in profile.id and profile.document_kind == "cups_raster_general_sweep": - case_mod = case_id % len(RASTER_GENERAL_CASES) - if case_mod in RASTERTOESCPX_DOTROWSTEP_ZERO_MODS: - return "known-rastertoescpx-dotrowstep-zero-fpe" - if "pwg_to_raster" in profile.id and profile.ppd_kind == "pwg_resolution_general": - dpi = GENERAL_RESOLUTIONS[case_id % len(GENERAL_RESOLUTIONS)] - if dpi == 65536: - return "known-libppd-65536dpi-fpe" - return "" - - -def _skip_short_image_aborts_enabled() -> bool: - return _env_flag("SMT_FUZZER_SKIP_SHORT_IMAGE_ABORTS") - - -def _llvm_profiles_enabled() -> bool: - return _env_flag("SMT_FUZZER_ENABLE_LLVM_PROFILES") - - -def _template_cycle_epochs() -> int: - value = os.environ.get("SMT_FUZZER_IMAGE_CYCLE_EPOCHS") or os.environ.get("SMT_FUZZER_TEMPLATE_CYCLE_EPOCHS", "1") - try: - return max(1, min(64, int(value))) - except ValueError: - return 1 - - -def _hazard_skip_after() -> int: - value = os.environ.get("SMT_FUZZER_HAZARD_SKIP_AFTER", "0") - try: - return max(0, int(value)) - except ValueError: - return 0 - - -def _semantic_skip_after() -> int: - value = os.environ.get("SMT_FUZZER_SEMANTIC_SKIP_AFTER", "0") - try: - return max(0, int(value)) - except ValueError: - return 0 - - -def _avoidance_probe_interval() -> int: - value = os.environ.get("SMT_FUZZER_AVOIDANCE_PROBE_INTERVAL", "0") - try: - return max(0, int(value)) - except ValueError: - return 0 - - -def _avoidance_skip_probe_rate() -> float: - value = os.environ.get("SMT_FUZZER_AVOIDANCE_SKIP_PROBE_RATE", "0") - try: - return max(0.0, min(1.0, float(value))) - except ValueError: - return 0.0 - - -def _avoidance_scheduler_penalty_cap() -> float: - value = os.environ.get("SMT_FUZZER_AVOIDANCE_SCHEDULER_PENALTY_CAP", "2.0") - try: - return max(0.0, min(12.0, float(value))) - except ValueError: - return 2.0 - - -def _scheduler_crash_penalty_disabled() -> bool: - return _env_flag("SMT_FUZZER_DISABLE_SCHEDULER_CRASH_PENALTY") - - -def _legacy_skip_state_enabled() -> bool: - return os.environ.get("SMT_FUZZER_LOAD_LEGACY_SKIP_STATE", "1").strip().lower() not in { - "0", - "false", - "no", - "off", - } - - -def _skip_warm_template_cache() -> bool: - return _env_flag("SMT_FUZZER_SKIP_WARM_TEMPLATE_CACHE") - - -def _env_flag(name: str) -> bool: - return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"} - - -def _count_llvm_profile_files(root: Path) -> int: - if not root.exists(): - return 0 - return sum(1 for path in root.rglob("*.profraw") if path.is_file() and path.stat().st_size > 0) - - -def run_cupstestppd(ppd_path: Path, output_path: Path) -> bool: - completed = subprocess.run( - ["cupstestppd", "-W", "none", str(ppd_path)], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - check=False, - ) - output_path.write_text(completed.stdout, encoding="utf-8") - return completed.returncode == 0 - - -def list_filters(profile: TargetProfile, ppd_path: Path, document_path: Path, stderr_path: Path) -> list[str]: - if profile.executor == "direct_filter": - return profile.expected_filters - try: - completed = subprocess.run( - [ - "/usr/sbin/cupsfilter", - "--list-filters", - "-p", - str(ppd_path), - "-e", - "-i", - profile.input_mime, - "-m", - profile.output_mime, - str(document_path), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - timeout=8, - check=False, - ) - except subprocess.TimeoutExpired as exc: - stderr_path.write_text(str(exc), encoding="utf-8") - return [] - stderr_path.write_text(completed.stderr, encoding="utf-8") - return [line.strip() for line in completed.stdout.splitlines() if line.strip()] - - -def build_job_options(profile: TargetProfile, case_id: int) -> str: - if profile.executor != "direct_filter" or not _job_options_enabled(): - return "" - slot = case_id % JOB_OPTION_PERIOD - page_size = PAGE_SIZES[slot % len(PAGE_SIZES)][0] - color_model = JOB_COLOR_MODELS[(slot // len(PAGE_SIZES)) % len(JOB_COLOR_MODELS)] - quality = JOB_PRINT_QUALITIES[(slot // 7) % len(JOB_PRINT_QUALITIES)] - media = JOB_MEDIA_TYPES[(slot // 11) % len(JOB_MEDIA_TYPES)] - duplex = JOB_DUPLEX_MODES[(slot // 13) % len(JOB_DUPLEX_MODES)] - resolution = _job_resolution(profile, slot) - options: list[tuple[str, str]] = [ - ("PageSize", page_size), - ("PageRegion", page_size), - ("ColorModel", color_model), - ("PrintQuality", quality), - ("MediaType", media), - ("Duplex", duplex), - ] - if resolution: - options.append(("Resolution", resolution)) - if _target_family(profile.id).startswith(("image_to_", "pdf_to_", "postscript_to_", "text_to_")): - options.extend( - [ - ("fit-to-page", "true" if (slot // 17) % 2 else "false"), - ("scaling", JOB_SCALING_VALUES[(slot // 19) % len(JOB_SCALING_VALUES)]), - ("natural-scaling", JOB_NATURAL_SCALING_VALUES[(slot // 23) % len(JOB_NATURAL_SCALING_VALUES)]), - ("orientation-requested", JOB_ORIENTATIONS[(slot // 29) % len(JOB_ORIENTATIONS)]), - ("print-scaling", JOB_PRINT_SCALING[(slot // 31) % len(JOB_PRINT_SCALING)]), - ] - ) - return " ".join(f"{key}={value}" for key, value in options) - - -def _job_resolution(profile: TargetProfile, slot: int) -> str: - if profile.ppd_kind.startswith("pwg_resolution"): - if profile.ppd_kind == "pwg_resolution_general": - dpi = GENERAL_RESOLUTIONS[slot % len(GENERAL_RESOLUTIONS)] - elif profile.ppd_kind == "pwg_resolution_sweep": - dpi = GENERIC_RESOLUTIONS[slot % len(GENERIC_RESOLUTIONS)] - else: - dpi = COVERAGE_RESOLUTIONS[slot % len(COVERAGE_RESOLUTIONS)] - return f"{dpi}x{dpi}dpi" - if any(token in profile.id for token in ("raster", "image", "pdf", "postscript")): - dpi = COVERAGE_RESOLUTIONS[slot % len(COVERAGE_RESOLUTIONS)] - return f"{dpi}x{dpi}dpi" - return "" - - -def _job_options_enabled() -> bool: - return os.environ.get("SMT_FUZZER_DISABLE_JOB_OPTIONS", "").strip().lower() not in { - "1", - "true", - "yes", - "on", - } - - -def build_command( - profile: TargetProfile, - ppd_path: Path, - document_path: Path, - *, - job_options: str = "", -) -> list[str]: - if profile.executor == "cupsfilter": - return [ - "/usr/sbin/cupsfilter", - "-p", - str(ppd_path), - "-e", - "-i", - profile.input_mime, - "-m", - profile.output_mime, - str(document_path), - ] - if profile.executor == "direct_filter": - return [profile.filter_binary, "1", "smt", "smt", "1", job_options, str(document_path)] - raise ValueError(f"unknown executor: {profile.executor}") - - -def format_command(profile: TargetProfile, ppd_path: Path, command: list[str], case_dir: Path | None = None) -> str: - if profile.executor == "direct_filter": - overrides = build_env_overrides(profile, ppd_path, case_dir) - prefix = " ".join(f"{key}={shlex.quote(value)}" for key, value in overrides.items()) - return f"{prefix} {shlex.join(command)}" - return shlex.join(command) - - -def build_env(profile: TargetProfile, ppd_path: Path, case_dir: Path | None = None) -> dict[str, str]: - env = os.environ.copy() - env.update(build_env_overrides(profile, ppd_path, case_dir)) - return env - - -def build_env_overrides(profile: TargetProfile, ppd_path: Path, case_dir: Path | None = None) -> dict[str, str]: - if profile.executor != "direct_filter": - return {} - overrides = {"PPD": str(ppd_path), "SMT_FUZZER_TARGET_ID": profile.id} - if _uses_local_cups_filter(profile.filter_binary): - overrides["LD_LIBRARY_PATH"] = _local_filter_library_path() - overrides["ASAN_OPTIONS"] = os.environ.get( - "ASAN_OPTIONS", - "abort_on_error=0:detect_leaks=0:symbolize=1:exitcode=86", - ) - trace_lib = _dynamic_compare_trace_lib() - if trace_lib and case_dir is not None and _dynamic_compare_trace_enabled(case_dir): - overrides["SMT_FUZZER_COMPARE_TRACE"] = str(case_dir / "compare_trace.tsv") - overrides["SMT_FUZZER_COMPARE_TRACE_LIMIT"] = os.environ.get("SMT_FUZZER_COMPARE_TRACE_LIMIT", "256") - inherited_preload = os.environ.get("LD_PRELOAD", "") - overrides["LD_PRELOAD"] = trace_lib if not inherited_preload else f"{trace_lib}:{inherited_preload}" - asan_options = overrides.get("ASAN_OPTIONS", os.environ.get("ASAN_OPTIONS", "")) - if asan_options: - overrides["ASAN_OPTIONS"] = _append_asan_option(asan_options, "verify_asan_link_order=0") - if _llvm_profiles_enabled(): - if case_dir is not None: - overrides["LLVM_PROFILE_FILE"] = str(case_dir / "llvm.profraw") - else: - profile_dir = os.environ.get("SMT_FUZZER_LLVM_PROFILE_DIR", "work/llvm-profraw") - Path(profile_dir).mkdir(parents=True, exist_ok=True) - overrides["LLVM_PROFILE_FILE"] = str(Path(profile_dir) / f"{profile.id}-%p-%m.profraw") - elif os.environ.get("SMT_FUZZER_LLVM_PROFILE_DIR"): - profile_dir = os.environ["SMT_FUZZER_LLVM_PROFILE_DIR"] - Path(profile_dir).mkdir(parents=True, exist_ok=True) - overrides["LLVM_PROFILE_FILE"] = str(Path(profile_dir) / f"{profile.id}-%p-%m.profraw") - return overrides - - -def _dynamic_compare_trace_lib() -> str: - value = os.environ.get("SMT_FUZZER_DYNAMIC_COMPARE_TRACE_LIB", "").strip() - if not value: - return "" - path = Path(value) - return str(path) if path.exists() else "" - - -def _dynamic_compare_trace_enabled(case_dir: Path) -> bool: - case_id = _case_id_from_dir(case_dir) - max_cases = _positive_int_env("SMT_FUZZER_DYNAMIC_COMPARE_TRACE_MAX_CASES", 0) - if max_cases and case_id >= max_cases: - return False - every = _positive_int_env("SMT_FUZZER_DYNAMIC_COMPARE_TRACE_EVERY", 1) - return case_id % every == 0 - - -def _case_id_from_dir(case_dir: Path) -> int: - match = re.search(r"case-(\d+)$", case_dir.name) - if not match: - return 0 - return int(match.group(1)) - - -def _positive_int_env(name: str, default: int) -> int: - try: - value = int(os.environ.get(name, str(default))) - except ValueError: - return default - return value if value > 0 else default - - -def _append_asan_option(options: str, option: str) -> str: - key = option.split("=", 1)[0] - parts = [part for part in options.split(":") if part] - if any(part == key or part.startswith(f"{key}=") for part in parts): - return options - return ":".join(parts + [option]) - - -def _uses_local_cups_filter(filter_binary: str) -> bool: - return filter_binary.startswith("/data/pre-gsoc/cups-filters") - - -def _local_filter_library_path() -> str: - paths: list[str] = [] - _append_env_or_default_dirs( - paths, - "SMT_FUZZER_LIBPPD_ASAN", - [ - "/data/pre-gsoc/libppd-origin-latest/.libs", - "/data/pre-gsoc/libppd/.libs", - ], - ) - _append_env_or_default_dirs( - paths, - "SMT_FUZZER_LIBCUPSFILTERS_ASAN", - [ - "/data/pre-gsoc/libcupsfilters-master-asan/.libs", - "/data/pre-gsoc/libcupsfilters/.libs", - ], - ) - _append_env_or_default_dirs( - paths, - "SMT_FUZZER_PDFIO_LIB", - [ - "/data/pre-gsoc/env/pdfio-install/lib", - ], - ) - inherited = os.environ.get("LD_LIBRARY_PATH", "") - if inherited: - paths.extend(inherited.split(":")) - return ":".join(_dedupe_paths(paths)) - - -def _append_env_or_default_dirs(paths: list[str], env_name: str, defaults: list[str]) -> None: - env_value = os.environ.get(env_name) - if env_value: - paths.extend(env_value.split(":")) - return - for candidate in defaults: - if Path(candidate).exists(): - paths.append(candidate) - return - - -def _dedupe_paths(paths: list[str]) -> list[str]: - seen: set[str] = set() - result: list[str] = [] - for path in paths: - if not path or path in seen: - continue - seen.add(path) - result.append(path) - return result - - -def classify( - profile: TargetProfile, - stderr_text: str, - returncode: int | None, - timed_out: bool, -) -> tuple[bool, str]: - text = stderr_text.lower() - if timed_out: - return False, "timeout" - if "asan runtime does not come first" in text: - return False, "infra-asan-runtime-order" - if profile.oracle == "rastertopclx_signal_11" and "rastertopclx" in text and "crashed on signal 11" in text: - return True, "rastertopclx signal 11" - if "addresssanitizer" in text or "segmentation fault" in text or "crashed on signal" in text: - return True, "stderr crash/sanitizer" - if returncode is not None and returncode < 0: - return True, f"signal {-returncode}" - if returncode in {86, 134, 139}: - return True, f"returncode {returncode}" - if profile.oracle == "reached_only": - return False, "reached_only" - return False, "" - - -def _process_discovery_result( - result: CaseResult, - root: Path, - discovery_mode: str, - state: DiscoveryState, -) -> dict[str, Any]: - if discovery_mode != "coverage": - _update_target_after_result(state, result, retained=False, new_feature_count=0, new_signature=None) - return {} - - features = extract_case_features(result) - stderr_path = Path(result.stderr_path) - stderr_text = stderr_path.read_text(encoding="utf-8", errors="replace") if stderr_path.exists() else "" - shape_bundle = build_result_shape_bundle(result, stderr_text) - features.update(shape_feature_tokens(shape_bundle)) - new_features = sorted(features - state.seen_features) - state.seen_features.update(features) - retained = bool(new_features) - if retained: - state.retained_cases += 1 - _retain_interesting_case(root, result, features, new_features) - - extra: dict[str, Any] = { - "coverage_feature_count": len(features), - "new_feature_count": len(new_features), - "retained_for_coverage": retained, - "semantic_shape": compact_shape_record(shape_bundle), - "semantic_runtime_key": semantic_runtime_key(result.target_id, str(shape_bundle.get("semantic_input_hash") or "")), - } - if retained: - extra["new_features"] = new_features[:32] - - new_signature: bool | None = None - if result.crashed: - signature = compute_crash_signature(asdict(result), stderr_text) - new_signature = signature not in state.seen_crash_signatures - state.seen_crash_signatures.add(signature) - if new_signature: - state.unique_crashes += 1 - else: - state.repeat_crashes += 1 - runtime_suppressed = False - if state.runtime_skip_enabled: - runtime_suppressed = record_runtime_crash_suppression( - state, - TargetProfile( - id=result.target_id, - description=result.target_description, - ppd_kind=result.ppd_kind, - document_kind=result.document_kind, - executor="", - input_mime="", - output_mime="", - expected_filters=[], - cases=0, - oracle="", - filter_binary="", - ), - result.case_id, - signature, - ) - semantic_runtime_suppressed = record_semantic_crash_suppression( - state, - result, - shape_bundle, - signature, - retained=retained, - ) - else: - semantic_runtime_suppressed = False - _record_quarantine(root, result, signature, new_signature) - extra.update( - { - "crash_signature": signature, - "new_crash_signature": new_signature, - "quarantined_repeat": not new_signature, - "runtime_suppressed_shape": runtime_suppressed, - "runtime_suppressed_semantic_shape": semantic_runtime_suppressed, - } - ) - - _update_target_after_result( - state, - result, - retained=retained, - new_feature_count=len(new_features), - new_signature=new_signature, - ) - return extra - - -def _update_target_after_result( - state: DiscoveryState, - result: CaseResult, - *, - retained: bool, - new_feature_count: int, - new_signature: bool | None, -) -> None: - stats = _target_stats(state, result.target_id) - stats.completed += 1 - state.completed_cases += 1 - if retained: - stats.retained_cases += 1 - stats.new_features += new_feature_count - if result.timed_out: - stats.timeouts += 1 - if result.crashed: - stats.crashes += 1 - if new_signature is True: - stats.unique_crashes += 1 - elif new_signature is False: - stats.repeat_crashes += 1 - - -def extract_case_features(result: CaseResult) -> set[str]: - features = { - f"target:{result.target_id}", - f"ppd:{result.ppd_kind}", - f"document:{result.document_kind}", - f"oracle:{result.oracle or 'none'}", - f"returncode:{result.returncode}", - f"reached:{result.reached_expected_filter}", - } - if result.timed_out: - features.add("timeout") - if result.crashed: - features.add("crash") - features.update(_job_option_features(result.job_options)) - - stderr_path = Path(result.stderr_path) - stderr_text = stderr_path.read_text(encoding="utf-8", errors="replace") if stderr_path.exists() else "" - features.update(_stderr_features(stderr_text)) - features.update(_document_header_features(Path(result.document_path))) - features.update(_llvm_profile_features(result)) - return features - - -def _job_option_features(options: str) -> set[str]: - if not options: - return {"job-options:empty"} - features = {"job-options:present"} - try: - parts = shlex.split(options) - except ValueError: - parts = options.split() - features.add("job-options:parse-error") - for part in parts: - if "=" not in part: - features.add(f"job-option:{part}") - continue - key, value = part.split("=", 1) - if not key: - continue - features.add(f"job-option-key:{key}") - if value: - features.add(f"job-option:{key}={value}") - return features - - -def _stderr_features(stderr_text: str) -> set[str]: - features: set[str] = set() - state_markers = ( - "cffilterimagetopdf:", - "cffilterimagetoraster:", - "ppdfilterimagetops", - "before scaling:", - "using portrait orientation", - "using landscape orientation", - "xpages =", - "ypages =", - "xposition=", - "yposition=", - "pageleft=", - "pageright=", - "pagewidth=", - "pagebottom=", - "pagetop=", - "pagelength=", - "cupswidth =", - "cupsheight =", - "cupsbitspercolor =", - "cupsbitsperpixel =", - "cupsbytesperline =", - "cupscolororder =", - "cupscolorspace =", - "img->colorspace =", - "orientation:", - "formatting page", - ) - for raw_line in stderr_text.splitlines(): - line = raw_line.strip() - if not line: - continue - lowered = line.lower() - if "addresssanitizer" in lowered: - features.add("stderr:asan") - if line.startswith("SUMMARY:"): - features.add("stderr:" + line) - if "fpe" in lowered: - features.add("stderr:fpe") - if "job completed" in lowered: - features.add("stderr:job-completed") - if "no page printed" in lowered: - features.add("stderr:no-page-printed") - if "not an integer multiple" in lowered: - features.add("stderr:resolution-not-multiple") - if "reducing by factor" in lowered: - features.add("stderr:resolution-reducing") - features.add("stderr:" + _normalize_numeric_feature(line)) - if "raising by factor" in lowered: - features.add("stderr:resolution-raising") - features.add("stderr:" + _normalize_numeric_feature(line)) - if "input color mode" in lowered: - features.add("stderr:" + line) - if "dotrowstep" in lowered or "dotrowcount" in lowered or "dotbuffer" in lowered: - features.add("stderr:" + _normalize_numeric_feature(line)) - if line.startswith("PAGE:") or line.startswith("INFO: Finished page"): - features.add("stderr:" + line) - if any(marker in lowered for marker in state_markers): - features.add("stderr-state:" + _normalize_numeric_feature(line)) - return features - - -def _normalize_numeric_feature(line: str) -> str: - return re.sub(r"(? set[str]: - profile_path = Path(result.work_dir) / "llvm.profraw" - if not profile_path.exists(): - return set() - try: - size = profile_path.stat().st_size - except OSError: - return set() - if size <= 0: - return set() - return { - "llvm-profraw:present", - f"llvm-profraw-size:{_size_bucket(size)}", - } - - -def _size_bucket(size: int) -> str: - if size < 1024: - return "lt1k" - if size < 16 * 1024: - return "1k-16k" - if size < 256 * 1024: - return "16k-256k" - if size < 1024 * 1024: - return "256k-1m" - return "ge1m" - - -def _document_header_features(document_path: Path) -> set[str]: - if not document_path.exists(): - return set() - data = document_path.read_bytes() - if data.startswith(b"%PDF-"): - return _pdf_features(data) - if data.startswith(b"%!PS"): - return _postscript_features(data) - if data.startswith(b"\x89PNG\r\n\x1a\n"): - return _png_features(data) - if len(data) >= 2 and data[:1] == b"P" and data[1:2] in {b"1", b"2", b"3", b"4", b"5", b"6"}: - return _pnm_features(data) - if data.startswith(b"#CUPS-COMMAND"): - return { - "doc-format:cups-command", - f"doc-size:{len(data)}", - f"doc-lines:{data.count(bytes([10]))}", - } - sync = data[:4].decode("latin1", errors="replace") - if sync in {"3SaR", "2SaR"}: - return _raster_features(data, sync) - if data.startswith(b"%") or data[:32].isascii(): - return { - "doc-format:text-like", - f"doc-size:{len(data)}", - f"doc-lines:{data.count(bytes([10]))}", - } - if len(data) < 4 + 424: - return {f"doc-size:{len(data)}"} - return {f"doc-format:binary", f"doc-sync:{sync}", f"doc-size:{len(data)}"} - - -def _raster_features(data: bytes, sync: str) -> set[str]: - if len(data) < 4 + 424: - return {f"doc-sync:{sync}", "doc-header:short", f"doc-size:{len(data)}"} - header = data[4 : 4 + 1796] - try: - x_res = _u32(header, OFF_HW_RESOLUTION) - y_res = _u32(header, OFF_HW_RESOLUTION + 4) - width = _u32(header, OFF_CUPS_WIDTH) - height = _u32(header, OFF_CUPS_HEIGHT) - bpp = _u32(header, OFF_CUPS_BITS_PER_PIXEL) - bpl = _u32(header, OFF_CUPS_BYTES_PER_LINE) - order = _u32(header, OFF_CUPS_COLOR_ORDER) - color_space = _u32(header, OFF_CUPS_COLOR_SPACE) - compression = _u32(header, OFF_CUPS_COMPRESSION) - row_count = _u32(header, OFF_CUPS_ROW_COUNT) - except struct.error: - return {f"doc-sync:{sync}", "doc-header:short"} - return { - f"doc-sync:{sync}", - f"doc-res:{x_res}x{y_res}", - f"doc-size:{width}x{height}", - f"doc-bpp:{bpp}", - f"doc-bpl-bucket:{_bucket(bpl)}", - f"doc-color-order:{order}", - f"doc-color-space:{color_space}", - f"doc-compression:{compression}", - f"doc-row-count:{row_count}", - } - - -def _pdf_features(data: bytes) -> set[str]: - return { - "doc-format:pdf", - f"doc-size:{len(data)}", - f"doc-pdf-version:{data[:8].decode('latin1', errors='replace')}", - f"doc-pdf-objects:{data.count(b' obj')}", - f"doc-pdf-streams:{data.count(b'stream')}", - } - - -def _postscript_features(data: bytes) -> set[str]: - return { - "doc-format:postscript", - f"doc-size:{len(data)}", - f"doc-lines:{data.count(bytes([10]))}", - f"doc-ps-showpage:{data.count(b'showpage')}", - } - - -def _png_features(data: bytes) -> set[str]: - if len(data) < 33 or data[12:16] != b"IHDR": - return {"doc-format:png", "doc-png:short", f"doc-size:{len(data)}"} - width, height = struct.unpack(">II", data[16:24]) - bit_depth = data[24] - color_type = data[25] - return { - "doc-format:png", - f"doc-size:{len(data)}", - f"doc-image-size:{width}x{height}", - f"doc-png-bit-depth:{bit_depth}", - f"doc-png-color-type:{color_type}", - } - - -def _pnm_features(data: bytes) -> set[str]: - tokens = [] - for raw_line in data.splitlines(): - line = raw_line.split(b"#", 1)[0].strip() - if not line: - continue - tokens.extend(line.split()) - if len(tokens) >= 4: - break - magic = tokens[0].decode("ascii", errors="replace") if tokens else "P?" - width = tokens[1].decode("ascii", errors="replace") if len(tokens) > 1 else "?" - height = tokens[2].decode("ascii", errors="replace") if len(tokens) > 2 else "?" - return { - "doc-format:pnm", - f"doc-size:{len(data)}", - f"doc-pnm-magic:{magic}", - f"doc-image-size:{width}x{height}", - } - - -def _u32(buffer: bytes, offset: int) -> int: - return struct.unpack_from(" str: - if value < 8: - return "lt8" - if value < 64: - return "lt64" - if value < 512: - return "lt512" - if value < 4096: - return "lt4096" - return "ge4096" - - -def _retain_interesting_case(root: Path, result: CaseResult, features: set[str], new_features: list[str]) -> None: - dest = root / "corpus" / "interesting" / result.target_id / f"case-{result.case_id:06d}" - dest.mkdir(parents=True, exist_ok=True) - for source_name in ["candidate.ppd", "command.txt", "stderr.txt", "meta.json"]: - source = Path(result.work_dir) / source_name - if source.exists(): - shutil.copy2(source, dest / source_name) - document_source = Path(result.document_path) - if document_source.exists(): - shutil.copy2(document_source, dest / document_source.name) - (dest / "features.json").write_text( - json.dumps( - { - "features": sorted(features), - "new_features": new_features, - "source_work_dir": result.work_dir, - }, - indent=2, - sort_keys=True, - ) - + "\n", - encoding="utf-8", - ) - - -def _record_quarantine(root: Path, result: CaseResult, signature: str, new_signature: bool) -> None: - quarantine = root / "quarantine" - quarantine.mkdir(parents=True, exist_ok=True) - record = { - "target_id": result.target_id, - "case_id": result.case_id, - "signature": signature, - "new_signature": new_signature, - "work_dir": result.work_dir, - "command_line": result.command_line, - "job_options": result.job_options, - "stderr_path": result.stderr_path, - } - with (quarantine / ("unique.jsonl" if new_signature else "repeats.jsonl")).open("a", encoding="utf-8") as handle: - handle.write(json.dumps(record, sort_keys=True) + "\n") - if new_signature: - dest = quarantine / "unique" / f"{result.target_id}-case-{result.case_id:06d}" - dest.mkdir(parents=True, exist_ok=True) - for source_name in ["candidate.ppd", "command.txt", "stderr.txt", "meta.json"]: - source = Path(result.work_dir) / source_name - if source.exists(): - shutil.copy2(source, dest / source_name) - document_source = Path(result.document_path) - if document_source.exists(): - shutil.copy2(document_source, dest / document_source.name) - - -def _write_run_records(result: CaseResult, timeline, commands, extra: dict[str, Any] | None = None) -> None: - commands.write(f"{result.target_id} case-{result.case_id:04d}: {result.command_line}\n") - commands.flush() - record = { - "target_id": result.target_id, - "case_id": result.case_id, - "ppd_kind": result.ppd_kind, - "document_kind": result.document_kind, - "document_description": result.document_description, - "command_line": result.command_line, - "job_options": result.job_options, - "env_overrides": result.env_overrides, - "returncode": result.returncode, - "timed_out": result.timed_out, - "crashed": result.crashed, - "oracle": result.oracle, - "duration_ms": result.duration_ms, - "cupstestppd_ok": result.cupstestppd_ok, - "reached_expected_filter": result.reached_expected_filter, - "filters": result.filters, - "work_dir": result.work_dir, - "stderr_path": result.stderr_path, - "stderr_tail": result.stderr_tail, - } - if extra: - record.update(extra) - timeline.write(json.dumps(record, sort_keys=True) + "\n") - timeline.flush() - - -def _write_skip_record( - profile: TargetProfile, - case_id: int, - case_dir: Path, - skip_reason: str, - timeline, - commands, -) -> None: - commands.write(f"{profile.id} case-{case_id:04d}: SKIP {skip_reason}\n") - commands.flush() - timeline.write( - json.dumps( - { - "target_id": profile.id, - "case_id": case_id, - "ppd_kind": profile.ppd_kind, - "document_kind": profile.document_kind, - "skipped": True, - "skip_reason": skip_reason, - "crashed": False, - "timed_out": False, - "oracle": "skipped-known-shallow-crash", - "work_dir": str(case_dir), - }, - sort_keys=True, - ) - + "\n" - ) - timeline.flush() - - -def _tail_lines(text: str, limit: int) -> list[str]: - lines = [line for line in text.splitlines() if line.strip()] - return lines[-limit:] diff --git a/parser-fuzzers/src/parser_fuzzers/semantic_shapes.py b/parser-fuzzers/src/parser_fuzzers/semantic_shapes.py deleted file mode 100644 index 63a2ddd..0000000 --- a/parser-fuzzers/src/parser_fuzzers/semantic_shapes.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.feedback.semantic_shapes`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.feedback.semantic_shapes") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/solver.py b/parser-fuzzers/src/parser_fuzzers/solver.py deleted file mode 100644 index 974d36d..0000000 --- a/parser-fuzzers/src/parser_fuzzers/solver.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.solver`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.solver") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/source_constraints.py b/parser-fuzzers/src/parser_fuzzers/source_constraints.py deleted file mode 100644 index c1fe190..0000000 --- a/parser-fuzzers/src/parser_fuzzers/source_constraints.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.source_constraints`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.source_constraints") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/structure_mutator.py b/parser-fuzzers/src/parser_fuzzers/structure_mutator.py deleted file mode 100644 index edf4202..0000000 --- a/parser-fuzzers/src/parser_fuzzers/structure_mutator.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.structure_mutator`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.structure_mutator") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/structured_templates.py b/parser-fuzzers/src/parser_fuzzers/structured_templates.py deleted file mode 100644 index ab2ef97..0000000 --- a/parser-fuzzers/src/parser_fuzzers/structured_templates.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.structured_templates`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.structured_templates") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/template_feedback.py b/parser-fuzzers/src/parser_fuzzers/template_feedback.py deleted file mode 100644 index a2811be..0000000 --- a/parser-fuzzers/src/parser_fuzzers/template_feedback.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.feedback.template_feedback`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.feedback.template_feedback") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/template_synth.py b/parser-fuzzers/src/parser_fuzzers/template_synth.py deleted file mode 100644 index 29ca357..0000000 --- a/parser-fuzzers/src/parser_fuzzers/template_synth.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.template_synth`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.template_synth") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/validation.py b/parser-fuzzers/src/parser_fuzzers/validation.py deleted file mode 100644 index 71b09ca..0000000 --- a/parser-fuzzers/src/parser_fuzzers/validation.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.core.validation`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.core.validation") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/src/parser_fuzzers/z3_guard.py b/parser-fuzzers/src/parser_fuzzers/z3_guard.py deleted file mode 100644 index 49706c2..0000000 --- a/parser-fuzzers/src/parser_fuzzers/z3_guard.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for :mod:`parser_fuzzers.generator.z3_guard`.""" - -from __future__ import annotations - -import sys as _sys -from importlib import import_module as _import_module - -_impl = _import_module("parser_fuzzers.generator.z3_guard") -_sys.modules[__name__] = _impl diff --git a/parser-fuzzers/targets.sh b/parser-fuzzers/targets.sh new file mode 100755 index 0000000..afd90be --- /dev/null +++ b/parser-fuzzers/targets.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +CF_CORE12_TARGETS=( + fuzz_cupsfilters_format_cups_raster + fuzz_cupsfilters_format_image_jpeg_bounded + fuzz_cupsfilters_format_image_png_bounded + fuzz_cupsfilters_format_image_tiff_bounded + fuzz_cupsfilters_state_pwg_to_raster_scale_down + fuzz_cupsfilters_state_pwg_to_raster_scale_up + fuzz_cupsfilters_state_raster_to_apple + fuzz_cupsfilters_state_raster_to_pwg + fuzz_cupsfilters_raster_to_pclx_mode3_codec + fuzz_cupsfilters_raster_to_pclx_mode10_codec + fuzz_cupsfilters_state_text_to_text_layout + fuzz_cupsfilters_text_to_text_selection_oracle +) + +core12_list_targets() { + printf '%s\n' "${CF_CORE12_TARGETS[@]}" +} + +core12_has_target() { + local requested="$1" target + for target in "${CF_CORE12_TARGETS[@]}"; do + [[ "$target" == "$requested" ]] && return 0 + done + return 1 +} + +core12_target_max_len() { + case "$1" in + *format_cups_raster) printf '%s\n' $((4 * 1024 * 1024)) ;; + *format_image_*) printf '%s\n' $((2 * 1024 * 1024)) ;; + *state_pwg_to_raster_scale_*) printf '%s\n' 4111 ;; + *state_raster_to_*) printf '%s\n' 4112 ;; + *raster_to_pclx_mode3_codec|*raster_to_pclx_mode10_codec) + printf '%s\n' 4116 ;; + *state_text_to_text_layout) printf '%s\n' 4120 ;; + *text_to_text_selection_oracle) printf '%s\n' 280 ;; + *) return 2 ;; + esac +} + +core12_target_rss_limit_mb() { + case "$1" in + *state_pwg_to_raster_scale_*|*state_raster_to_*) printf '%s\n' 768 ;; + *) printf '%s\n' 1024 ;; + esac +} + +core12_target_timeout_sec() { + core12_has_target "$1" || return 2 + printf '%s\n' 15 +} + +core12_target_detect_leaks() { + core12_has_target "$1" || return 2 + printf '%s\n' 1 +} diff --git a/parser-fuzzers/tests/test_afl.py b/parser-fuzzers/tests/test_afl.py deleted file mode 100644 index 38ae263..0000000 --- a/parser-fuzzers/tests/test_afl.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -import sys -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from parser_fuzzers.afl import afl_build_env, build_afl_plan - - -class AFLTests(unittest.TestCase): - def test_afl_build_env_uses_afl_wrappers(self) -> None: - env = afl_build_env(ROOT / "configs") - self.assertEqual(env["CC"], "afl-clang-fast") - self.assertEqual(env["CXX"], "afl-clang-fast++") - self.assertIn("address", env["CFLAGS"]) - - def test_dictionary_config_materializes_merged_dictionary(self) -> None: - plan = build_afl_plan( - ROOT, - "configs", - "ppd_ipp_parser", - "A1", - "harnesses/bin/ppd_ipp_parser", - ) - self.assertIsNotNone(plan.dictionary) - dictionary = Path(plan.dictionary or "") - self.assertTrue(dictionary.exists()) - self.assertIn("-x", plan.argv) - self.assertIn('"*PPD-Adobe:"', dictionary.read_text(encoding="utf-8")) - self.assertFalse(any("/private/local/reproducer" in arg for arg in plan.argv)) - - def test_cmplog_config_adds_cmplog_binary(self) -> None: - plan = build_afl_plan( - ROOT, - "configs", - "ppd_ipp_parser", - "A2", - "harnesses/bin/ppd_ipp_parser", - ) - self.assertIn("-c", plan.argv) - self.assertEqual(plan.cmplog_binary, "harnesses/bin/ppd_ipp_parser.cmplog") - self.assertTrue(any("CmpLog config requested" in warning for warning in plan.warnings)) - - def test_smt_config_imports_smt_corpus(self) -> None: - smt_dir = ROOT / "work" / "corpus" / "smt" - smt_dir.mkdir(parents=True, exist_ok=True) - (smt_dir / "unit-smt-input.bin").write_bytes(b"SMT") - plan = build_afl_plan( - ROOT, - "configs", - "image_options_parser", - "A4", - "harnesses/bin/image_options_parser", - ) - input_dir = Path(plan.input_dir) - self.assertTrue((input_dir / "smt-unit-smt-input.bin").exists()) - self.assertIsNotNone(plan.dictionary) - - def test_standard_afl_plan_accepts_template_seed_directory(self) -> None: - seed_dir = ROOT / "seeds" / "public" - output_dir = ROOT / "work" / "unit-afl-output" - plan = build_afl_plan( - ROOT, - "configs", - "template_probe_pwg", - "A1", - "work/afl/bin/template_probe", - input_dir=seed_dir, - output_dir=output_dir, - duration_sec=123, - timeout_ms=777, - memory_mb=256, - ) - self.assertEqual(plan.input_dir, str(seed_dir)) - self.assertEqual(plan.output_dir, str(output_dir)) - self.assertIn("-V", plan.argv) - self.assertIn("123", plan.argv) - self.assertIn("-t", plan.argv) - self.assertIn("777", plan.argv) - self.assertNotIn("-n", plan.argv) - self.assertEqual(plan.argv[-2:], ["work/afl/bin/template_probe", "@@"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_afl_dynamic_bridge.py b/parser-fuzzers/tests/test_afl_dynamic_bridge.py deleted file mode 100644 index 9c67dc1..0000000 --- a/parser-fuzzers/tests/test_afl_dynamic_bridge.py +++ /dev/null @@ -1,92 +0,0 @@ -from __future__ import annotations - -import json -import tempfile -import unittest -from pathlib import Path - -from parser_fuzzers.afl_dynamic_bridge import ( - MAGIC, - DOCUMENT_MARK, - OPTIONS_MARK, - PPD_MARK, - augment_pwg_bundle_seed_dir, - parse_pwg_bundle, - write_dynamic_afl_dictionary, -) - - -class AFLDynamicBridgeTests(unittest.TestCase): - def test_writes_dynamic_dictionary(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - profile = root / "dynamic.json" - base = root / "base.dict" - out = root / "dynamic.dict" - profile.write_text( - json.dumps( - { - "schema_version": "dynamic-compare-hints-v1", - "tokens": {"PageSize": 5, "application/vnd.cups-pwg": 3}, - "ppd_options": {"ColorModel": 2}, - "magic_tokens": {"RaS2": 1}, - "records": [], - } - ), - encoding="utf-8", - ) - base.write_text('"--SMT-PPD--"\n', encoding="utf-8") - - manifest = write_dynamic_afl_dictionary(profile, out, base_dictionary=base) - content = out.read_text(encoding="utf-8") - - self.assertGreaterEqual(manifest["tokens_added"], 3) - self.assertIn('"PageSize"', content) - self.assertIn('"PageSize="', content) - self.assertIn('"application/vnd.cups-pwg"', content) - self.assertIn('"ColorModel"', content) - - def test_augments_pwg_bundle_seeds(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - seed_dir = root / "seeds" - seed_dir.mkdir() - seed = seed_dir / "seed-0000.pwg-bundle" - seed.write_bytes( - MAGIC - + PPD_MARK - + b"*PPD-Adobe: \"4.3\"\n" - + OPTIONS_MARK - + b"PageSize=Letter" - + DOCUMENT_MARK - + b"RaS2\x00\x00" - ) - profile = root / "dynamic.json" - profile.write_text( - json.dumps( - { - "schema_version": "dynamic-compare-hints-v1", - "tokens": {"cupsBitsPerPixel": 4}, - "ppd_options": {"cupsBitsPerPixel": 4}, - "magic_tokens": {}, - "records": [], - } - ), - encoding="utf-8", - ) - - manifest = augment_pwg_bundle_seed_dir(seed_dir, profile, limit=2) - generated = sorted(seed_dir.glob("dynamic-option-*.pwg-bundle")) - parsed = parse_pwg_bundle(generated[0].read_bytes()) - - self.assertGreaterEqual(manifest["created"], 1) - self.assertIsNotNone(parsed) - assert parsed is not None - ppd, options, document = parsed - self.assertIn(b"*cupsBitsPerPixel: 24", ppd) - self.assertIn(b"cupsBitsPerPixel=24", options) - self.assertEqual(document, b"RaS2\x00\x00") - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_afl_feedback.py b/parser-fuzzers/tests/test_afl_feedback.py deleted file mode 100644 index 1dce7bf..0000000 --- a/parser-fuzzers/tests/test_afl_feedback.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import annotations - -import sys -import tempfile -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from parser_fuzzers.afl_feedback import import_afl_artifacts, resolve_afl_instance_dir - - -class AFLFeedbackImportTests(unittest.TestCase): - def test_imports_queue_and_crashes_as_runner_feedback_cases(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - instance = root / "out" / "default" - queue = instance / "queue" - crashes = instance / "crashes" - queue.mkdir(parents=True) - crashes.mkdir(parents=True) - (queue / "id:000000,orig:seed").write_bytes(b"2SaRqueue") - (queue / "id:000001,dup").write_bytes(b"2SaRqueue") - (crashes / "id:000000,sig:11").write_bytes(b"2SaRcrash") - (crashes / "README.txt").write_text("metadata\n", encoding="utf-8") - - output = root / "feedback" - summary = import_afl_artifacts( - afl_out=root, - target_id="pwg_to_pdf_feedback_f30", - output_run_dir=output, - extension=".pwg", - queue_limit=16, - crash_limit=16, - queue_mode="all", - ) - - self.assertEqual(summary.queue_imported, 1) - self.assertEqual(summary.crashes_imported, 1) - self.assertEqual(summary.duplicates_skipped, 1) - self.assertTrue( - ( - output - / "corpus" - / "interesting" - / "pwg_to_pdf_feedback_f30" - / "case-000000" - / "meta.json" - ).exists() - ) - self.assertTrue( - ( - output - / "quarantine" - / "unique" - / "pwg_to_pdf_feedback_f30-afl-case-000001" - / "document.pwg" - ).exists() - ) - self.assertTrue((output / "timeline.jsonl").exists()) - - def test_default_import_skips_original_seed_queue_entries(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - queue = root / "out" / "queue" - crashes = root / "out" / "crashes" - queue.mkdir(parents=True) - crashes.mkdir(parents=True) - (queue / "id:000000,time:0,execs:0,orig:seed").write_bytes(b"2SaRseed") - (queue / "id:000001,src:000000,time:10,execs:7,op:havoc").write_bytes(b"2SaRnew") - - summary = import_afl_artifacts( - afl_out=root / "out", - target_id="pwg_to_pdf_feedback_f30", - output_run_dir=root / "feedback", - extension=".pwg", - ) - - self.assertEqual(summary.queue_mode, "new") - self.assertEqual(summary.queue_imported, 1) - imported_sources = [item.source_path for item in summary.imported] - self.assertTrue(any("src:000000" in path for path in imported_sources)) - self.assertFalse(any("orig:seed" in path for path in imported_sources)) - - def test_resolves_campaign_or_out_dir(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - (root / "out" / "default" / "queue").mkdir(parents=True) - self.assertEqual(resolve_afl_instance_dir(root), root / "out" / "default") - self.assertEqual(resolve_afl_instance_dir(root / "out"), root / "out" / "default") - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_arithmetic_explorer.py b/parser-fuzzers/tests/test_arithmetic_explorer.py deleted file mode 100644 index e7c551e..0000000 --- a/parser-fuzzers/tests/test_arithmetic_explorer.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -import sys -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from parser_fuzzers.arithmetic_explorer import MOD32, iter_arithmetic_params - - -class ArithmeticExplorerTests(unittest.TestCase): - def test_generator_produces_valid_cross_input_relations(self) -> None: - params = [next(iter_arithmetic_params()) for _ in range(3)] - for item in params: - self.assertEqual(item.input_y_dpi % item.output_dpi, 0) - self.assertGreaterEqual(item.bytes_per_line, 1) - - def test_generator_reaches_32bit_product_boundary(self) -> None: - iterator = iter_arithmetic_params() - boundary = None - for _ in range(2000): - item = next(iterator) - if item.bytes_per_line * item.y_factor >= MOD32: - boundary = item - break - self.assertIsNotNone(boundary) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_auto_expand.py b/parser-fuzzers/tests/test_auto_expand.py deleted file mode 100644 index ecf45ce..0000000 --- a/parser-fuzzers/tests/test_auto_expand.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import annotations - -import json -import tempfile -import unittest -from pathlib import Path - -from parser_fuzzers.auto_expand import build_auto_expand_plan, discover_campaign_runs, write_auto_expand_plan -from parser_fuzzers.document_harness import make_document - - -class AutoExpandTests(unittest.TestCase): - def test_auto_expand_builds_frontier_profile_and_plan(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - run_dir = root / "work" / "feedback-campaign" / "20260602-000000" - case_dir = run_dir / "corpus" / "interesting" / "cups_target" / "case-000001" - case_dir.mkdir(parents=True) - document_path = case_dir / "document.ras" - document_path.write_bytes(make_document("cups_raster_structural_sweep", 1).data) - (case_dir / "meta.json").write_text( - json.dumps( - { - "target_id": "cups_target", - "case_id": 1, - "document_path": str(document_path), - "crashed": False, - "timed_out": False, - } - ), - encoding="utf-8", - ) - (run_dir / "summary.concise.json").write_text( - json.dumps( - { - "target_stats": { - "cups_target": { - "completed": 1000, - "retained_cases": 0, - "skipped": 0, - "timeouts": 0, - "crashes": 0, - }, - "suppressed_target": { - "completed": 0, - "retained_cases": 0, - "skipped": 200, - "timeouts": 0, - "crashes": 0, - }, - } - } - ), - encoding="utf-8", - ) - with (run_dir / "timeline.jsonl").open("w", encoding="utf-8") as handle: - for case_id in range(10): - handle.write( - json.dumps( - { - "target_id": "cups_target", - "case_id": case_id, - "retained_for_coverage": False, - "new_feature_count": 0, - "crashed": False, - "timed_out": False, - } - ) - + "\n" - ) - - output_profile = root / "work" / "template-feedback" / "auto.json" - plan_output = root / "work" / "template-feedback" / "auto-plan.json" - plan = build_auto_expand_plan( - search_root=root / "work", - output_profile=output_profile, - stale_window=10, - duration_sec=60, - ) - write_auto_expand_plan(plan, plan_output) - - self.assertEqual(discover_campaign_runs(root / "work"), [run_dir]) - self.assertTrue(output_profile.exists()) - self.assertTrue(plan_output.exists()) - self.assertEqual(plan.expansion_level, 2) - self.assertEqual(plan.profile_cups, 1) - self.assertIn("SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL=2", plan.recommended_command) - self.assertIn("--skip-probe-rate 0.01", plan.recommended_command) - self.assertEqual(plan.target_actions["suppressed_target"], "probe-runtime-suppressed-family") - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_baseline_compare.py b/parser-fuzzers/tests/test_baseline_compare.py deleted file mode 100644 index 560ea2f..0000000 --- a/parser-fuzzers/tests/test_baseline_compare.py +++ /dev/null @@ -1,100 +0,0 @@ -from __future__ import annotations - -import tempfile -import unittest -from pathlib import Path - -from parser_fuzzers.baseline_compare import ( - build_comparison_payload, - inspect_oss_fuzz_cups_filters, - render_comparison_markdown, -) - - -class BaselineCompareTests(unittest.TestCase): - def test_inspect_oss_fuzz_project_reads_dockerfile_build_script_source(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - project = Path(tmp) / "projects" / "cups-filters" - project.mkdir(parents=True) - (project / "project.yaml").write_text("language: c++\n", encoding="utf-8") - (project / "Dockerfile").write_text( - "FROM gcr.io/oss-fuzz-base/base-builder\n" - "RUN cp $SRC/fuzzing/projects/cups-filters/oss_fuzz_build.sh $SRC/build.sh\n", - encoding="utf-8", - ) - (project / "run_tests.sh").write_text("make check\n", encoding="utf-8") - - status = inspect_oss_fuzz_cups_filters(tmp) - - self.assertTrue(status.project_exists) - self.assertFalse(status.has_project_build_sh) - self.assertIn("oss_fuzz_build.sh", status.build_sh_source) - self.assertEqual(len(status.helper_commands), 4) - - def test_build_comparison_payload_computes_selected_delta(self) -> None: - baseline = { - "run_dir": "base", - "run": { - "elapsed_sec": 60, - "cases": 10, - "retained_cases": 4, - "coverage_features": 20, - "crashes": 2, - "unique_crashes": 1, - }, - "derived": { - "features_per_min": 20.0, - "retained_density": 0.4, - "crash_density": 0.2, - }, - "llvm_cov": { - "totals": { - "functions": {"covered": 5, "percent": 10.0}, - "lines": {"covered": 50, "percent": 12.5}, - "branches": {"covered": 7, "percent": 3.5}, - } - }, - } - optimized = { - "run_dir": "opt", - "run": { - "elapsed_sec": 60, - "cases": 12, - "retained_cases": 8, - "coverage_features": 36, - "crashes": 1, - "unique_crashes": 1, - }, - "derived": { - "features_per_min": 36.0, - "retained_density": 0.666667, - "crash_density": 0.083333, - }, - "llvm_cov": { - "totals": { - "functions": {"covered": 8, "percent": 16.0}, - "lines": {"covered": 70, "percent": 17.5}, - "branches": {"covered": 9, "percent": 4.5}, - } - }, - } - with tempfile.TemporaryDirectory() as tmp: - status = inspect_oss_fuzz_cups_filters(tmp) - - payload = build_comparison_payload( - comparison_id="cmp", - config_path="config.yaml", - baseline=baseline, - optimized=optimized, - oss_fuzz_status=status, - ) - markdown = render_comparison_markdown(payload) - - self.assertEqual(payload["metrics"]["delta"]["coverage_features"], 16.0) - self.assertEqual(payload["metrics"]["delta"]["llvm_functions_percent"], 6.0) - self.assertIn("Local Fair Comparison", markdown) - self.assertIn("features/min", markdown) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_coverage_discovery.py b/parser-fuzzers/tests/test_coverage_discovery.py deleted file mode 100644 index cb392fa..0000000 --- a/parser-fuzzers/tests/test_coverage_discovery.py +++ /dev/null @@ -1,602 +0,0 @@ -from __future__ import annotations - -import json -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -from parser_fuzzers.multitarget_runner import ( - DiscoveryState, - TargetDiscoveryStats, - TargetProfile, - _combined_skip_reason, - _coverage_stagnated, - _choose_next_profile, - _is_novel_discovery, - _run_dir_size_bytes, - _target_scheduler_score, - build_command, - case_family_key, - case_hazard_key, - case_shape_key, - build_env_overrides, - build_job_options, - coverage_skip_reason, - extract_case_features, - find_latest_runtime_skip_state, - load_profiles, - load_runtime_skip_state, - record_runtime_crash_suppression, - remap_filter_binary, - run_case, - runtime_skip_reason, -) -from parser_fuzzers.image_templates import image_feedback_instance - - -class CoverageDiscoveryTests(unittest.TestCase): - def test_filter_root_remaps_direct_filter_binaries(self) -> None: - self.assertEqual( - remap_filter_binary("/data/pre-gsoc/cups-filters/pwgtopdf", "/tmp/filters"), - "/tmp/filters/pwgtopdf", - ) - - def test_load_profiles_applies_filter_root_only_to_direct_filters(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - config = Path(tmp) / "targets.yaml" - config.write_text( - """ -targets: - - id: direct - ppd_kind: pwgtopdf_plain - document_kind: pwg_raster_feedback_sweep - executor: direct_filter - filter_binary: /data/pre-gsoc/cups-filters/pwgtopdf - - id: routed - ppd_kind: rastertopclx - document_kind: text - executor: cupsfilter -""", - encoding="utf-8", - ) - - direct, routed = load_profiles(config, filter_root="/tmp/filters") - - self.assertEqual(direct.filter_binary, "/tmp/filters/pwgtopdf") - self.assertEqual(routed.filter_binary, "") - - def test_skips_known_rastertoescpx_dotrowstep_zero_cases(self) -> None: - profile = _profile("cups_raster_to_rastertoescpx_general", "cups_raster_general_sweep", "rastertoescpx_size_sweep") - - self.assertEqual(coverage_skip_reason(profile, 3), "known-rastertoescpx-dotrowstep-zero-fpe") - self.assertEqual(coverage_skip_reason(profile, 15), "known-rastertoescpx-dotrowstep-zero-fpe") - self.assertEqual(coverage_skip_reason(profile, 0), "") - - def test_skips_known_libppd_65536dpi_cases(self) -> None: - profile = _profile("pwg_to_raster_general", "pwg_raster_general_sweep", "pwg_resolution_general") - - self.assertEqual(coverage_skip_reason(profile, 16), "known-libppd-65536dpi-fpe") - self.assertEqual(coverage_skip_reason(profile, 33), "known-libppd-65536dpi-fpe") - self.assertEqual(coverage_skip_reason(profile, 15), "") - - def test_extract_case_features_includes_document_header(self) -> None: - profile = _profile("feature_test", "cups_raster_coverage_sweep", "rastertopclx_plain") - with tempfile.TemporaryDirectory() as tmp: - result = run_case(profile, 0, Path(tmp), timeout_sec=1, capture_stdout=False) - features = extract_case_features(result) - - self.assertIn("target:feature_test", features) - self.assertIn("doc-sync:3SaR", features) - self.assertTrue(any(feature.startswith("doc-size:") for feature in features)) - - def test_extract_case_features_recognizes_pdf_and_png(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - pdf_result = run_case( - _profile("pdf_feature_test", "pdf_coverage_sweep", "pdftopdf_coverage_options"), - 0, - Path(tmp) / "pdf", - timeout_sec=1, - capture_stdout=False, - ) - png_result = run_case( - _profile("png_feature_test", "image_coverage_sweep", "imagetoraster_coverage_options"), - 0, - Path(tmp) / "png", - timeout_sec=1, - capture_stdout=False, - ) - - pdf_features = extract_case_features(pdf_result) - png_features = extract_case_features(png_result) - - self.assertIn("doc-format:pdf", pdf_features) - self.assertIn("doc-format:png", png_features) - self.assertTrue(any(feature.startswith("doc-image-size:") for feature in png_features)) - - def test_runtime_skip_suppresses_seen_crash_shape(self) -> None: - profile = _profile("image_to_imagetoraster_coverage", "image_coverage_sweep", "imagetoraster_coverage_options") - state = DiscoveryState(runtime_skip_enabled=True, crash_skip_after=1) - signature = "SUMMARY: AddressSanitizer: heap-buffer-overflow example/image_scale.c:123 in sample_scale" - - self.assertEqual(runtime_skip_reason(profile, 1, state), "") - self.assertTrue(record_runtime_crash_suppression(state, profile, 1, signature)) - - self.assertEqual(case_shape_key(profile, 1), case_shape_key(profile, 721)) - self.assertTrue(runtime_skip_reason(profile, 721, state).startswith("runtime-known-crash-shape:")) - self.assertEqual(runtime_skip_reason(profile, 2, state), "") - - def test_image_feedback_epoch_changes_runtime_skip_shape_when_enabled(self) -> None: - profile = _profile("image_to_imagetoraster_feedback", "image_feedback_sweep", "imagetoraster_coverage_options") - state = DiscoveryState(runtime_skip_enabled=True, crash_skip_after=1) - - with patch.dict("os.environ", {"SMT_FUZZER_IMAGE_CYCLE_EPOCHS": "8"}, clear=False): - self.assertNotEqual(case_shape_key(profile, 1), case_shape_key(profile, 721)) - self.assertTrue(record_runtime_crash_suppression(state, profile, 1, "sig")) - self.assertEqual(runtime_skip_reason(profile, 721, state), "") - - def test_image_feedback_epoch_changes_generated_instance_when_enabled(self) -> None: - with patch.dict( - "os.environ", - { - "SMT_FUZZER_IMAGE_CYCLE_EPOCHS": "8", - "SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL": "2", - }, - clear=False, - ): - first = image_feedback_instance(1) - second = image_feedback_instance(721) - - self.assertNotEqual( - ( - first.image_format, - first.width, - first.height, - first.payload_delta, - first.comment_style, - first.objective, - ), - ( - second.image_format, - second.width, - second.height, - second.payload_delta, - second.comment_style, - second.objective, - ), - ) - - def test_runtime_skip_suppresses_repeated_image_hazard(self) -> None: - profile = _profile("image_to_imagetops_feedback", "image_feedback_sweep", "imagetops_coverage_options") - state = DiscoveryState(runtime_skip_enabled=True, crash_skip_after=1, hazard_skip_after=2) - - with patch.dict( - "os.environ", - { - "SMT_FUZZER_IMAGE_CYCLE_EPOCHS": "8", - "SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL": "2", - }, - clear=False, - ): - target_hazard = case_hazard_key(profile, 1) - matching = [ - index - for index in range(2, 2000) - if case_hazard_key(profile, index) == target_hazard - ] - self.assertGreaterEqual(len(matching), 2) - self.assertTrue(record_runtime_crash_suppression(state, profile, 1, "sig")) - self.assertEqual(runtime_skip_reason(profile, matching[0], state), "") - self.assertTrue(record_runtime_crash_suppression(state, profile, matching[0], "sig")) - - skipped = _combined_skip_reason(profile, matching[1], "coverage", state, skip_probe_rate=0.0) - probed = _combined_skip_reason(profile, matching[1], "coverage", state, skip_probe_rate=1.0) - - self.assertTrue(skipped.startswith("runtime-known-crash-hazard:")) - self.assertEqual(probed, "") - - def test_runtime_skip_can_seed_previous_discovery_state(self) -> None: - profile = _profile("image_to_imagetoraster_coverage", "image_coverage_sweep", "imagetoraster_coverage_options") - signature = "SUMMARY: AddressSanitizer: heap-buffer-overflow example/image_scale.c:123 in sample_scale" - - with tempfile.TemporaryDirectory() as tmp: - state_path = Path(tmp) / "discovery_state.json" - state_path.write_text( - json.dumps( - { - "suppressed_case_shapes": [ - { - "shape": case_shape_key(profile, 1), - "signature": signature, - } - ] - } - ), - encoding="utf-8", - ) - state = DiscoveryState(runtime_skip_enabled=True, crash_skip_after=1) - loaded = load_runtime_skip_state(state, state_path) - - self.assertEqual(loaded, 1) - self.assertIn(signature, state.seen_crash_signatures) - self.assertTrue(runtime_skip_reason(profile, 721, state).startswith("runtime-known-crash-shape:")) - self.assertEqual(runtime_skip_reason(profile, 2, state), "") - - def test_generalized_skip_can_seed_family_suppression(self) -> None: - profile = _profile("cups_raster_to_rastertoescpx_coverage", "cups_raster_coverage_sweep", "raster_coverage_options") - signature = "SUMMARY: AddressSanitizer: FPE example/raster_filter.c:42 in process_line" - - with tempfile.TemporaryDirectory() as tmp: - state_path = Path(tmp) / "discovery_state.json" - state_path.write_text( - json.dumps( - { - "suppressed_case_shapes": [ - { - "shape": case_shape_key(profile, case_id), - "signature": signature, - } - for case_id in range(3) - ] - } - ), - encoding="utf-8", - ) - state = DiscoveryState(runtime_skip_enabled=True, generalized_skip_enabled=True, family_skip_after=3) - loaded = load_runtime_skip_state( - state, - state_path, - generalized_skip=True, - family_skip_after=3, - ) - - self.assertEqual(loaded, 3) - self.assertIn(case_family_key(profile), state.suppressed_case_families) - self.assertTrue(runtime_skip_reason(profile, 99, state).startswith("runtime-known-crash-family:")) - - def test_runtime_skip_auto_discovers_latest_useful_state(self) -> None: - profile = _profile("cups_raster_to_rastertopclx_feedback", "cups_raster_feedback_sweep", "raster_coverage_options") - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - older = root / "feedback-campaign" / "20260101-000000" - newer = root / "structural-campaign" / "20260102-000000" - empty = root / "feedback-campaign" / "20260103-000000" - older.mkdir(parents=True) - newer.mkdir(parents=True) - empty.mkdir(parents=True) - (older / "discovery_state.json").write_text( - json.dumps( - { - "suppressed_case_shapes": [ - {"shape": case_shape_key(profile, 1), "signature": "older"} - ] - } - ), - encoding="utf-8", - ) - (newer / "discovery_state.json").write_text( - json.dumps( - { - "suppressed_case_shapes": [ - {"shape": case_shape_key(profile, 2), "signature": "newer"} - ] - } - ), - encoding="utf-8", - ) - (empty / "discovery_state.json").write_text( - json.dumps({"suppressed_case_shapes": []}), - encoding="utf-8", - ) - - latest = find_latest_runtime_skip_state(root) - - self.assertEqual(latest, newer / "discovery_state.json") - - def test_seed_state_loads_recorded_family_suppression(self) -> None: - profile = _profile("pwg_to_pdf_feedback", "pwg_raster_feedback_sweep", "pwgtopdf_coverage_options") - family = case_family_key(profile) - - with tempfile.TemporaryDirectory() as tmp: - state_path = Path(tmp) / "discovery_state.json" - state_path.write_text( - json.dumps( - { - "suppressed_case_families": [ - {"family": family, "signature": "SUMMARY: AddressSanitizer: example"} - ] - } - ), - encoding="utf-8", - ) - state = DiscoveryState(runtime_skip_enabled=True, generalized_skip_enabled=True) - loaded = load_runtime_skip_state(state, state_path, generalized_skip=True) - - self.assertEqual(loaded, 0) - self.assertIn(family, state.suppressed_case_families) - self.assertTrue(runtime_skip_reason(profile, 123, state).startswith("runtime-known-crash-family:")) - - def test_target_family_normalizes_campaign_suffixes(self) -> None: - coverage_profile = _profile("cups_raster_to_rastertopclx_coverage", "cups_raster_general_sweep", "rastertopclx_plain") - general_profile = _profile("cups_raster_to_rastertopclx_general", "cups_raster_general_sweep", "rastertopclx_plain") - feedback_profile = _profile("cups_raster_to_rastertopclx_feedback", "cups_raster_general_sweep", "rastertopclx_plain") - - state = DiscoveryState(runtime_skip_enabled=True, generalized_skip_enabled=True, family_skip_after=1) - self.assertTrue(record_runtime_crash_suppression(state, coverage_profile, 0, "sig")) - - self.assertEqual(case_family_key(coverage_profile), case_family_key(general_profile)) - self.assertEqual(case_family_key(coverage_profile), case_family_key(feedback_profile)) - self.assertTrue(runtime_skip_reason(general_profile, 10, state).startswith("runtime-known-crash-family:")) - self.assertTrue(runtime_skip_reason(feedback_profile, 10, state).startswith("runtime-known-crash-family:")) - - def test_novelty_scheduler_prefers_productive_targets(self) -> None: - cold = _profile("cold_target", "text_coverage_sweep", "texttopdf_coverage_options") - hot = _profile("hot_target", "text_coverage_sweep", "texttopdf_coverage_options") - state = DiscoveryState() - state.target_stats = { - cold.id: TargetDiscoveryStats(submitted=10, completed=10), - hot.id: TargetDiscoveryStats(submitted=10, completed=10, retained_cases=5, new_features=20), - } - state.scheduler_credit = {cold.id: 0.0, hot.id: 0.0} - inflight = {cold.id: 0, hot.id: 0} - - picks = [_choose_next_profile([cold, hot], state, inflight, "novelty").id for _ in range(12)] - - self.assertGreater(picks.count(hot.id), picks.count(cold.id)) - - def test_scheduler_fills_minimum_target_share(self) -> None: - starved = _profile("starved_target", "image_feedback_sweep", "imagetoraster_coverage_options") - hot = _profile("hot_target", "image_feedback_sweep", "imagetopdf_coverage_options") - state = DiscoveryState() - state.target_stats = { - starved.id: TargetDiscoveryStats(submitted=10, completed=10), - hot.id: TargetDiscoveryStats(submitted=100, completed=100, retained_cases=30, new_features=80), - } - state.scheduler_credit = {starved.id: 0.0, hot.id: 0.0} - inflight = {starved.id: 0, hot.id: 0} - - pick = _choose_next_profile( - [starved, hot], - state, - inflight, - "novelty", - min_target_share=0.25, - ) - - self.assertEqual(pick.id, starved.id) - - def test_scheduler_caps_over_budget_hot_target(self) -> None: - cold = _profile("cold_target", "image_feedback_sweep", "imagetoraster_coverage_options") - hot = _profile("hot_target", "image_feedback_sweep", "imagetopdf_coverage_options") - state = DiscoveryState() - state.target_stats = { - cold.id: TargetDiscoveryStats(submitted=40, completed=40), - hot.id: TargetDiscoveryStats(submitted=100, completed=100, retained_cases=50, new_features=120), - } - state.scheduler_credit = {cold.id: 0.0, hot.id: 0.0} - inflight = {cold.id: 0, hot.id: 0} - - pick = _choose_next_profile( - [cold, hot], - state, - inflight, - "novelty", - max_target_share=0.60, - ) - - self.assertEqual(pick.id, cold.id) - - def test_scheduler_deweights_crash_dominated_targets(self) -> None: - state = DiscoveryState() - state.target_stats = { - "clean": TargetDiscoveryStats(completed=500, retained_cases=5, new_features=20), - "crashy": TargetDiscoveryStats( - completed=500, - retained_cases=5, - new_features=20, - crashes=250, - repeat_crashes=248, - runtime_suppressed=200, - ), - } - - self.assertLess(_target_scheduler_score(state, "crashy"), _target_scheduler_score(state, "clean") * 0.25) - - def test_scheduler_deweights_skip_only_targets(self) -> None: - state = DiscoveryState() - state.target_stats = { - "fresh": TargetDiscoveryStats(), - "suppressed": TargetDiscoveryStats(skipped=200, runtime_suppressed=200), - } - - self.assertLess(_target_scheduler_score(state, "suppressed"), _target_scheduler_score(state, "fresh")) - - def test_scheduler_deweights_seeded_crash_hazard_targets(self) -> None: - state = DiscoveryState() - state.target_stats = { - "clean": TargetDiscoveryStats(completed=20, retained_cases=3, new_features=12), - "image_to_imagetops_feedback_semantic": TargetDiscoveryStats( - completed=20, - retained_cases=3, - new_features=12, - ), - } - state.suppressed_case_hazards = { - ( - "target:image_to_imagetops_feedback|ppd:coverage_options|doc:image_feedback_sweep|" - "fmt:png_rgb|objective:postscript:ps-showpage-image|payload:exact|interlace:0" - ): "SUMMARY: AddressSanitizer: SEGV example/image.c:75 in close_image" - } - - self.assertLess( - _target_scheduler_score(state, "image_to_imagetops_feedback_semantic"), - _target_scheduler_score(state, "clean"), - ) - - def test_scheduler_periodically_probes_suppressed_targets(self) -> None: - clean = _profile("clean", "text_semantic_sweep", "texttopdf_coverage_options") - suppressed = _profile( - "image_to_imagetops_feedback_semantic", - "image_feedback_sweep", - "imagetops_coverage_options", - ) - state = DiscoveryState() - state.target_stats = { - clean.id: TargetDiscoveryStats(submitted=31, completed=31, retained_cases=8, new_features=40), - suppressed.id: TargetDiscoveryStats(submitted=1, completed=1), - } - state.scheduler_credit = {clean.id: 0.0, suppressed.id: 0.0} - state.suppressed_case_hazards = { - ( - "target:image_to_imagetops_feedback|ppd:imagetops_coverage_options|doc:image_feedback_sweep|" - "fmt:png_rgb|objective:postscript:ps-showpage-image|payload:exact|interlace:0" - ): "SUMMARY: AddressSanitizer: SEGV example/image.c:75 in close_image" - } - - with patch.dict("os.environ", {"SMT_FUZZER_AVOIDANCE_PROBE_INTERVAL": "32"}, clear=False): - pick = _choose_next_profile( - [clean, suppressed], - state, - {clean.id: 0, suppressed.id: 0}, - "novelty", - ) - - self.assertEqual(pick.id, suppressed.id) - - def test_avoidance_probe_can_bypass_runtime_family_skip(self) -> None: - profile = _profile( - "image_to_imagetops_feedback_semantic", - "image_feedback_sweep", - "imagetops_coverage_options", - ) - state = DiscoveryState(runtime_skip_enabled=True, generalized_skip_enabled=True) - family = case_family_key(profile) - state.suppressed_case_families = {family: "SUMMARY: AddressSanitizer: SEGV example/image.c:75"} - - with patch.dict("os.environ", {"SMT_FUZZER_AVOIDANCE_SKIP_PROBE_RATE": "1.0"}, clear=False): - reason = _combined_skip_reason(profile, 0, "coverage", state, skip_probe_rate=0.0) - - self.assertEqual(reason, "") - - def test_runtime_skip_probe_can_execute_suppressed_case(self) -> None: - profile = _profile("image_to_imagetoraster_coverage", "image_coverage_sweep", "imagetoraster_coverage_options") - state = DiscoveryState(runtime_skip_enabled=True, crash_skip_after=1) - self.assertTrue(record_runtime_crash_suppression(state, profile, 1, "sig")) - - skipped = _combined_skip_reason(profile, 721, "coverage", state, skip_probe_rate=0.0) - probed = _combined_skip_reason(profile, 721, "coverage", state, skip_probe_rate=1.0) - - self.assertTrue(skipped.startswith("runtime-known-crash-shape:")) - self.assertEqual(probed, "") - - def test_coverage_stagnation_requires_completed_cases_and_timeout(self) -> None: - state = DiscoveryState() - - self.assertFalse(_coverage_stagnated("coverage", 60, state, 0.0, 120.0)) - - state.completed_cases = 100 - self.assertFalse(_coverage_stagnated("coverage", 60, state, 100.0, 120.0)) - self.assertTrue(_coverage_stagnated("coverage", 60, state, 0.0, 120.0)) - self.assertFalse(_coverage_stagnated("crash", 60, state, 0.0, 120.0)) - self.assertFalse(_coverage_stagnated("coverage", 0, state, 0.0, 120.0)) - - def test_novel_discovery_recognizes_retention_and_new_crash(self) -> None: - self.assertTrue(_is_novel_discovery({"retained_for_coverage": True})) - self.assertTrue(_is_novel_discovery({"new_crash_signature": True})) - self.assertFalse(_is_novel_discovery({"new_crash_signature": False})) - self.assertFalse(_is_novel_discovery(None)) - - def test_short_png_abort_skip_is_optional_and_format_aware(self) -> None: - profile = _profile("image_to_imagetops_feedback", "image_feedback_sweep", "imagetops_coverage_options") - - with patch.dict( - "os.environ", - { - "SMT_FUZZER_SKIP_SHORT_IMAGE_ABORTS": "1", - "SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL": "1", - }, - clear=False, - ): - case_id = next( - index - for index in range(720) - if image_feedback_instance(index).objective == "short_payload" - and image_feedback_instance(index).image_format.startswith("png") - ) - reason = coverage_skip_reason(profile, case_id) - - self.assertEqual(reason, "low-value-short-png-libpng-abort") - - def test_run_dir_size_counts_files(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - (root / "a").write_bytes(b"1234") - nested = root / "nested" - nested.mkdir() - (nested / "b").write_bytes(b"123456") - - self.assertEqual(_run_dir_size_bytes(root), 10) - - def test_local_filter_env_prefers_explicit_asan_library_paths(self) -> None: - profile = _profile("image_to_imagetops_coverage", "image_coverage_sweep", "imagetops_coverage_options") - profile = TargetProfile( - id=profile.id, - description=profile.description, - ppd_kind=profile.ppd_kind, - document_kind=profile.document_kind, - executor=profile.executor, - input_mime=profile.input_mime, - output_mime=profile.output_mime, - expected_filters=profile.expected_filters, - cases=profile.cases, - oracle=profile.oracle, - filter_binary="/data/pre-gsoc/cups-filters/imagetops", - ) - env = { - "SMT_FUZZER_LIBPPD_ASAN": "/tmp/libppd-asan", - "SMT_FUZZER_LIBCUPSFILTERS_ASAN": "/tmp/libcupsfilters-asan", - "SMT_FUZZER_PDFIO_LIB": "/tmp/pdfio-lib", - "LD_LIBRARY_PATH": "/tmp/inherited", - } - - with patch.dict("os.environ", env, clear=False): - overrides = build_env_overrides(profile, Path("candidate.ppd")) - - self.assertEqual(overrides["PPD"], "candidate.ppd") - self.assertEqual( - overrides["LD_LIBRARY_PATH"].split(":")[:4], - ["/tmp/libppd-asan", "/tmp/libcupsfilters-asan", "/tmp/pdfio-lib", "/tmp/inherited"], - ) - - def test_direct_filter_job_options_are_populated_and_passed_as_argv5(self) -> None: - profile = _profile("image_to_imagetopdf_coverage", "image_feedback_sweep", "imagetopdf_coverage_options") - - options = build_job_options(profile, 17) - command = build_command(profile, Path("candidate.ppd"), Path("document.png"), job_options=options) - - self.assertIn("PageSize=", options) - self.assertIn("ColorModel=", options) - self.assertIn("scaling=", options) - self.assertEqual(command[5], options) - self.assertEqual(command[6], "document.png") - - -def _profile(target_id: str, document_kind: str, ppd_kind: str) -> TargetProfile: - return TargetProfile( - id=target_id, - description="test", - ppd_kind=ppd_kind, - document_kind=document_kind, - executor="direct_filter", - input_mime="application/test", - output_mime="", - expected_filters=[], - cases=1, - oracle="crash_or_signal", - filter_binary="/bin/true", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_crash_avoidance.py b/parser-fuzzers/tests/test_crash_avoidance.py deleted file mode 100644 index e257b08..0000000 --- a/parser-fuzzers/tests/test_crash_avoidance.py +++ /dev/null @@ -1,162 +0,0 @@ -from __future__ import annotations - -import json -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -from parser_fuzzers.constraint_repair import repair_image_goal -from parser_fuzzers.crash_avoidance import preferred_crash_avoidance_profile -from parser_fuzzers.format_specs import image_goals_for_target -from parser_fuzzers.output_feedback import choose_image_goal - - -class CrashAvoidanceTests(unittest.TestCase): - def test_crash_hazard_profile_loads_suppressed_image_hazards(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - state_path = _write_state(Path(tmp)) - with patch.dict( - "os.environ", - { - "SMT_FUZZER_CRASH_AVOIDANCE": "1", - "SMT_FUZZER_CRASH_AVOIDANCE_STATE": str(state_path), - }, - clear=False, - ): - profile = preferred_crash_avoidance_profile() - - self.assertEqual(len(profile.hazards), 1) - self.assertEqual(profile.hazards[0].target_id, "image_to_imagetoraster_feedback") - self.assertEqual(profile.hazards[0].image_format, "png_rgb") - self.assertEqual(profile.hazards[0].payload, "exact") - - def test_repair_image_goal_avoids_exact_old_hazard(self) -> None: - goal = next( - item - for item in image_goals_for_target("image_to_imagetoraster_feedback") - if item.name == "raster-rgb24" - ) - with tempfile.TemporaryDirectory() as tmp: - state_path = _write_state(Path(tmp)) - with patch.dict( - "os.environ", - { - "SMT_FUZZER_CRASH_AVOIDANCE": "1", - "SMT_FUZZER_CRASH_AVOIDANCE_STATE": str(state_path), - }, - clear=False, - ): - repaired = repair_image_goal( - goal=goal, - slot=0, - expansion_level=3, - target_id="image_to_imagetoraster_feedback", - ) - - self.assertEqual(repaired.payload_delta, 0) - self.assertEqual(repaired.png_interlace, 0) - self.assertNotEqual(repaired.image_format, "png_rgb") - self.assertIn(repaired.solved_by, {"z3-structure-avoid", "fallback-structure"}) - - def test_generalized_avoidance_applies_across_neighbor_objectives(self) -> None: - goal = next( - item - for item in image_goals_for_target("image_to_imagetops_feedback") - if item.name == "ps-commented-pnm" - ) - with tempfile.TemporaryDirectory() as tmp: - state_path = _write_state( - Path(tmp), - target_id="image_to_imagetops_feedback", - objective="postscript:ps-wide-maxval", - image_format="ppm", - signature="SUMMARY: AddressSanitizer: SEGV example/image.c:75 in close_image", - ) - with patch.dict( - "os.environ", - { - "SMT_FUZZER_CRASH_AVOIDANCE": "1", - "SMT_FUZZER_CRASH_AVOIDANCE_GENERALIZE": "1", - "SMT_FUZZER_CRASH_AVOIDANCE_STATE": str(state_path), - }, - clear=False, - ): - repaired = repair_image_goal( - goal=goal, - slot=0, - expansion_level=3, - target_id="image_to_imagetops_feedback", - ) - - self.assertEqual(repaired.payload_delta, 0) - self.assertEqual(repaired.png_interlace, 0) - self.assertNotEqual(repaired.image_format, "ppm") - self.assertIn(repaired.solved_by, {"z3-structure-avoid", "fallback-structure"}) - - def test_goal_selection_deprioritizes_old_crash_objective(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - state_path = _write_state( - Path(tmp), - target_id="image_to_imagetops_feedback", - objective="postscript:ps-showpage-image", - image_format="png_rgb", - signature="SUMMARY: AddressSanitizer: SEGV example/image.c:75 in close_image", - ) - with patch.dict( - "os.environ", - { - "SMT_FUZZER_AUTO_DIMENSIONS": "0", - "SMT_FUZZER_CRASH_AVOIDANCE": "1", - "SMT_FUZZER_CRASH_AVOIDANCE_STATE": str(state_path), - }, - clear=False, - ): - goal = choose_image_goal( - target_id="image_to_imagetops_feedback_semantic", - slot=0, - ) - - self.assertNotEqual(goal.name, "ps-showpage-image") - - -def _write_state( - root: Path, - *, - target_id: str = "image_to_imagetoraster_feedback", - objective: str = "cups-raster:raster-rgb24", - image_format: str = "png_rgb", - signature: str = ( - "SUMMARY: AddressSanitizer: heap-buffer-overflow " - "example/image_scale.c:123 in sample_scale" - ), -) -> Path: - state_path = root / "discovery_state.json" - state_path.write_text( - json.dumps( - { - "suppressed_case_hazards": [ - { - "hazard": ( - f"target:{target_id}|" - "ppd:coverage_options|" - "doc:image_feedback_sweep|" - f"fmt:{image_format}|" - f"objective:{objective}|" - "payload:exact|" - "interlace:0" - ), - "signature": signature, - } - ] - }, - indent=2, - ) - + "\n", - encoding="utf-8", - ) - return state_path - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_crash_dedup.py b/parser-fuzzers/tests/test_crash_dedup.py deleted file mode 100644 index 4ce93a4..0000000 --- a/parser-fuzzers/tests/test_crash_dedup.py +++ /dev/null @@ -1,83 +0,0 @@ -from __future__ import annotations - -import json -import tempfile -import unittest -from pathlib import Path - -from parser_fuzzers.crash_dedup import dedup_run - - -class CrashDedupTests(unittest.TestCase): - def test_deduplicates_asan_summary_and_excludes_infra(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - stderr_a = root / "a.stderr" - stderr_b = root / "b.stderr" - stderr_infra = root / "infra.stderr" - stderr_a.write_text( - "==11==ERROR: AddressSanitizer: FPE on unknown address 0x111\n" - "SUMMARY: AddressSanitizer: FPE example/raster_filter.c:42 in process_line\n", - encoding="utf-8", - ) - stderr_b.write_text( - "==22==ERROR: AddressSanitizer: FPE on unknown address 0x222\n" - "SUMMARY: AddressSanitizer: FPE example/raster_filter.c:42 in process_line\n", - encoding="utf-8", - ) - stderr_infra.write_text( - "==33==ASan runtime does not come first in initial library list\n", - encoding="utf-8", - ) - records = [ - _record(0, stderr_a, "case-a"), - _record(1, stderr_b, "case-b"), - _record(2, stderr_infra, "case-infra"), - ] - with (root / "timeline.jsonl").open("w", encoding="utf-8") as handle: - for record in records: - handle.write(json.dumps(record) + "\n") - - summary = dedup_run(root) - - self.assertEqual(summary.crash_records, 3) - self.assertEqual(summary.infra_excluded_records, 1) - self.assertEqual(summary.unique_crashes, 1) - self.assertEqual(summary.clusters[0].count, 2) - - def test_generic_libc_summary_uses_first_project_frame_context(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - stderr_path = root / "pdftoraster.stderr" - stderr_path.write_text( - "SUMMARY: AddressSanitizer: SEGV ../sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S:328 in __memset_avx2_unaligned_erms\n" - " #0 0x7f in __memset_avx2_unaligned_erms ../sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S:328\n" - " #1 0x7f in memset /usr/include/x86_64-linux-gnu/bits/string_fortified.h:59\n" - " #2 0x7f in write_page_image cupsfilters/pdftoraster.c:2013\n", - encoding="utf-8", - ) - with (root / "timeline.jsonl").open("w", encoding="utf-8") as handle: - handle.write(json.dumps(_record(0, stderr_path, "case-pdf")) + "\n") - - summary = dedup_run(root) - - self.assertEqual(summary.unique_crashes, 1) - self.assertIn("write_page_image cupsfilters/pdftoraster.c:2013", summary.clusters[0].signature) - - -def _record(case_id: int, stderr_path: Path, work_dir: str) -> dict[str, object]: - return { - "target_id": "cups_raster_to_rastertoescpx_general", - "case_id": case_id, - "work_dir": work_dir, - "command_line": "target @@", - "stderr_path": str(stderr_path), - "crashed": True, - "timed_out": False, - "oracle": "stderr crash/sanitizer", - "returncode": 86, - } - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_document_harness.py b/parser-fuzzers/tests/test_document_harness.py deleted file mode 100644 index 3702eec..0000000 --- a/parser-fuzzers/tests/test_document_harness.py +++ /dev/null @@ -1,124 +0,0 @@ -from __future__ import annotations - -import sys -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from parser_fuzzers.document_harness import HEADER_SIZE, make_document - - -class DocumentHarnessTests(unittest.TestCase): - def test_text_document_is_plain_seed(self) -> None: - document = make_document("text", 0) - self.assertEqual(document.mime, "text/plain") - self.assertIn(b"SMT multi-target", document.data) - - def test_cups_raster_has_magic_and_page_data(self) -> None: - document = make_document("cups_raster_mode10", 0) - self.assertEqual(document.data[:4], b"3SaR") - self.assertGreater(len(document.data), HEADER_SIZE) - - def test_cups_raster_boundary_sweep_has_magic(self) -> None: - document = make_document("cups_raster_boundary_sweep", 3) - self.assertEqual(document.data[:4], b"3SaR") - self.assertGreater(len(document.data), HEADER_SIZE) - - def test_pwg_raster_has_magic_and_page_data(self) -> None: - document = make_document("pwg_raster_resolution_stress", 1) - self.assertEqual(document.data[:4], b"2SaR") - self.assertGreater(len(document.data), HEADER_SIZE) - - def test_pwg_raster_boundary_sweep_has_magic(self) -> None: - document = make_document("pwg_raster_boundary_sweep", 7) - self.assertEqual(document.data[:4], b"2SaR") - self.assertGreater(len(document.data), HEADER_SIZE) - - def test_coverage_raster_sweep_can_generate_multipage_input(self) -> None: - document = make_document("cups_raster_coverage_sweep", 1) - self.assertEqual(document.data[:4], b"3SaR") - self.assertGreater(len(document.data), HEADER_SIZE * 2) - - def test_coverage_pwg_sweep_has_magic(self) -> None: - document = make_document("pwg_raster_coverage_sweep", 1) - self.assertEqual(document.data[:4], b"2SaR") - self.assertGreater(len(document.data), HEADER_SIZE * 2) - - def test_structural_sweeps_have_magic(self) -> None: - cups = make_document("cups_raster_structural_sweep", 3) - pwg = make_document("pwg_raster_structural_sweep", 4) - - self.assertEqual(cups.data[:4], b"3SaR") - self.assertEqual(pwg.data[:4], b"2SaR") - self.assertGreater(len(cups.data), HEADER_SIZE) - self.assertGreater(len(pwg.data), HEADER_SIZE) - - def test_pdf_coverage_sweep_has_pdf_magic(self) -> None: - document = make_document("pdf_coverage_sweep", 0) - self.assertEqual(document.mime, "application/pdf") - self.assertTrue(document.data.startswith(b"%PDF-")) - self.assertIn(b"xref", document.data) - - def test_pdf_semantic_sweep_exercises_pdf_features(self) -> None: - documents = [make_document("pdf_semantic_sweep", index) for index in range(8)] - payloads = [document.data for document in documents] - - self.assertTrue(all(document.mime == "application/pdf" for document in documents)) - self.assertTrue(all(payload.startswith(b"%PDF-") for payload in payloads)) - self.assertTrue(all(b"xref" in payload for payload in payloads)) - self.assertTrue(any(b"/Rotate 90" in payload for payload in payloads)) - self.assertTrue(any(b"/Filter /FlateDecode" in payload for payload in payloads)) - self.assertTrue(any(b"/XObject" in payload for payload in payloads)) - - def test_image_coverage_sweep_includes_pnm_and_png(self) -> None: - png = make_document("image_coverage_sweep", 0) - pnm = make_document("image_coverage_sweep", 2) - - self.assertTrue(pnm.data.startswith(b"P6")) - self.assertTrue(png.data.startswith(b"\x89PNG\r\n\x1a\n")) - - def test_image_feedback_sweep_includes_structured_images(self) -> None: - documents = [make_document("image_feedback_sweep", index) for index in range(12)] - prefixes = {document.data[:2] for document in documents} - - self.assertTrue(any(document.data.startswith(b"\x89PNG\r\n\x1a\n") for document in documents)) - self.assertTrue(any(prefix in {b"P4", b"P5", b"P6"} for prefix in prefixes)) - self.assertTrue(all(document.mime in {"image/png", "image/x-portable-anymap"} for document in documents)) - - def test_text_and_command_coverage_sweeps(self) -> None: - text = make_document("text_coverage_sweep", 2) - command = make_document("command_coverage_sweep", 0) - - self.assertEqual(text.mime, "text/plain") - self.assertEqual(command.mime, "application/vnd.cups-command") - self.assertTrue(command.data.startswith(b"#CUPS-COMMAND")) - - def test_text_and_command_semantic_sweeps(self) -> None: - text_documents = [make_document("text_semantic_sweep", index) for index in range(10)] - command_documents = [make_document("command_semantic_sweep", index) for index in range(10)] - - self.assertTrue(all(document.mime == "text/plain" for document in text_documents)) - self.assertTrue(any(b"\f" in document.data for document in text_documents)) - self.assertTrue(any(b"\x1b" in document.data for document in text_documents)) - self.assertTrue(all(document.data.startswith(b"#CUPS-COMMAND") for document in command_documents)) - self.assertTrue(any(b"SetAlignment" in document.data for document in command_documents)) - - def test_postscript_coverage_sweep_has_magic(self) -> None: - document = make_document("postscript_coverage_sweep", 1) - self.assertEqual(document.mime, "application/postscript") - self.assertTrue(document.data.startswith(b"%!PS")) - - def test_postscript_semantic_sweep_exercises_language_features(self) -> None: - documents = [make_document("postscript_semantic_sweep", index) for index in range(8)] - payloads = [document.data for document in documents] - - self.assertTrue(all(document.mime == "application/postscript" for document in documents)) - self.assertTrue(all(payload.startswith(b"%!PS") for payload in payloads)) - self.assertTrue(any(b"setpagedevice" in payload for payload in payloads)) - self.assertTrue(any(b" image" in payload for payload in payloads)) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_dynamic_constraints.py b/parser-fuzzers/tests/test_dynamic_constraints.py deleted file mode 100644 index 9ecdeac..0000000 --- a/parser-fuzzers/tests/test_dynamic_constraints.py +++ /dev/null @@ -1,150 +0,0 @@ -from __future__ import annotations - -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -from parser_fuzzers.dynamic_constraints import build_dynamic_compare_profile -from parser_fuzzers.multitarget_runner import TargetProfile, build_env_overrides -from parser_fuzzers.structured_templates import pwg_structural_instance - - -class DynamicConstraintTests(unittest.TestCase): - def test_summarizes_compare_trace_tokens(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - run_dir = Path(tmp) - case_dir = run_dir / "target" / "case-0000" - case_dir.mkdir(parents=True) - (case_dir / "compare_trace.tsv").write_text( - "\n".join( - [ - "pid\tpc\top\tret\tlen\ta_hex\tb_hex\ta_ascii\tb_ascii", - "1\t0x1\tstrcmp\t0\t9\t\t\tPageSize\tPageSize", - "1\t0x2\tmemcmp\t1\t4\t\t\t2SaR\tRaS2", - "1\t0x3\tstrcmp\t1\t24\t\t\tapplication/vnd.cups-pwg.\ttext/plain", - ] - ) - + "\n", - encoding="utf-8", - ) - - profile = build_dynamic_compare_profile(run_dir) - - self.assertEqual(profile["summary"]["trace_files"], 1) - self.assertEqual(profile["summary"]["compare_records"], 3) - self.assertEqual(profile["ppd_options"]["PageSize"], 1) - self.assertEqual(profile["magic_tokens"]["2SaR"], 1) - self.assertEqual(profile["tokens"]["application/vnd.cups-pwg"], 1) - - def test_runner_env_enables_dynamic_compare_trace(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - trace_lib = root / "libtrace.so" - trace_lib.write_bytes(b"placeholder") - case_dir = root / "case-0000" - case_dir.mkdir() - profile = TargetProfile( - id="pwg_to_pdf", - description="", - ppd_kind="pwgtopdf_coverage_options", - document_kind="pwg_raster_structural_sweep", - executor="direct_filter", - input_mime="application/vnd.cups-pwg", - output_mime="", - expected_filters=["pwgtopdf"], - cases=1, - oracle="crash_or_signal", - filter_binary="/data/pre-gsoc/cups-filters/pwgtopdf", - ) - - with patch.dict( - "os.environ", - { - "SMT_FUZZER_DYNAMIC_COMPARE_TRACE_LIB": str(trace_lib), - "SMT_FUZZER_COMPARE_TRACE_LIMIT": "7", - }, - clear=False, - ): - env = build_env_overrides(profile, root / "candidate.ppd", case_dir) - - self.assertEqual(env["SMT_FUZZER_COMPARE_TRACE"], str(case_dir / "compare_trace.tsv")) - self.assertEqual(env["SMT_FUZZER_COMPARE_TRACE_LIMIT"], "7") - self.assertIn(str(trace_lib), env["LD_PRELOAD"]) - self.assertIn("verify_asan_link_order=0", env["ASAN_OPTIONS"]) - - def test_runner_env_can_sample_dynamic_compare_trace(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - trace_lib = root / "libtrace.so" - trace_lib.write_bytes(b"placeholder") - traced_case = root / "case-0002" - skipped_case = root / "case-0003" - max_skipped_case = root / "case-0006" - traced_case.mkdir() - skipped_case.mkdir() - max_skipped_case.mkdir() - profile = TargetProfile( - id="pwg_to_pdf", - description="", - ppd_kind="pwgtopdf_coverage_options", - document_kind="pwg_raster_structural_sweep", - executor="direct_filter", - input_mime="application/vnd.cups-pwg", - output_mime="", - expected_filters=["pwgtopdf"], - cases=1, - oracle="crash_or_signal", - filter_binary="/data/pre-gsoc/cups-filters/pwgtopdf", - ) - - with patch.dict( - "os.environ", - { - "SMT_FUZZER_DYNAMIC_COMPARE_TRACE_LIB": str(trace_lib), - "SMT_FUZZER_DYNAMIC_COMPARE_TRACE_EVERY": "2", - "SMT_FUZZER_DYNAMIC_COMPARE_TRACE_MAX_CASES": "5", - }, - clear=False, - ): - traced = build_env_overrides(profile, root / "candidate.ppd", traced_case) - skipped = build_env_overrides(profile, root / "candidate.ppd", skipped_case) - max_skipped = build_env_overrides(profile, root / "candidate.ppd", max_skipped_case) - - self.assertIn("SMT_FUZZER_COMPARE_TRACE", traced) - self.assertNotIn("SMT_FUZZER_COMPARE_TRACE", skipped) - self.assertNotIn("SMT_FUZZER_COMPARE_TRACE", max_skipped) - - def test_dynamic_profile_can_bias_template_objective(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - profile_path = Path(tmp) / "dynamic.json" - profile_path.write_text( - """ -{ - "schema_version": "dynamic-compare-hints-v1", - "summary": {"compare_records": 4}, - "tokens": {"cupsBytesPerLine": 4}, - "ppd_options": {"cupsBytesPerLine": 4}, - "magic_tokens": {}, - "records": [] -} -""".lstrip(), - encoding="utf-8", - ) - - with patch.dict( - "os.environ", - { - "SMT_FUZZER_DYNAMIC_CONSTRAINTS": str(profile_path), - "SMT_FUZZER_SOURCE_CONSTRAINT_RATE": "1.0", - }, - clear=False, - ): - instance = pwg_structural_instance(1) - - self.assertEqual(instance.objective, "short_line") - self.assertIn(instance.solved_by, {"z3-source", "fallback"}) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_loop_metrics.py b/parser-fuzzers/tests/test_loop_metrics.py deleted file mode 100644 index 3be6028..0000000 --- a/parser-fuzzers/tests/test_loop_metrics.py +++ /dev/null @@ -1,71 +0,0 @@ -from __future__ import annotations - -import json -import sys -import tempfile -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from parser_fuzzers.loop_metrics import write_standard_loop_metrics - - -class LoopMetricsTests(unittest.TestCase): - def test_summarizes_template_afl_feedback_campaign(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - campaign = Path(tmp) / "campaign" - template_run = campaign / "template" / "run" - feedback_run = campaign / "feedback-template" / "run" - template_run.mkdir(parents=True) - feedback_run.mkdir(parents=True) - _write_metrics(template_run / "standard_metrics.json", execs=100, features=20, corpus=7) - _write_metrics(feedback_run / "standard_metrics.json", execs=80, features=25, corpus=9) - _write_metrics(campaign / "afl-standard-metrics.json", execs=1000, features=12, corpus=30, crashes=1) - (campaign / "seed-export.json").write_text( - json.dumps({"exported": 4, "targets": ["t"], "extensions": [".pwg"]}), - encoding="utf-8", - ) - (campaign / "afl-import.json").write_text( - json.dumps({"imported": [{"source": "afl-queue"}, {"source": "afl-crash"}], "crashes_imported": 1}), - encoding="utf-8", - ) - (campaign / "feedback-profile-build.json").write_text( - json.dumps({"pwg_seeds": 2}), - encoding="utf-8", - ) - (campaign / "loop_manifest.json").write_text( - json.dumps({"template_run": str(template_run), "feedback_run": str(feedback_run)}), - encoding="utf-8", - ) - - payload = write_standard_loop_metrics(campaign) - - self.assertEqual(payload["summary"]["template_features"], 20) - self.assertEqual(payload["summary"]["afl_crashes"], 1) - self.assertEqual(payload["summary"]["feedback_feature_delta_vs_template"], 5) - self.assertEqual(payload["afl_import"]["source_counts"], {"afl-crash": 1, "afl-queue": 1}) - self.assertTrue((campaign / "loop_standard_metrics.json").exists()) - - -def _write_metrics(path: Path, *, execs: int, features: int, corpus: int, crashes: int = 0) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - json.dumps( - { - "standard": { - "execs_done": execs, - "coverage_features": features, - "corpus_count": corpus, - "crashes": crashes, - "timeouts": 0, - } - } - ), - encoding="utf-8", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_patcher.py b/parser-fuzzers/tests/test_patcher.py deleted file mode 100644 index 22177ad..0000000 --- a/parser-fuzzers/tests/test_patcher.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import annotations - -import sys -import tempfile -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from parser_fuzzers.hashing import sha256_bytes, sha256_file -from parser_fuzzers.models import BranchEvent, Patch, SolverResult -from parser_fuzzers.patcher import apply_patches, apply_solver_result - - -class PatcherTests(unittest.TestCase): - def test_apply_patches_rewrites_expected_bytes(self) -> None: - patch = Patch(offset=1, old_hex="00", new_hex="41", width=1) - self.assertEqual(apply_patches(b"x\x00z", [patch]), b"xAz") - - def test_apply_patches_rejects_old_byte_mismatch(self) -> None: - patch = Patch(offset=1, old_hex="00", new_hex="41", width=1) - with self.assertRaises(ValueError): - apply_patches(b"xyz", [patch]) - - def test_apply_solver_result_writes_candidate(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - input_path = root / "input.bin" - input_path.write_bytes(b"\x00abc") - event = BranchEvent( - target_id="unit", - input_path=str(input_path), - input_sha256=sha256_file(input_path), - offset=0, - width=1, - endianness="little", - signed=False, - op="eq", - rhs=0x41, - description="unit test", - ) - result = SolverResult( - status="sat", - solver_ms=1.0, - patches=[Patch(offset=0, old_hex="00", new_hex="41", width=1)], - reason="unit", - event=event, - ) - output = apply_solver_result(result, input_path, root / "out") - self.assertTrue(output.exists()) - self.assertEqual(output.read_bytes(), b"Aabc") - self.assertNotEqual(sha256_bytes(output.read_bytes()), event.input_sha256) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_ppd_templates.py b/parser-fuzzers/tests/test_ppd_templates.py deleted file mode 100644 index fcd2e1c..0000000 --- a/parser-fuzzers/tests/test_ppd_templates.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import annotations - -import sys -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from parser_fuzzers.multitarget_runner import load_profiles -from parser_fuzzers.ppd_templates import make_ppd - - -class PPDTemplateTests(unittest.TestCase): - def test_rastertopclx_template_keeps_filter_and_literal_payload(self) -> None: - ppd = make_ppd("rastertopclx", 3) - self.assertIn('*cupsFilter: "application/vnd.cups-raster 0 rastertopclx"', ppd) - self.assertIn("*cupsPCL EndJob", ppd) - self.assertNotIn("%", ppd) - - def test_plain_rastertopclx_template_has_no_string_payload(self) -> None: - ppd = make_ppd("rastertopclx_plain", 3) - self.assertIn('*cupsFilter: "application/vnd.cups-raster 0 rastertopclx"', ppd) - self.assertNotIn("*cupsPCL EndJob", ppd) - - def test_string_sweep_template_uses_generic_values(self) -> None: - ppd = make_ppd("rastertopclx_string_sweep", 5) - self.assertIn("*cupsPCL EndJob", ppd) - self.assertNotIn("%", ppd) - - def test_single_pagesize_template_keeps_required_page_region(self) -> None: - ppd = make_ppd("rastertoescpx_single_pagesize", 0) - self.assertIn('*cupsFilter: "application/vnd.cups-raster 0 rastertoescpx"', ppd) - self.assertEqual(ppd.count("*PageSize Letter:"), 1) - self.assertIn("*OpenUI *PageRegion", ppd) - - def test_parser_target_config_loads_all_profiles(self) -> None: - profiles = load_profiles(ROOT / "configs" / "parser_targets.yaml") - self.assertEqual(len(profiles), 4) - self.assertEqual(profiles[0].id, "ppd_text_to_rastertopclx_smoke") - - def test_explore_target_config_loads_all_profiles(self) -> None: - profiles = load_profiles(ROOT / "configs" / "parser_targets_explore.yaml") - self.assertEqual(len(profiles), 4) - self.assertEqual(profiles[0].id, "ppd_text_to_rastertopclx_explore") - - def test_coverage_template_adds_option_groups(self) -> None: - ppd = make_ppd("pwg_resolution_coverage", 2) - self.assertIn("*OpenUI *ColorModel", ppd) - self.assertIn("*OpenUI *PrintQuality", ppd) - self.assertIn("*OpenUI *MediaType", ppd) - self.assertNotIn("65536x65536dpi", ppd) - - def test_coverage_target_config_loads_profiles(self) -> None: - profiles = load_profiles(ROOT / "configs" / "parser_targets_coverage.yaml") - self.assertEqual(len(profiles), 20) - self.assertEqual(profiles[0].id, "cups_raster_to_rastertopclx_coverage") - - def test_pdf_and_image_coverage_templates_have_filter_lines(self) -> None: - pdf = make_ppd("pdftoraster_coverage_options", 0) - image = make_ppd("imagetoraster_coverage_options", 0) - - self.assertIn('*cupsFilter: "application/pdf 0 pdftoraster"', pdf) - self.assertIn('*cupsFilter: "image/x-portable-anymap 0 imagetoraster"', image) - self.assertIn("*OpenUI *ColorModel", pdf) - - def test_command_coverage_template_has_command_filter(self) -> None: - ppd = make_ppd("commandtoescpx_coverage_options", 0) - self.assertIn('*cupsFilter: "application/vnd.cups-command 0 commandtoescpx"', ppd) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_run_metrics.py b/parser-fuzzers/tests/test_run_metrics.py deleted file mode 100644 index e367fc5..0000000 --- a/parser-fuzzers/tests/test_run_metrics.py +++ /dev/null @@ -1,92 +0,0 @@ -from __future__ import annotations - -import json -import tempfile -import unittest -from pathlib import Path - -from parser_fuzzers.run_metrics import summarize_run_metrics - - -class RunMetricsTests(unittest.TestCase): - def test_summarize_run_metrics_combines_summary_dedup_and_timeline(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - (root / "summary.concise.json").write_text( - json.dumps( - { - "run_id": "run-1", - "elapsed_sec": 120.0, - "cases": 10, - "retained_cases": 4, - "coverage_features": 12, - "crashes": 2, - "unique_crashes": 1, - "timeouts": 0, - "skipped": 3, - "pruned_cases": 2, - "targets": 2, - "run_dir_bytes": 1073741824, - "stop_reason": "duration", - "target_stats": { - "target-a": { - "completed": 5, - "retained_cases": 3, - "crashes": 0, - "timeouts": 0, - "runtime_suppressed": 0, - } - }, - } - ) - + "\n", - encoding="utf-8", - ) - (root / "dedup.json").write_text( - json.dumps( - { - "crash_records": 2, - "unique_crashes": 1, - "clusters": [ - { - "target_id": "target-b", - "count": 2, - "signature": "sig", - "representative_work_dir": "case", - } - ], - } - ) - + "\n", - encoding="utf-8", - ) - (root / "timeline.jsonl").write_text( - "\n".join( - [ - json.dumps({"document_description": "x via y/z3-structure-avoid"}), - json.dumps({"new_crash_signature": True}), - ] - ) - + "\n", - encoding="utf-8", - ) - - payload = summarize_run_metrics(root) - - self.assertEqual(payload["run"]["cases"], 10) - self.assertEqual(payload["derived"]["timeline_records"], 2) - self.assertEqual(payload["derived"]["z3_structure_avoid_records"], 1) - self.assertEqual(payload["derived"]["new_crash_signature_records"], 1) - self.assertEqual(payload["derived"]["retained_per_min"], 2.0) - self.assertEqual(payload["derived"]["cases_per_sec"], 0.083) - self.assertEqual(payload["standard"]["execs_done"], 10) - self.assertEqual(payload["standard"]["execs_per_sec"], 0.083) - self.assertEqual(payload["standard"]["coverage_features_per_hour"], 360.0) - self.assertEqual(payload["standard"]["run_dir_gb"], 1.0) - self.assertEqual(payload["standard"]["pruned_cases"], 2) - self.assertEqual(payload["dedup"]["unique_crash_signatures"], 1) - self.assertEqual(payload["target_stats"]["target-a"]["retained_density"], 0.6) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_run_recovery.py b/parser-fuzzers/tests/test_run_recovery.py deleted file mode 100644 index 7b6bba9..0000000 --- a/parser-fuzzers/tests/test_run_recovery.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -import json -import tempfile -import unittest -from pathlib import Path - -from parser_fuzzers.run_recovery import recover_run_summary - - -class RunRecoveryTests(unittest.TestCase): - def test_recovers_concise_summary_from_timeline(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - (root / "run_manifest.json").write_text( - json.dumps( - { - "run_id": "test-run", - "workers": 2, - "timeout_sec": 5, - "duration_sec": 60, - "targets": [{"id": "image_to_ps"}], - "runtime_skip": True, - } - ), - encoding="utf-8", - ) - records = [ - { - "target_id": "image_to_ps", - "case_id": 0, - "crashed": True, - "timed_out": False, - "oracle": "stderr crash/sanitizer", - "new_crash_signature": True, - "retained_for_coverage": True, - "new_feature_count": 3, - }, - { - "target_id": "image_to_ps", - "case_id": 1, - "skipped": True, - "skip_reason": "runtime-known-crash-shape:asan", - "crashed": False, - "timed_out": False, - }, - ] - with (root / "timeline.jsonl").open("w", encoding="utf-8") as handle: - for record in records: - handle.write(json.dumps(record) + "\n") - - summary = recover_run_summary(root) - - self.assertEqual(summary["cases"], 1) - self.assertEqual(summary["crashes"], 1) - self.assertEqual(summary["skipped"], 1) - self.assertEqual(summary["unique_crashes"], 1) - self.assertEqual(summary["coverage_features"], 3) - self.assertTrue((root / "summary.concise.json").exists()) - self.assertTrue((root / "summary.json").exists()) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_run_set_metrics.py b/parser-fuzzers/tests/test_run_set_metrics.py deleted file mode 100644 index 9ec4d84..0000000 --- a/parser-fuzzers/tests/test_run_set_metrics.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -import json -import tempfile -import unittest -from pathlib import Path - -from parser_fuzzers.run_set_metrics import summarize_run_set - - -class RunSetMetricsTests(unittest.TestCase): - def test_summarize_run_set_aggregates_campaigns(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - self._write_run(root / "campaign-a" / "run-1", cases=10, features=5, crashes=2, unique=1) - self._write_run(root / "campaign-a" / "run-2", cases=20, features=8, crashes=0, unique=0) - self._write_run(root / "campaign-b" / "run-3", cases=5, features=4, crashes=1, unique=1) - - payload = summarize_run_set([root / "campaign-a", root / "campaign-b"]) - - self.assertEqual(payload["run_count"], 3) - self.assertEqual(payload["aggregate"]["cases"], 35) - self.assertEqual(payload["aggregate"]["coverage_features_sum"], 17) - self.assertEqual(payload["aggregate"]["unique_crashes_sum"], 2) - self.assertEqual(payload["campaigns"]["campaign-a"]["cases"], 30) - - def _write_run(self, run_dir: Path, *, cases: int, features: int, crashes: int, unique: int) -> None: - run_dir.mkdir(parents=True) - (run_dir / "timeline.jsonl").write_text("{}\n", encoding="utf-8") - (run_dir / "summary.concise.json").write_text( - json.dumps( - { - "run_id": run_dir.name, - "elapsed_sec": 60.0, - "cases": cases, - "retained_cases": features, - "coverage_features": features, - "crashes": crashes, - "unique_crashes": unique, - "timeouts": 0, - "skipped": 0, - "targets": 1, - "stop_reason": "duration", - } - ) - + "\n", - encoding="utf-8", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_seed_export.py b/parser-fuzzers/tests/test_seed_export.py deleted file mode 100644 index b5f0672..0000000 --- a/parser-fuzzers/tests/test_seed_export.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -import json -import sys -import tempfile -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from parser_fuzzers.afl_integration.seed_export import export_template_seeds - - -class SeedExportTests(unittest.TestCase): - def test_exports_retained_documents_as_afl_seeds(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - case_dir = root / "run" / "corpus" / "interesting" / "pwg_target" / "case-000001" - case_dir.mkdir(parents=True) - (case_dir / "document.pwg").write_bytes(b"RaS2") - (case_dir / "document.pdf").write_bytes(b"%PDF") - out_dir = root / "seeds" - - summary = export_template_seeds( - run_dir=root / "run", - output_dir=out_dir, - target_ids=["pwg_target"], - extensions=["pwg"], - ) - - self.assertEqual(summary.exported, 1) - exported = sorted(out_dir.glob("*.pwg")) - self.assertEqual(len(exported), 1) - self.assertEqual(exported[0].read_bytes(), b"RaS2") - ordinary_files = sorted(path.name for path in out_dir.iterdir() if not path.name.startswith(".")) - self.assertEqual(ordinary_files, [exported[0].name]) - manifest_path = out_dir.parent / "seeds-seed_export_manifest.json" - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - self.assertEqual(manifest["exported_by_target"], {"pwg_target": 1}) - self.assertEqual(manifest["manifest_path"], str(manifest_path)) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_seed_policy.py b/parser-fuzzers/tests/test_seed_policy.py deleted file mode 100644 index 08ca10f..0000000 --- a/parser-fuzzers/tests/test_seed_policy.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import annotations - -import sys -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from parser_fuzzers.validation import load_yaml - - -class SeedPolicyTests(unittest.TestCase): - def test_private_reproducers_are_not_allowed_in_seed(self) -> None: - for meta_path in sorted((ROOT / "bugs").glob("*/meta.yaml")): - meta = load_yaml(meta_path) - self.assertIs(meta["known_poc_allowed_in_seed"], False, meta_path) - - def test_public_seeds_do_not_copy_private_reproducer_names(self) -> None: - seed_names = {path.name for path in (ROOT / "seeds" / "public").glob("*") if path.is_file()} - private_markers = ("poc", "repro", "crash", "asan", "issue") - self.assertFalse( - [name for name in seed_names if any(marker in name.lower() for marker in private_markers)] - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_semantic_shapes.py b/parser-fuzzers/tests/test_semantic_shapes.py deleted file mode 100644 index 7f7cb37..0000000 --- a/parser-fuzzers/tests/test_semantic_shapes.py +++ /dev/null @@ -1,197 +0,0 @@ -from __future__ import annotations - -import unittest -from types import SimpleNamespace - -from parser_fuzzers.document_harness import make_cups_raster, make_image -from parser_fuzzers.multitarget_runner import ( - DiscoveryState, - TargetProfile, - planned_semantic_runtime_key, - record_semantic_crash_suppression, - runtime_skip_reason, -) -from parser_fuzzers.ppd_templates import make_ppd -from parser_fuzzers.semantic_shapes import ( - build_planned_shape, - output_depth_score, - parse_job_options, - parse_document_bytes, - parse_output_bytes, - shape_feature_tokens, -) - - -class SemanticShapeTests(unittest.TestCase): - def test_png_semantic_shape_is_structural_and_hashable(self) -> None: - profile = _profile() - document = make_image( - image_format="png_rgb", - width=17, - height=3, - channels=3, - png_interlace=1, - ) - - shape = build_planned_shape( - target_id=profile.id, - ppd_kind=profile.ppd_kind, - document_kind=profile.document_kind, - input_mime=profile.input_mime, - output_mime=profile.output_mime, - expected_filters=profile.expected_filters, - ppd_text=make_ppd(profile.ppd_kind, 0), - document_data=document, - ) - document_shape = shape["semantic_input"]["document"] - features = shape_feature_tokens( - { - **shape, - "path_shape": {"stderr_states": ["before-scaling"], "reached_expected_filter": True}, - "failure_shape": {"location": ""}, - "path_shape_hash": "path", - "failure_shape_hash": "failure", - "compound_shape_hash": "compound", - } - ) - - self.assertEqual(document_shape["format"], "png") - self.assertEqual(document_shape["interlace"], 1) - self.assertIn("IHDR", document_shape["structure"]) - self.assertTrue(shape["semantic_input_hash"]) - self.assertIn("shape-doc-format:png", features) - self.assertIn("shape-path-state:before-scaling", features) - self.assertTrue(any(feature.startswith("shape-path-depth:") for feature in features)) - self.assertNotIn("shape-compound:compound", features) - - def test_job_options_are_part_of_semantic_input_hash(self) -> None: - profile = _profile() - document = make_image(image_format="png_rgb", width=4, height=4, channels=3) - common = { - "target_id": profile.id, - "ppd_kind": profile.ppd_kind, - "document_kind": profile.document_kind, - "input_mime": profile.input_mime, - "output_mime": profile.output_mime, - "expected_filters": profile.expected_filters, - "ppd_text": make_ppd(profile.ppd_kind, 0), - "document_data": document, - } - - gray = build_planned_shape(**common, job_options="PageSize=A4 ColorModel=Gray Resolution=300x300dpi") - rgb = build_planned_shape(**common, job_options="PageSize=A4 ColorModel=RGB Resolution=300x300dpi") - parsed = parse_job_options("PageSize=A4 ColorModel=RGB") - - self.assertNotEqual(gray["semantic_input_hash"], rgb["semantic_input_hash"]) - self.assertEqual(parsed["ColorModel"], "RGB") - self.assertIn("PageSize", parsed["keys"]) - - def test_semantic_runtime_skip_requires_repeated_nonretained_failure(self) -> None: - profile = _profile() - shape_hash = planned_semantic_runtime_key(profile, 0).split("semantic-input:", 1)[1] - shape_bundle = { - "semantic_input_hash": shape_hash, - "failure_shape_hash": "failure123", - } - result = SimpleNamespace(target_id=profile.id) - state = DiscoveryState(runtime_skip_enabled=True, semantic_skip_after=2) - signature = "SUMMARY: AddressSanitizer: heap-buffer-overflow cupsfilters/image.c:1 in example" - - self.assertFalse( - record_semantic_crash_suppression(state, result, shape_bundle, signature, retained=True) - ) - self.assertEqual(runtime_skip_reason(profile, 0, state), "") - self.assertFalse( - record_semantic_crash_suppression(state, result, shape_bundle, signature, retained=False) - ) - self.assertEqual(runtime_skip_reason(profile, 0, state), "") - self.assertTrue( - record_semantic_crash_suppression(state, result, shape_bundle, signature, retained=False) - ) - - self.assertTrue(runtime_skip_reason(profile, 0, state).startswith("runtime-known-crash-semantic-shape:")) - - def test_document_parser_buckets_pnm_shape(self) -> None: - document = make_image(image_format="pbm", width=31, height=3, channels=1) - shape = parse_document_bytes(document) - - self.assertEqual(shape["format"], "pnm") - self.assertEqual(shape["magic"], "P4") - self.assertIn("width-boundary", shape["image_class"]) - - def test_pdf_output_shape_adds_deep_observation_features(self) -> None: - output = ( - b"%PDF-1.7\n" - b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n" - b"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n" - b"3 0 obj\n<< /Type /Page /Resources << >> /Contents 4 0 R >>\nendobj\n" - b"4 0 obj\n<< /Length 5 >>\nstream\nabcde\nendstream\nendobj\n" - b"xref\n0 5\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n" - ) - output_shape = parse_output_bytes(output) - features = shape_feature_tokens( - { - "semantic_input_hash": "input", - "path_shape": { - "stderr_states": ["imagetopdf", "before-scaling"], - "reached_expected_filter": True, - "return_class": "zero", - "stdout_size": "512-4k", - }, - "path_shape_hash": "path", - "output_shape": output_shape, - "output_shape_hash": "output", - "failure_shape": {"location": ""}, - "failure_shape_hash": "failure", - } - ) - - self.assertEqual(output_shape["format"], "pdf") - self.assertEqual(output_shape["object_bucket"], "2-4") - self.assertEqual(output_shape["page_bucket"], "1") - self.assertTrue(output_shape["has_xref"]) - self.assertTrue(output_shape["has_trailer"]) - self.assertTrue(output_shape["has_eof"]) - self.assertIn("size:", output_shape["structure"]) - self.assertIn("xref:1", output_shape["structure"]) - self.assertIn("filter:none", output_shape["structure"]) - self.assertGreaterEqual(output_depth_score(output_shape), 8) - self.assertIn("shape-output-format:pdf", features) - self.assertTrue(any(feature.startswith("shape-output-depth:") for feature in features)) - - def test_raster_output_shape_keeps_header_dimensions_in_structure(self) -> None: - output = make_cups_raster( - width=127, - height=31, - compression=0, - num_colors=3, - color_space=1, - bits_per_pixel=24, - ) - output_shape = parse_output_bytes(output) - - self.assertEqual(output_shape["format"], "cups-raster") - self.assertIn("w:65-256", output_shape["structure"]) - self.assertIn("h:17-64", output_shape["structure"]) - self.assertIn("bpp:24", output_shape["structure"]) - self.assertIn("color:1", output_shape["structure"]) - - -def _profile() -> TargetProfile: - return TargetProfile( - id="image_to_imagetops_feedback", - description="test image pipeline", - ppd_kind="imagetops_coverage_options", - document_kind="image_feedback_sweep", - executor="filter", - input_mime="image/png", - output_mime="application/postscript", - expected_filters=["imagetops"], - cases=1, - oracle="crash_or_signal", - filter_binary="", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_solver.py b/parser-fuzzers/tests/test_solver.py deleted file mode 100644 index 854af31..0000000 --- a/parser-fuzzers/tests/test_solver.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -import importlib.util -import sys -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from parser_fuzzers.hashing import sha256_bytes -from parser_fuzzers.models import BranchEvent -from parser_fuzzers.solver import MissingSolverError, condition_holds, read_event_value, solve_event - - -def make_event(data: bytes, op: str = "eq", rhs: int = 0x41) -> BranchEvent: - return BranchEvent( - target_id="unit", - input_path="input.bin", - input_sha256=sha256_bytes(data), - offset=0, - width=1, - endianness="little", - signed=False, - op=op, - rhs=rhs, - description="unit test event", - ) - - -class SolverTests(unittest.TestCase): - def test_bounded_fallback_solves_eq(self) -> None: - data = b"\x00abc" - event = make_event(data) - result = solve_event(event, data, allow_fallback=True) - self.assertEqual(result.status, "sat") - self.assertEqual(result.patches[0].new_hex, "41") - - @unittest.skipUnless(importlib.util.find_spec("z3") is not None, "z3-solver is not installed") - def test_z3_solves_eq(self) -> None: - data = b"\x00abc" - event = make_event(data) - result = solve_event(event, data, allow_fallback=False) - self.assertEqual(result.status, "sat") - self.assertIn("z3", result.reason) - - def test_strict_solver_reports_missing_z3(self) -> None: - if importlib.util.find_spec("z3") is not None: - self.skipTest("z3-solver is installed") - data = b"\x00abc" - event = make_event(data) - with self.assertRaises(MissingSolverError): - solve_event(event, data, allow_fallback=False) - - def test_condition_holds_for_patched_value(self) -> None: - data = b"\x00abc" - event = make_event(data) - result = solve_event(event, data, allow_fallback=True) - patched = bytes.fromhex(result.patches[0].new_hex) + data[1:] - self.assertTrue(condition_holds(event, read_event_value(event, patched))) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_source_constraints.py b/parser-fuzzers/tests/test_source_constraints.py deleted file mode 100644 index 926991e..0000000 --- a/parser-fuzzers/tests/test_source_constraints.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -import json -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -from parser_fuzzers.source_constraints import mine_source_constraints, write_source_constraint_profile -from parser_fuzzers.structured_templates import pwg_structural_instance - - -class SourceConstraintTests(unittest.TestCase): - def test_mines_field_and_option_hints_from_source(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - source = Path(tmp) / "sample_filter.cxx" - source.write_text( - "\n".join( - [ - 'if (header.cupsBytesPerLine < row_bytes) return 1;', - 'if (!strcmp(attr->name, "PageSize")) use_page();', - ] - ) - + "\n", - encoding="utf-8", - ) - - profile = mine_source_constraints([source]) - - self.assertGreaterEqual(profile["summary"]["records"], 2) - self.assertGreater(profile["families"]["pwg"]["fields"].get("bytes_per_line", 0), 0) - self.assertGreater(profile["ppd_options"].get("PageSize", 0), 0) - - def test_source_profile_biases_template_objective(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - profile_path = Path(tmp) / "source-hints.json" - profile = { - "schema_version": "source-constraint-hints-v1", - "summary": {}, - "families": {"pwg": {"fields": {"bytes_per_line": 3}}}, - "ppd_options": {}, - "records": [], - "template_bias": { - "pwg": { - "preferred_objectives": ["short_line"], - "preferred_feedback_variants": [1], - } - }, - } - write_source_constraint_profile(profile, profile_path) - - with patch.dict( - "os.environ", - { - "SMT_FUZZER_SOURCE_CONSTRAINTS": str(profile_path), - "SMT_FUZZER_SOURCE_CONSTRAINT_RATE": "1.0", - }, - clear=False, - ): - instance = pwg_structural_instance(0) - - self.assertEqual(instance.objective, "short_line") - self.assertIn(instance.solved_by, {"z3-source", "fallback"}) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_structure_mutator.py b/parser-fuzzers/tests/test_structure_mutator.py deleted file mode 100644 index cbece20..0000000 --- a/parser-fuzzers/tests/test_structure_mutator.py +++ /dev/null @@ -1,163 +0,0 @@ -from __future__ import annotations - -import json -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -from parser_fuzzers.constraint_repair import repair_image_goal -from parser_fuzzers.dimension_expander import expand_image_goals -from parser_fuzzers.format_specs import image_goals_for_target -from parser_fuzzers.image_templates import image_feedback_instance -from parser_fuzzers.output_feedback import build_output_feedback_profile, choose_image_goal -from parser_fuzzers.structure_mutator import mutate_image_structure - - -class StructureMutatorTests(unittest.TestCase): - def test_output_feedback_counts_objective_output_shapes(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - run_dir = Path(tmp) - (run_dir / "timeline.jsonl").write_text( - "\n".join( - [ - json.dumps( - { - "target_id": "image_to_imagetopdf_feedback", - "document_description": "feedback-driven image sweep via pdf:pdf-single-image/z3-structure", - "semantic_shape": { - "output": { - "format": "pdf", - "structure": "pdf:1.3|obj:5-16|stream:1|page:1|image:1", - } - }, - } - ), - json.dumps( - { - "target_id": "image_to_imagetopdf_feedback", - "document_description": "feedback-driven image sweep via pdf-wide-image/z3-structure", - "semantic_shape": {"output": {"format": "empty", "structure": "empty"}}, - } - ), - ] - ) - + "\n", - encoding="utf-8", - ) - - profile = build_output_feedback_profile(run_dir) - goal = choose_image_goal( - target_id="image_to_imagetopdf_feedback", - slot=0, - profile=profile, - ) - - self.assertEqual(profile.objective_output_counts["image_to_imagetopdf_feedback|pdf-single-image|pdf"], 1) - self.assertNotEqual(goal.name, "pdf-single-image") - - def test_repair_image_goal_satisfies_goal_constraints(self) -> None: - goal = next(item for item in image_goals_for_target("image_to_imagetopdf_feedback") if item.name == "pdf-wide-image") - repaired = repair_image_goal(goal=goal, slot=3, expansion_level=3) - - self.assertIn(repaired.image_format, goal.allowed_formats) - self.assertGreaterEqual(repaired.width, repaired.height * 4) - self.assertEqual(repaired.payload_delta, 0) - self.assertEqual(repaired.objective, "pdf-wide-image") - - def test_structure_mutator_selects_target_specific_output_goal(self) -> None: - mutated = mutate_image_structure( - target_id="image_to_imagetoraster_feedback", - slot=0, - expansion_level=3, - ) - - self.assertEqual(mutated.output_format_goal, "cups-raster") - self.assertTrue(mutated.objective.startswith("raster-")) - self.assertGreaterEqual(mutated.width * mutated.height, 1) - - def test_target_goal_sets_include_expanded_combinations(self) -> None: - pdf_names = {goal.name for goal in image_goals_for_target("image_to_imagetopdf_feedback")} - ps_names = {goal.name for goal in image_goals_for_target("image_to_imagetops_feedback")} - raster_names = {goal.name for goal in image_goals_for_target("image_to_imagetoraster_feedback")} - - self.assertIn("pdf-interlaced-png", pdf_names) - self.assertIn("pdf-large-rgb", pdf_names) - self.assertIn("ps-bitmap-image", ps_names) - self.assertIn("ps-large-rgb", ps_names) - self.assertIn("raster-bitmap-rows", raster_names) - self.assertIn("raster-large-rgb", raster_names) - - def test_auto_dimension_expander_generates_semantic_variants(self) -> None: - goals = image_goals_for_target("image_to_imagetopdf_feedback") - with patch.dict( - "os.environ", - { - "SMT_FUZZER_AUTO_DIMENSIONS": "1", - "SMT_FUZZER_AUTO_DIMENSION_BUDGET": "64", - }, - clear=False, - ): - expanded = expand_image_goals( - target_id="image_to_imagetopdf_feedback", - goals=goals, - slot=0, - ) - - auto_goals = [goal for goal in expanded if goal.name.startswith("auto-")] - auto_names = {goal.name for goal in auto_goals} - payload_policies = {goal.payload_policy for goal in auto_goals} - comment_styles = {goal.comment_style for goal in auto_goals} - interlace_values = {goal.png_interlace for goal in auto_goals} - - self.assertGreater(len(expanded), len(goals)) - self.assertIn("short", payload_policies) - self.assertIn("extra", payload_policies) - self.assertIn(3, comment_styles) - self.assertIn(1, interlace_values) - self.assertTrue(any(name.endswith("-wide-maxval") for name in auto_names)) - self.assertTrue(any(name.endswith("-large-area") for name in auto_names)) - - def test_auto_dimension_choice_can_enter_structure_repair(self) -> None: - base_goals = image_goals_for_target("image_to_imagetopdf_feedback") - with patch.dict( - "os.environ", - { - "SMT_FUZZER_AUTO_DIMENSIONS": "1", - "SMT_FUZZER_AUTO_DIMENSION_BUDGET": "64", - }, - clear=False, - ): - goal = choose_image_goal( - target_id="image_to_imagetopdf_feedback", - slot=len(base_goals), - ) - repaired = repair_image_goal(goal=goal, slot=17, expansion_level=3) - - self.assertTrue(goal.name.startswith("auto-")) - self.assertIn(repaired.image_format, goal.allowed_formats) - self.assertGreaterEqual(repaired.width, goal.min_width) - self.assertGreaterEqual(repaired.height, goal.min_height) - self.assertGreaterEqual(repaired.width * repaired.height, goal.min_area) - self.assertEqual(repaired.objective, goal.name) - - def test_image_feedback_instance_uses_structure_mutator_when_enabled(self) -> None: - with patch.dict( - "os.environ", - { - "SMT_FUZZER_STRUCTURE_MUTATOR": "1", - "SMT_FUZZER_TARGET_ID": "image_to_imagetopdf_feedback", - "SMT_FUZZER_IMAGE_EXPANSION_LEVEL": "3", - "SMT_FUZZER_IMAGE_CYCLE_EPOCHS": "1", - }, - clear=False, - ): - instance = image_feedback_instance(0) - - self.assertTrue(instance.objective.startswith("pdf:")) - self.assertIn(instance.solved_by, {"z3-structure", "fallback-structure"}) - self.assertGreaterEqual(instance.width, 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_template_feedback.py b/parser-fuzzers/tests/test_template_feedback.py deleted file mode 100644 index bf38bcd..0000000 --- a/parser-fuzzers/tests/test_template_feedback.py +++ /dev/null @@ -1,188 +0,0 @@ -from __future__ import annotations - -import json -import tempfile -import unittest -from pathlib import Path - -from parser_fuzzers.document_harness import make_document -from parser_fuzzers.template_feedback import build_feedback_profile, write_feedback_profile, load_feedback_profile - - -class TemplateFeedbackTests(unittest.TestCase): - def test_build_feedback_profile_extracts_interesting_raster_headers(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - case_dir = root / "corpus" / "interesting" / "cups_target" / "case-000001" - case_dir.mkdir(parents=True) - document_path = case_dir / "document.ras" - document_path.write_bytes(make_document("cups_raster_structural_sweep", 1).data) - (case_dir / "meta.json").write_text( - json.dumps( - { - "target_id": "cups_target", - "case_id": 1, - "document_path": str(document_path), - "crashed": False, - "timed_out": False, - } - ), - encoding="utf-8", - ) - - profile = build_feedback_profile(root) - - self.assertEqual(len(profile.cups), 1) - self.assertEqual(profile.cups[0].fields["width"], 15) - self.assertIn("bytes_per_line", profile.cups[0].fields) - - def test_feedback_profile_round_trip(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - case_dir = root / "quarantine" / "unique" / "pwg-case" - case_dir.mkdir(parents=True) - document_path = case_dir / "document.pwg" - document_path.write_bytes(make_document("pwg_raster_structural_sweep", 2).data) - (case_dir / "meta.json").write_text( - json.dumps( - { - "target_id": "pwg_target", - "case_id": 2, - "document_path": str(document_path), - "crashed": True, - "timed_out": False, - } - ), - encoding="utf-8", - ) - output = root / "feedback.json" - write_feedback_profile(build_feedback_profile(root), output) - loaded = load_feedback_profile(output) - - self.assertEqual(len(loaded.pwg), 1) - self.assertTrue(loaded.pwg[0].crashed) - - def test_build_feedback_profile_uses_frontier_across_runs(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - older = root / "run-old" - newer = root / "run-new" - _write_interesting_case(older, "cups_target", 1, "cups_raster_structural_sweep") - _write_interesting_case(newer, "cups_target", 2, "cups_raster_structural_sweep") - (newer / "timeline.jsonl").write_text( - json.dumps( - { - "target_id": "cups_target", - "case_id": 2, - "retained_for_coverage": True, - "new_feature_count": 4, - "reached_expected_filter": True, - } - ) - + "\n", - encoding="utf-8", - ) - - profile = build_feedback_profile([older, newer], max_cases_per_kind=1) - - self.assertEqual(len(profile.cups), 1) - self.assertEqual(profile.cups[0].case_id, 2) - - def test_build_feedback_profile_extracts_image_headers(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - case_dir = root / "corpus" / "interesting" / "image_target" / "case-000003" - case_dir.mkdir(parents=True) - document_path = case_dir / "document.png" - document_path.write_bytes(make_document("image_feedback_sweep", 0).data) - (case_dir / "meta.json").write_text( - json.dumps( - { - "target_id": "image_target", - "case_id": 3, - "document_path": str(document_path), - "crashed": False, - "timed_out": False, - } - ), - encoding="utf-8", - ) - - profile = build_feedback_profile(root) - - self.assertEqual(len(profile.images), 1) - self.assertEqual(profile.images[0].kind, "image") - self.assertIn("format_id", profile.images[0].fields) - self.assertIn("width", profile.images[0].fields) - - def test_build_feedback_profile_prefers_deep_non_crashing_image_frontier(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - _write_interesting_case(root, "image_target", 0, "image_feedback_sweep", crashed=True) - _write_interesting_case(root, "image_target", 9, "image_feedback_sweep", crashed=False) - (root / "timeline.jsonl").write_text( - "\n".join( - [ - json.dumps( - { - "target_id": "image_target", - "case_id": 0, - "retained_for_coverage": True, - "new_feature_count": 2, - "reached_expected_filter": True, - "crashed": True, - "semantic_shape": {"path": {"depth_score": 8}}, - } - ), - json.dumps( - { - "target_id": "image_target", - "case_id": 9, - "retained_for_coverage": True, - "new_feature_count": 20, - "reached_expected_filter": True, - "crashed": False, - "semantic_shape": {"path": {"depth_score": 16}}, - } - ), - ] - ) - + "\n", - encoding="utf-8", - ) - - profile = build_feedback_profile(root, max_cases_per_kind=1) - - self.assertEqual(len(profile.images), 1) - self.assertEqual(profile.images[0].case_id, 9) - - -def _write_interesting_case( - root: Path, - target_id: str, - case_id: int, - document_kind: str, - *, - crashed: bool = False, -) -> None: - case_dir = root / "corpus" / "interesting" / target_id / f"case-{case_id:06d}" - case_dir.mkdir(parents=True) - extension = ".png" if document_kind == "image_feedback_sweep" else ".ras" - document_path = case_dir / f"document{extension}" - document_path.write_bytes(make_document(document_kind, case_id).data) - (case_dir / "meta.json").write_text( - json.dumps( - { - "target_id": target_id, - "case_id": case_id, - "document_path": str(document_path), - "crashed": crashed, - "timed_out": False, - } - ), - encoding="utf-8", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_template_seed_generation.py b/parser-fuzzers/tests/test_template_seed_generation.py deleted file mode 100644 index 2f4f7e4..0000000 --- a/parser-fuzzers/tests/test_template_seed_generation.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -import json -import sys -import tempfile -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from parser_fuzzers.afl_integration.template_seed_generation import generate_template_seeds - - -class TemplateSeedGenerationTests(unittest.TestCase): - def test_generates_structured_documents_without_target_execution(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - out_dir = Path(tmp) / "seeds" - summary = generate_template_seeds( - document_kind="pwg_raster_feedback_sweep", - target_id="template_probe_pwg", - output_dir=out_dir, - count=4, - extensions=[".pwg"], - ) - - self.assertEqual(summary.generated, 4) - seed_files = sorted(out_dir.glob("*.pwg")) - self.assertEqual(len(seed_files), 4) - self.assertTrue(all(path.read_bytes().startswith(b"2SaR") for path in seed_files)) - ordinary_files = sorted(path.name for path in out_dir.iterdir() if not path.name.startswith(".")) - self.assertEqual(ordinary_files, [path.name for path in seed_files]) - manifest = json.loads((out_dir.parent / "seeds-template_seed_manifest.json").read_text(encoding="utf-8")) - self.assertEqual(manifest["document_kind"], "pwg_raster_feedback_sweep") - self.assertEqual(manifest["target_id"], "template_probe_pwg") - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_template_synth.py b/parser-fuzzers/tests/test_template_synth.py deleted file mode 100644 index 126aa12..0000000 --- a/parser-fuzzers/tests/test_template_synth.py +++ /dev/null @@ -1,265 +0,0 @@ -from __future__ import annotations - -import json -import os -import struct -import tempfile -import unittest -from unittest.mock import patch - -from parser_fuzzers.document_harness import ( - OFF_CUPS_BITS_PER_PIXEL, - OFF_CUPS_BYTES_PER_LINE, - OFF_CUPS_COLOR_SPACE, - OFF_CUPS_HEIGHT, - OFF_CUPS_ROW_COUNT, - OFF_CUPS_WIDTH, - OFF_HW_RESOLUTION, - make_document, -) -from parser_fuzzers.image_templates import image_feedback_instance -from parser_fuzzers.ppd_templates import PAGE_SIZES, make_ppd -from parser_fuzzers.template_synth import ( - synthesize_cups_raster_slots, - synthesize_image_slots, - synthesize_ppd_slots, - synthesize_pwg_raster_slots, -) -from parser_fuzzers.structured_templates import cups_structural_instance, pwg_structural_instance - - -class TemplateSynthTests(unittest.TestCase): - def test_cups_raster_slots_fill_document_header(self) -> None: - slots = synthesize_cups_raster_slots(17) - document = make_document("cups_raster_coverage_sweep", 17) - header = document.data[4 : 4 + 1796] - - self.assertEqual(_u32(header, OFF_CUPS_WIDTH), slots.width) - self.assertEqual(_u32(header, OFF_CUPS_HEIGHT), slots.height) - self.assertEqual(_u32(header, OFF_CUPS_BITS_PER_PIXEL), slots.bits_per_pixel) - self.assertEqual(_u32(header, OFF_CUPS_COLOR_SPACE), slots.color_space) - self.assertGreaterEqual(_u32(header, OFF_CUPS_BYTES_PER_LINE), (slots.width * slots.bits_per_pixel + 7) // 8) - - def test_pwg_slots_fill_document_header(self) -> None: - slots = synthesize_pwg_raster_slots(23) - document = make_document("pwg_raster_coverage_sweep", 23) - header = document.data[4 : 4 + 1796] - - self.assertEqual(_u32(header, OFF_CUPS_WIDTH), slots.width) - self.assertEqual(_u32(header, OFF_CUPS_HEIGHT), slots.height) - self.assertEqual(_u32(header, OFF_CUPS_BITS_PER_PIXEL), slots.bits_per_pixel) - self.assertEqual(_u32(header, OFF_HW_RESOLUTION), slots.x_res & 0xFFFFFFFF) - - def test_structural_cups_template_fills_relation_fields(self) -> None: - instance = cups_structural_instance(19) - document = make_document("cups_raster_structural_sweep", 19) - header = document.data[4 : 4 + 1796] - - self.assertEqual(_u32(header, OFF_CUPS_WIDTH), instance.get("width")) - self.assertEqual(_u32(header, OFF_CUPS_HEIGHT), instance.get("height")) - self.assertEqual(_u32(header, OFF_CUPS_BYTES_PER_LINE), instance.get("bytes_per_line")) - self.assertEqual(_u32(header, OFF_CUPS_ROW_COUNT), instance.get("row_count")) - self.assertGreaterEqual(len(document.data), 4 + 1796 + instance.get("bytes_per_line")) - - def test_structural_pwg_template_fills_relation_fields(self) -> None: - instance = pwg_structural_instance(23) - document = make_document("pwg_raster_structural_sweep", 23) - header = document.data[4 : 4 + 1796] - - self.assertEqual(_u32(header, OFF_CUPS_WIDTH), instance.get("width")) - self.assertEqual(_u32(header, OFF_CUPS_HEIGHT), instance.get("height")) - self.assertEqual(_u32(header, OFF_CUPS_BITS_PER_PIXEL), instance.get("bits_per_pixel")) - self.assertEqual(_u32(header, OFF_CUPS_BYTES_PER_LINE), instance.get("bytes_per_line")) - self.assertEqual(_u32(header, OFF_CUPS_ROW_COUNT), instance.get("row_count")) - - def test_feedback_templates_use_profile_seed_neighborhood(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - profile_path = os.path.join(tmp, "feedback.json") - _write_feedback_profile_fixture(profile_path) - - with patch.dict("os.environ", {"SMT_FUZZER_TEMPLATE_FEEDBACK": profile_path}): - cups = make_document("cups_raster_feedback_sweep", 1) - pwg = make_document("pwg_raster_feedback_sweep", 1) - - cups_header = cups.data[4 : 4 + 1796] - pwg_header = pwg.data[4 : 4 + 1796] - self.assertEqual(cups.data[:4], b"3SaR") - self.assertEqual(pwg.data[:4], b"2SaR") - self.assertIn(_u32(cups_header, OFF_CUPS_WIDTH), {31, 32, 33}) - self.assertGreaterEqual(_u32(cups_header, OFF_CUPS_BYTES_PER_LINE), 1) - self.assertIn(_u32(pwg_header, OFF_CUPS_WIDTH), {63, 64, 65}) - - def test_feedback_expansion_level_enables_wider_variants(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - profile_path = os.path.join(tmp, "feedback.json") - _write_feedback_profile_fixture(profile_path) - - with patch.dict( - "os.environ", - { - "SMT_FUZZER_TEMPLATE_FEEDBACK": profile_path, - "SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL": "2", - }, - ): - cups = make_document("cups_raster_feedback_sweep", 10) - - self.assertIn("wide_pad_16", cups.description) - - def test_afl_feedback_seed_fields_are_sanitized_before_generation(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - profile_path = os.path.join(tmp, "afl-feedback.json") - _write_huge_afl_feedback_fixture(profile_path) - - with patch.dict( - "os.environ", - { - "SMT_FUZZER_TEMPLATE_FEEDBACK": profile_path, - "SMT_FUZZER_TEMPLATE_EXPANSION_LEVEL": "2", - }, - ): - document = make_document("pwg_raster_feedback_sweep", 0) - - header = document.data[4 : 4 + 1796] - self.assertLess(len(document.data), 1024 * 1024) - self.assertLessEqual(_u32(header, OFF_CUPS_HEIGHT), 16) - self.assertLessEqual(_u32(header, OFF_CUPS_BYTES_PER_LINE), 4096) - self.assertLessEqual(_u32(header, OFF_CUPS_ROW_COUNT), 64) - - def test_image_slots_fill_image_template(self) -> None: - slots = synthesize_image_slots(5) - document = make_document("image_coverage_sweep", 5) - - if slots.image_format.startswith("png"): - self.assertTrue(document.data.startswith(b"\x89PNG\r\n\x1a\n")) - self.assertEqual(struct.unpack(">I", document.data[16:20])[0], slots.width) - self.assertEqual(struct.unpack(">I", document.data[20:24])[0], slots.height) - else: - self.assertTrue(document.data[:1] == b"P") - self.assertIn(str(slots.width).encode("ascii"), document.data.splitlines()[1]) - - def test_image_feedback_expansion_includes_post_scaling_valid_objective(self) -> None: - with patch.dict( - "os.environ", - { - "SMT_FUZZER_IMAGE_EXPANSION_LEVEL": "3", - "SMT_FUZZER_IMAGE_VALID_BIAS": "1", - "SMT_FUZZER_IMAGE_CYCLE_EPOCHS": "1", - }, - clear=False, - ): - instances = [image_feedback_instance(index) for index in range(160)] - - deep = [item for item in instances if item.objective == "post_scaling_valid"] - self.assertTrue(deep) - self.assertTrue(any(item.width * item.height >= 3072 for item in deep)) - self.assertTrue(all(item.payload_delta == 0 for item in deep)) - - def test_ppd_slots_fill_page_size_and_options(self) -> None: - slots = synthesize_ppd_slots(19) - ppd = make_ppd("pdftoraster_coverage_options", 19) - page_name = PAGE_SIZES[slots.page_size_index][0] - - self.assertIn(f"*DefaultPageSize: {page_name}", ppd) - self.assertIn("*OpenUI *ColorModel", ppd) - self.assertIn("*OpenUI *PrintQuality", ppd) - - -def _u32(buffer: bytes, offset: int) -> int: - return struct.unpack_from(" None: - with open(profile_path, "w", encoding="utf-8") as handle: - json.dump( - { - "source_run_dir": "test", - "cups": [ - { - "kind": "cups", - "source": "test", - "target_id": "cups_raster_to_rastertopclx_structural", - "case_id": 1, - "document_path": "", - "crashed": True, - "timed_out": False, - "fields": { - "width": 31, - "height": 5, - "bits_per_pixel": 8, - "bytes_per_line": 30, - "row_count": 5, - "payload_rows": 5, - "color_space": 3, - "num_colors": 1, - "color_order": 0, - "compression": 10, - "x_res": 360, - "y_res": 1200, - }, - } - ], - "pwg": [ - { - "kind": "pwg", - "source": "test", - "target_id": "pwg_to_pclm_structural", - "case_id": 2, - "document_path": "", - "crashed": False, - "timed_out": False, - "fields": { - "width": 63, - "height": 4, - "bits_per_pixel": 16, - "bytes_per_line": 126, - "row_count": 4, - "payload_rows": 4, - "color_space": 18, - "num_colors": 1, - "color_order": 0, - "compression": 0, - "x_res": 600, - "y_res": 1200, - }, - } - ], - }, - handle, - ) - - -def _write_huge_afl_feedback_fixture(profile_path: str) -> None: - with open(profile_path, "w", encoding="utf-8") as handle: - json.dump( - { - "source_run_dir": "afl", - "cups": [], - "pwg": [ - { - "kind": "pwg", - "source": "afl-crash", - "target_id": "pwg_to_pdf_feedback_f30", - "case_id": 134, - "document_path": "", - "crashed": True, - "timed_out": False, - "fields": { - "width": 3, - "height": 256016, - "bits_per_pixel": 24, - "bytes_per_line": 13, - "row_count": 268500991, - "payload_rows": 268500991, - "x_res": 150, - "y_res": 65535, - }, - } - ], - "images": [], - }, - handle, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/tests/test_validation.py b/parser-fuzzers/tests/test_validation.py deleted file mode 100644 index c1b2554..0000000 --- a/parser-fuzzers/tests/test_validation.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -import sys -import tempfile -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) - -from parser_fuzzers.validation import validate_all, validate_bug_metadata, validate_bug_suite - - -class ValidationTests(unittest.TestCase): - def test_project_metadata_validates(self) -> None: - issues = validate_all(ROOT / "bugs", ROOT / "configs", require_local_artifacts=False) - errors = [issue for issue in issues if issue.level == "error"] - self.assertEqual(errors, []) - - def test_public_bug_suite_is_optional(self) -> None: - issues = validate_bug_suite(ROOT / "bugs", require_local_artifacts=False) - self.assertEqual([issue for issue in issues if issue.level == "error"], []) - self.assertEqual(list((ROOT / "bugs").glob("*/meta.yaml")), []) - - def test_clone_only_validation_demotes_missing_artifacts(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - meta = Path(tmp) / "meta.yaml" - meta.write_text( - "\n".join( - [ - "id: X1", - "title: clone-only metadata", - "component: cups-filters", - "bug_type: CWE-000", - "target_component: parser", - "poc_path: /private/local/reproducer.bin", - "known_poc_allowed_in_seed: false", - "timeout_sec: 3", - "memory_mb: 128", - "report_path: /definitely/missing/report.md", - "oracle:", - " reached: {}", - " triggered: {}", - " detected: {}", - ] - ) - + "\n", - encoding="utf-8", - ) - clone_issues = validate_bug_metadata(meta, require_local_artifacts=False) - strict_issues = validate_bug_metadata(meta, require_local_artifacts=True) - self.assertEqual([issue for issue in clone_issues if issue.level == "error"], []) - self.assertEqual(sum(1 for issue in clone_issues if issue.level == "warning"), 2) - self.assertEqual(sum(1 for issue in strict_issues if issue.level == "error"), 2) - - -if __name__ == "__main__": - unittest.main() diff --git a/parser-fuzzers/work/.gitkeep b/parser-fuzzers/work/.gitkeep deleted file mode 100644 index 8b13789..0000000 --- a/parser-fuzzers/work/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/parser-fuzzers/work/corpus/smt/.gitkeep b/parser-fuzzers/work/corpus/smt/.gitkeep deleted file mode 100644 index 8b13789..0000000 --- a/parser-fuzzers/work/corpus/smt/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -