From bf8894e70a40662af012a33f85c8cbbe05fb19c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Tue, 24 Mar 2026 10:02:30 +0000 Subject: [PATCH] feat: add coverage report generation with cargo-llvm-cov MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add coverage infrastructure for the Hyperlight project: - Justfile: add coverage-run, coverage, coverage-html, coverage-lcov, and coverage-ci recipes using cargo-llvm-cov for LLVM source-based code coverage. Tests run once via coverage-run; report recipes just generate the desired output format from collected profdata. - CI: add Coverage.yml weekly workflow (Monday 06:00 UTC, manual trigger) running on kvm/amd with self-built guest binaries - coverage-ci mirrors test-like-ci by running multiple test phases with different feature combinations (default, single-driver, crashdump, tracing) and merging profdata into a single unified report - Coverage summary is displayed in the GitHub Actions Job Summary for quick viewing; full HTML report is downloadable as an artifact - docs: add how-to-run-coverage.md with local and CI usage instructions Guest/no_std crates are excluded from coverage because they define coverage instrumentation. Coverage targets host-side crates only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Doru Blânzeanu --- .github/workflows/Coverage.yml | 108 ++++++++++++++++++++++++++++++++ Justfile | 99 +++++++++++++++++++++++++++++ docs/how-to-run-coverage.md | 111 +++++++++++++++++++++++++++++++++ 3 files changed, 318 insertions(+) create mode 100644 .github/workflows/Coverage.yml create mode 100644 docs/how-to-run-coverage.md diff --git a/.github/workflows/Coverage.yml b/.github/workflows/Coverage.yml new file mode 100644 index 000000000..9753ecd30 --- /dev/null +++ b/.github/workflows/Coverage.yml @@ -0,0 +1,108 @@ +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json + +name: Weekly Coverage + +on: + pull_request: + paths: + - .github/workflows/Coverage.yml + schedule: + # Runs every Monday at 06:00 UTC + - cron: '0 6 * * 1' + workflow_dispatch: # Allow manual trigger + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: full + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + coverage: + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + hypervisor: [kvm] + cpu: [amd] + runs-on: ${{ fromJson( + format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-{0}-{1}"]', + matrix.hypervisor, + matrix.cpu)) }} + steps: + - uses: actions/checkout@v6 + + - uses: hyperlight-dev/ci-setup-workflow@v1.9.0 + with: + rust-toolchain: "1.89" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Fix cargo home permissions + run: | + sudo chown -R $(id -u):$(id -g) /opt/cargo || true + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "${{ runner.os }}-debug" + cache-on-failure: "true" + + - name: Build guest binaries + run: just guests + + - name: Install nightly toolchain + run: | + rustup toolchain install nightly + rustup component add llvm-tools --toolchain nightly + + - name: Generate coverage report + run: just coverage-ci ${{ matrix.hypervisor }} + + - name: Coverage summary + run: | + echo '## Code Coverage Report' >> $GITHUB_STEP_SUMMARY + echo '' >> $GITHUB_STEP_SUMMARY + if [ -f target/coverage/summary.txt ]; then + echo '```' >> $GITHUB_STEP_SUMMARY + cat target/coverage/summary.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + else + echo 'Coverage report was not generated.' >> $GITHUB_STEP_SUMMARY + fi + echo '' >> $GITHUB_STEP_SUMMARY + echo '> For a detailed per-file breakdown, download the **HTML coverage report** from the Artifacts section below.' >> $GITHUB_STEP_SUMMARY + + - name: Upload HTML coverage report + uses: actions/upload-artifact@v7 + with: + name: coverage-html-${{ matrix.hypervisor }}-${{ matrix.cpu }} + path: target/coverage/html/ + if-no-files-found: error + + - name: Upload LCOV coverage report + uses: actions/upload-artifact@v7 + with: + name: coverage-lcov-${{ matrix.hypervisor }}-${{ matrix.cpu }} + path: target/coverage/lcov.info + if-no-files-found: error + + notify-failure: + runs-on: ubuntu-latest + needs: [coverage] + if: always() && needs.coverage.result == 'failure' + permissions: + issues: write + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Notify Coverage Failure + run: ./dev/notify-ci-failure.sh --title="Weekly Coverage Failure - ${{ github.run_number }}" --labels="area/ci-periodics,area/testing,lifecycle/needs-review" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Justfile b/Justfile index d71630d47..8b0578c7f 100644 --- a/Justfile +++ b/Justfile @@ -459,6 +459,105 @@ fuzz-trace-timed max_time fuzz-target="fuzz_guest_trace": build-trace-fuzzers: cargo +nightly fuzz build fuzz_guest_trace --features trace +#################### +### COVERAGE ####### +#################### + +# install cargo-llvm-cov if not already installed and ensure nightly toolchain + llvm-tools are available +ensure-cargo-llvm-cov: + command -v cargo-llvm-cov >/dev/null 2>&1 || cargo install cargo-llvm-cov --locked + rustup toolchain install nightly 2>/dev/null + rustup component add llvm-tools --toolchain nightly 2>/dev/null + +# host-side packages to collect coverage for (guest/no_std crates are excluded because they +# define #[panic_handler] and cannot be compiled for the host target under coverage instrumentation) +coverage-packages := "-p hyperlight-common -p hyperlight-host -p hyperlight-testing -p hyperlight-component-util -p hyperlight-component-macro" + +# run all tests and examples with coverage instrumentation, collecting profdata without +# generating a report. Mirrors test-like-ci + run-examples-like-ci to exercise all code paths +# across all feature combinations. Uses nightly for branch coverage. +# +# Uses the show-env approach so that cargo produces separate binaries per feature combination. +# This avoids "mismatched data" warnings that occur when `cargo llvm-cov --no-report` recompiles +# a crate with different features, overwriting the previous binary and orphaning its profraw data. +# +# (run `just guests` first to build guest binaries) +coverage-run hypervisor="kvm": ensure-cargo-llvm-cov + #!/usr/bin/env bash + set -euo pipefail + + # Set up coverage instrumentation environment variables (RUSTFLAGS, LLVM_PROFILE_FILE, etc.) + # and clean previous artifacts. All subsequent cargo commands inherit instrumentation. + source <(cargo +nightly llvm-cov show-env --export-prefix --branch) + cargo +nightly llvm-cov clean --workspace + + # tests with default features (all drivers; skip stress tests — too slow under instrumentation) + cargo +nightly test {{ coverage-packages }} --tests -- --skip stress_test + + # tests with single driver + build-metadata + cargo +nightly test {{ coverage-packages }} --no-default-features --features build-metadata,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --tests -- --skip stress_test + + # isolated tests (require running separately due to global state) + cargo +nightly test -p hyperlight-host --lib -- sandbox::uninitialized::tests::test_log_trace --exact --ignored + cargo +nightly test -p hyperlight-host --lib -- sandbox::outb::tests::test_log_outb_log --exact --ignored + cargo +nightly test -p hyperlight-host --test integration_test -- log_message --exact --ignored + cargo +nightly test -p hyperlight-host --no-default-features -F function_call_metrics,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --lib -- metrics::tests::test_metrics_are_emitted --exact + + # integration test with executable_heap feature + cargo +nightly test {{ coverage-packages }} --no-default-features -F executable_heap,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --test integration_test -- execute_on_heap + + # crashdump tests + example + cargo +nightly test {{ coverage-packages }} --no-default-features --features crashdump,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --tests -- test_crashdump + cargo +nightly run --no-default-features --features crashdump,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --example crashdump + + # tracing feature tests (host-side only; hyperlight-guest-tracing is no_std) + cargo +nightly test -p hyperlight-common --no-default-features --features trace_guest --tests -- --skip stress_test + cargo +nightly test -p hyperlight-host --no-default-features --features trace_guest,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --tests -- --skip stress_test + + # examples: metrics, logging, tracing + cargo +nightly run --example metrics + cargo +nightly run --no-default-features -F function_call_metrics,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --example metrics + cargo +nightly run --example logging + cargo +nightly run --example tracing + cargo +nightly run --no-default-features -F function_call_metrics,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --example tracing + cargo +nightly test --no-default-features -F gdb,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --example guest-debugging + +# generate a text coverage summary to stdout +# for this to work you need to run `coverage-run hypervisor` beforehand +coverage hypervisor="kvm": + #!/usr/bin/env bash + set -euo pipefail + source <(cargo +nightly llvm-cov show-env --export-prefix --branch) + cargo +nightly llvm-cov report + +# generate an HTML coverage report to target/coverage/html/ +# for this to work you need to run `coverage-run hypervisor` beforehand +coverage-html hypervisor="kvm": + #!/usr/bin/env bash + set -euo pipefail + source <(cargo +nightly llvm-cov show-env --export-prefix --branch) + cargo +nightly llvm-cov report --html --output-dir target/coverage/html + +# generate LCOV coverage output to target/coverage/lcov.info +# for this to work you need to run `coverage-run hypervisor` beforehand +coverage-lcov hypervisor="kvm": + #!/usr/bin/env bash + set -euo pipefail + source <(cargo +nightly llvm-cov show-env --export-prefix --branch) + mkdir -p target/coverage + cargo +nightly llvm-cov report --lcov --output-path target/coverage/lcov.info + +# generate all coverage reports for CI: HTML + LCOV + text summary. +# (run `just guests` first to build guest binaries) +coverage-ci hypervisor="kvm": (coverage-run hypervisor) + #!/usr/bin/env bash + set -euo pipefail + source <(cargo +nightly llvm-cov show-env --export-prefix --branch) + mkdir -p target/coverage + cargo +nightly llvm-cov report --html --output-dir target/coverage/html + cargo +nightly llvm-cov report --lcov --output-path target/coverage/lcov.info + cargo +nightly llvm-cov report | tee target/coverage/summary.txt + ################### ### FLATBUFFERS ### ################### diff --git a/docs/how-to-run-coverage.md b/docs/how-to-run-coverage.md new file mode 100644 index 000000000..7d2b41038 --- /dev/null +++ b/docs/how-to-run-coverage.md @@ -0,0 +1,111 @@ +# How to Run Coverage + +This guide explains how to generate code coverage reports for Hyperlight. + +## Prerequisites + +- A working Rust toolchain +- Rust nightly toolchain (required for branch coverage; installed automatically by the `just` recipes) +- [cargo-llvm-cov](https://github.com/taiki-e/cargo-llvm-cov) (installed automatically by the `just` recipes) +- Guest binaries must be built first: `just guests` + +## Local Usage + +Coverage is a two-step process: first **collect** profiling data by running the tests, then **generate** a report in the desired format. + +### Step 1: Run tests with coverage instrumentation + +Build guest binaries (required before running coverage): + +```sh +just guests +``` + +Run all tests and examples under coverage instrumentation: + +```sh +just coverage-run +``` + +This collects profiling data without generating a report. You only need to run this once — you can then generate multiple report formats from the same data. + +### Step 2: Generate a report + +#### Text Summary + +Print a coverage summary to the terminal: + +```sh +just coverage +``` + +#### HTML Report + +Generate a browsable HTML report in `target/coverage/html/`: + +```sh +just coverage-html +``` + +Open `target/coverage/html/index.html` in a browser to explore per-file and per-line coverage. + +#### LCOV Output + +Generate an LCOV file at `target/coverage/lcov.info` for use with external tools or CI integrations: + +```sh +just coverage-lcov +``` + +## Available Recipes + +| Recipe | Output | Description | +|---|---|---| +| `just coverage-run` | profiling data | Runs tests with coverage instrumentation (must be run first) | +| `just coverage` | stdout | Text summary of line coverage | +| `just coverage-html` | `target/coverage/html/` | HTML report for browsing | +| `just coverage-lcov` | `target/coverage/lcov.info` | LCOV format for tooling | +| `just coverage-ci ` | All of the above | CI recipe: runs tests + generates HTML + LCOV + text summary | + +> **Note:** `coverage`, `coverage-html`, and `coverage-lcov` require `coverage-run` to have been executed first. Only `coverage-ci` runs tests and generates all reports in a single command. + +## CI Integration + +Coverage runs automatically on a **weekly schedule** (every Monday at 06:00 UTC) via the `Coverage.yml` workflow. It can also be triggered manually from the Actions tab using `workflow_dispatch`. The workflow runs on a single configuration (kvm/amd) to keep resource usage reasonable. It: + +1. Builds guest binaries (`just guests`) +2. Runs `just coverage-ci kvm` — this mirrors `test-like-ci` by running multiple test phases with different feature combinations and merging the results into a single coverage report +3. Displays a coverage summary directly in the **GitHub Actions Job Summary** (visible on the workflow run page) +4. Uploads the full HTML report and LCOV file as downloadable build artifacts + +### Viewing Coverage Results + +- **Quick view**: Open the workflow run in the Actions tab — the coverage table is displayed in the **Job Summary** section at the bottom of the run page. +- **Detailed view**: Download the `coverage-html-*` artifact from the Artifacts section, extract the ZIP, and open `index.html` in a browser for per-file, per-line drill-down. +- **Tooling integration**: Download the `coverage-lcov-*` artifact for use with IDE plugins, Codecov, Coveralls, or other coverage services. + +## Why cargo-llvm-cov + +We use [cargo-llvm-cov](https://github.com/taiki-e/cargo-llvm-cov) over alternatives like [tarpaulin](https://github.com/xd009642/tarpaulin) and [grcov](https://github.com/mozilla/grcov) for three reasons: + +- **Cross-platform**: Works on Linux, macOS, and Windows. Tarpaulin's default `ptrace` backend is Linux-only (can be configured to use LLVM, but has some limitations), which doesn't fit Hyperlight's multi-OS target. `cargo-llvm-cov` uses LLVM's built-in instrumentation, so coverage works the same way everywhere. +- **Simple setup**: A single `cargo install` plus `rustup component add llvm-tools` — no manual profdata wrangling or multi-step pipelines. `grcov` requires setting environment variables, collecting raw profiles, and running a separate post-processing step. +- **Accurate**: LLVM source-based instrumentation provides precise line, region, and branch coverage with minimal overhead, directly mapped to source code. Tarpaulin's `ptrace` approach can produce inaccuracies on certain language features and is limited to line coverage by default. + +## How It Works + +`cargo-llvm-cov` instruments Rust code using LLVM's source-based code coverage. It replaces `cargo test` — when you run `cargo llvm-cov`, it compiles the project with coverage instrumentation, runs the test suite, and then merges the raw profiling data into a human-readable report. The nightly toolchain is used to enable **branch coverage** (`--branch`). + +The `coverage-run` recipe mirrors the `test-like-ci` + `run-examples-like-ci` workflows by running all test phases and examples with different feature combinations: + +1. **Default features** — all drivers enabled (kvm + mshv3 + build-metadata) +2. **Single driver** — only one hypervisor driver + build-metadata +3. **Isolated tests** — tests that require running separately due to global state +4. **Integration tests** — including `executable_heap` feature +5. **Crashdump** — tests + example with the `crashdump` feature enabled +6. **Tracing** — tests with `trace_guest` feature (host-side crates only) +7. **Examples** — metrics, logging, tracing (with and without `function_call_metrics`), guest-debugging (with `gdb` feature) + +Each phase uses `--no-report` to accumulate raw profiling data. The report recipes (`coverage`, `coverage-html`, `coverage-lcov`) then generate the desired output format from the collected data. `coverage-ci` combines both steps into a single command. + +Coverage is collected for the host-side workspace crates (`hyperlight_common`, `hyperlight_host`, `hyperlight_testing`, `hyperlight_component_util`, `hyperlight_component_macro`). Guest crates (`hyperlight-guest`, `hyperlight-guest-bin`, `hyperlight-guest-capi`, `hyperlight-guest-tracing`) and the `fuzz` crate are excluded because guest crates are `no_std` and cannot be compiled for the host target under coverage instrumentation.