feat: add coverage report generation with cargo-llvm-cov

.github/workflows/Coverage.yml

@@ -0,0 +1,108 @@
+# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
+
+name: Weekly Coverage
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/Coverage.yml
+  schedule:
+    # Runs every Monday at 06:00 UTC
+    - cron: '0 6 * * 1'
+  workflow_dispatch: # Allow manual trigger
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: full
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  coverage:
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        hypervisor: [kvm]
+        cpu: [amd]
+    runs-on: ${{ fromJson(
+        format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-{0}-{1}"]',
+          matrix.hypervisor,
+          matrix.cpu)) }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: hyperlight-dev/ci-setup-workflow@v1.9.0
+        with:
+          rust-toolchain: "1.89"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Fix cargo home permissions
+        run: |
+          sudo chown -R $(id -u):$(id -g) /opt/cargo || true
+
+      - name: Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: "${{ runner.os }}-debug"
+          cache-on-failure: "true"
+
+      - name: Build guest binaries
+        run: just guests
+
+      - name: Install nightly toolchain
+        run: |
+          rustup toolchain install nightly
+          rustup component add llvm-tools --toolchain nightly
+
+      - name: Generate coverage report
+        run: just coverage-ci ${{ matrix.hypervisor }}
+
+      - name: Coverage summary
+        run: |
+          echo '## Code Coverage Report' >> $GITHUB_STEP_SUMMARY
+          echo '' >> $GITHUB_STEP_SUMMARY
+          if [ -f target/coverage/summary.txt ]; then
+            echo '```' >> $GITHUB_STEP_SUMMARY
+            cat target/coverage/summary.txt >> $GITHUB_STEP_SUMMARY
+            echo '```' >> $GITHUB_STEP_SUMMARY
+          else
+            echo 'Coverage report was not generated.' >> $GITHUB_STEP_SUMMARY
+          fi
+          echo '' >> $GITHUB_STEP_SUMMARY
+          echo '> For a detailed per-file breakdown, download the **HTML coverage report** from the Artifacts section below.' >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload HTML coverage report
+        uses: actions/upload-artifact@v7
+        with:
+          name: coverage-html-${{ matrix.hypervisor }}-${{ matrix.cpu }}
+          path: target/coverage/html/
+          if-no-files-found: error
+
+      - name: Upload LCOV coverage report
+        uses: actions/upload-artifact@v7
+        with:
+          name: coverage-lcov-${{ matrix.hypervisor }}-${{ matrix.cpu }}
+          path: target/coverage/lcov.info
+          if-no-files-found: error
+
+  notify-failure:
+    runs-on: ubuntu-latest
+    needs: [coverage]
+    if: always() && needs.coverage.result == 'failure'
+    permissions:
+      issues: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Notify Coverage Failure
+        run: ./dev/notify-ci-failure.sh --title="Weekly Coverage Failure - ${{ github.run_number }}" --labels="area/ci-periodics,area/testing,lifecycle/needs-review"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Justfile

@@ -459,6 +459,105 @@ fuzz-trace-timed max_time fuzz-target="fuzz_guest_trace":
 build-trace-fuzzers:
     cargo +nightly fuzz build fuzz_guest_trace --features trace
 
+####################
+### COVERAGE #######
+####################
+
+# install cargo-llvm-cov if not already installed and ensure nightly toolchain + llvm-tools are available
+ensure-cargo-llvm-cov:
+    command -v cargo-llvm-cov >/dev/null 2>&1 || cargo install cargo-llvm-cov --locked
+    rustup toolchain install nightly 2>/dev/null
+    rustup component add llvm-tools --toolchain nightly 2>/dev/null
+
+# host-side packages to collect coverage for (guest/no_std crates are excluded because they
+# define #[panic_handler] and cannot be compiled for the host target under coverage instrumentation)
+coverage-packages := "-p hyperlight-common -p hyperlight-host -p hyperlight-testing -p hyperlight-component-util -p hyperlight-component-macro"
+
+# run all tests and examples with coverage instrumentation, collecting profdata without
+# generating a report. Mirrors test-like-ci + run-examples-like-ci to exercise all code paths
+# across all feature combinations. Uses nightly for branch coverage.
+#
+# Uses the show-env approach so that cargo produces separate binaries per feature combination.
+# This avoids "mismatched data" warnings that occur when `cargo llvm-cov --no-report` recompiles
+# a crate with different features, overwriting the previous binary and orphaning its profraw data.
+#
+# (run `just guests` first to build guest binaries)
+coverage-run hypervisor="kvm": ensure-cargo-llvm-cov
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    # Set up coverage instrumentation environment variables (RUSTFLAGS, LLVM_PROFILE_FILE, etc.)
+    # and clean previous artifacts. All subsequent cargo commands inherit instrumentation.
+    source <(cargo +nightly llvm-cov show-env --export-prefix --branch)
+    cargo +nightly llvm-cov clean --workspace
+
+    # tests with default features (all drivers; skip stress tests — too slow under instrumentation)
+    cargo +nightly test {{ coverage-packages }} --tests -- --skip stress_test
+
+    # tests with single driver + build-metadata
+    cargo +nightly test {{ coverage-packages }} --no-default-features --features build-metadata,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --tests -- --skip stress_test
+
+    # isolated tests (require running separately due to global state)
+    cargo +nightly test -p hyperlight-host --lib -- sandbox::uninitialized::tests::test_log_trace --exact --ignored
+    cargo +nightly test -p hyperlight-host --lib -- sandbox::outb::tests::test_log_outb_log --exact --ignored
+    cargo +nightly test -p hyperlight-host --test integration_test -- log_message --exact --ignored
+    cargo +nightly test -p hyperlight-host --no-default-features -F function_call_metrics,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --lib -- metrics::tests::test_metrics_are_emitted --exact
+
+    # integration test with executable_heap feature
+    cargo +nightly test {{ coverage-packages }} --no-default-features -F executable_heap,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --test integration_test -- execute_on_heap
+
+    # crashdump tests + example
+    cargo +nightly test {{ coverage-packages }} --no-default-features --features crashdump,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --tests -- test_crashdump
+    cargo +nightly run --no-default-features --features crashdump,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --example crashdump
+
+    # tracing feature tests (host-side only; hyperlight-guest-tracing is no_std)
+    cargo +nightly test -p hyperlight-common --no-default-features --features trace_guest --tests -- --skip stress_test
+    cargo +nightly test -p hyperlight-host --no-default-features --features trace_guest,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --tests -- --skip stress_test
+
+    # examples: metrics, logging, tracing
+    cargo +nightly run --example metrics
+    cargo +nightly run --no-default-features -F function_call_metrics,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --example metrics
+    cargo +nightly run --example logging
+    cargo +nightly run --example tracing
+    cargo +nightly run --no-default-features -F function_call_metrics,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --example tracing
+    cargo +nightly test --no-default-features -F gdb,{{ if hypervisor == "mshv3" { "mshv3" } else { "kvm" } }} --example guest-debugging
+
+# generate a text coverage summary to stdout
+# for this to work you need to run `coverage-run hypervisor` beforehand
+coverage hypervisor="kvm":
+    #!/usr/bin/env bash
+    set -euo pipefail
+    source <(cargo +nightly llvm-cov show-env --export-prefix --branch)
+    cargo +nightly llvm-cov report
+
+# generate an HTML coverage report to target/coverage/html/
+# for this to work you need to run `coverage-run hypervisor` beforehand
+coverage-html hypervisor="kvm":
+    #!/usr/bin/env bash
+    set -euo pipefail
+    source <(cargo +nightly llvm-cov show-env --export-prefix --branch)
+    cargo +nightly llvm-cov report --html --output-dir target/coverage/html
+
+# generate LCOV coverage output to target/coverage/lcov.info
+# for this to work you need to run `coverage-run hypervisor` beforehand
+coverage-lcov hypervisor="kvm":
+    #!/usr/bin/env bash
+    set -euo pipefail
+    source <(cargo +nightly llvm-cov show-env --export-prefix --branch)
+    mkdir -p target/coverage
+    cargo +nightly llvm-cov report --lcov --output-path target/coverage/lcov.info
+
+# generate all coverage reports for CI: HTML + LCOV + text summary.
+# (run `just guests` first to build guest binaries)
+coverage-ci hypervisor="kvm": (coverage-run hypervisor)
+    #!/usr/bin/env bash
+    set -euo pipefail
+    source <(cargo +nightly llvm-cov show-env --export-prefix --branch)
+    mkdir -p target/coverage
+    cargo +nightly llvm-cov report --html --output-dir target/coverage/html
+    cargo +nightly llvm-cov report --lcov --output-path target/coverage/lcov.info
+    cargo +nightly llvm-cov report | tee target/coverage/summary.txt
+
 ###################
 ### FLATBUFFERS ###
 ###################

docs/how-to-run-coverage.md

@@ -0,0 +1,111 @@
+# How to Run Coverage
+
+This guide explains how to generate code coverage reports for Hyperlight.
+
+## Prerequisites
+
+- A working Rust toolchain
+- Rust nightly toolchain (required for branch coverage; installed automatically by the `just` recipes)
+- [cargo-llvm-cov](https://github.com/taiki-e/cargo-llvm-cov) (installed automatically by the `just` recipes)
+- Guest binaries must be built first: `just guests`
+
+## Local Usage
+
+Coverage is a two-step process: first **collect** profiling data by running the tests, then **generate** a report in the desired format.
+
+### Step 1: Run tests with coverage instrumentation
+
+Build guest binaries (required before running coverage):
+
+```sh
+just guests
+```
+
+Run all tests and examples under coverage instrumentation:
+
+```sh
+just coverage-run
+```
+
+This collects profiling data without generating a report. You only need to run this once — you can then generate multiple report formats from the same data.
+
+### Step 2: Generate a report
+
+#### Text Summary
+
+Print a coverage summary to the terminal:
+
+```sh
+just coverage
+```
+
+#### HTML Report
+
+Generate a browsable HTML report in `target/coverage/html/`:
+
+```sh
+just coverage-html
+```
+
+Open `target/coverage/html/index.html` in a browser to explore per-file and per-line coverage.
+
+#### LCOV Output
+
+Generate an LCOV file at `target/coverage/lcov.info` for use with external tools or CI integrations:
+
+```sh
+just coverage-lcov
+```
+
+## Available Recipes
+
+| Recipe | Output | Description |
+|---|---|---|
+| `just coverage-run` | profiling data | Runs tests with coverage instrumentation (must be run first) |
+| `just coverage` | stdout | Text summary of line coverage |
+| `just coverage-html` | `target/coverage/html/` | HTML report for browsing |
+| `just coverage-lcov` | `target/coverage/lcov.info` | LCOV format for tooling |
+| `just coverage-ci <hypervisor>` | All of the above | CI recipe: runs tests + generates HTML + LCOV + text summary |
+
+> **Note:** `coverage`, `coverage-html`, and `coverage-lcov` require `coverage-run` to have been executed first. Only `coverage-ci` runs tests and generates all reports in a single command.
+
+## CI Integration
+
+Coverage runs automatically on a **weekly schedule** (every Monday at 06:00 UTC) via the `Coverage.yml` workflow. It can also be triggered manually from the Actions tab using `workflow_dispatch`. The workflow runs on a single configuration (kvm/amd) to keep resource usage reasonable. It:
+
+1. Builds guest binaries (`just guests`)
+2. Runs `just coverage-ci kvm` — this mirrors `test-like-ci` by running multiple test phases with different feature combinations and merging the results into a single coverage report
+3. Displays a coverage summary directly in the **GitHub Actions Job Summary** (visible on the workflow run page)
+4. Uploads the full HTML report and LCOV file as downloadable build artifacts
+
+### Viewing Coverage Results
+
+- **Quick view**: Open the workflow run in the Actions tab — the coverage table is displayed in the **Job Summary** section at the bottom of the run page.
+- **Detailed view**: Download the `coverage-html-*` artifact from the Artifacts section, extract the ZIP, and open `index.html` in a browser for per-file, per-line drill-down.
+- **Tooling integration**: Download the `coverage-lcov-*` artifact for use with IDE plugins, Codecov, Coveralls, or other coverage services.
+
+## Why cargo-llvm-cov
+
+We use [cargo-llvm-cov](https://github.com/taiki-e/cargo-llvm-cov) over alternatives like [tarpaulin](https://github.com/xd009642/tarpaulin) and [grcov](https://github.com/mozilla/grcov) for three reasons:
+
+- **Cross-platform**: Works on Linux, macOS, and Windows. Tarpaulin's default `ptrace` backend is Linux-only (can be configured to use LLVM, but has some limitations), which doesn't fit Hyperlight's multi-OS target. `cargo-llvm-cov` uses LLVM's built-in instrumentation, so coverage works the same way everywhere.
+- **Simple setup**: A single `cargo install` plus `rustup component add llvm-tools` — no manual profdata wrangling or multi-step pipelines. `grcov` requires setting environment variables, collecting raw profiles, and running a separate post-processing step.
+- **Accurate**: LLVM source-based instrumentation provides precise line, region, and branch coverage with minimal overhead, directly mapped to source code. Tarpaulin's `ptrace` approach can produce inaccuracies on certain language features and is limited to line coverage by default.
+
+## How It Works
+
+`cargo-llvm-cov` instruments Rust code using LLVM's source-based code coverage. It replaces `cargo test` — when you run `cargo llvm-cov`, it compiles the project with coverage instrumentation, runs the test suite, and then merges the raw profiling data into a human-readable report. The nightly toolchain is used to enable **branch coverage** (`--branch`).
+
+The `coverage-run` recipe mirrors the `test-like-ci` + `run-examples-like-ci` workflows by running all test phases and examples with different feature combinations:
+
+1. **Default features** — all drivers enabled (kvm + mshv3 + build-metadata)
+2. **Single driver** — only one hypervisor driver + build-metadata
+3. **Isolated tests** — tests that require running separately due to global state
+4. **Integration tests** — including `executable_heap` feature
+5. **Crashdump** — tests + example with the `crashdump` feature enabled
+6. **Tracing** — tests with `trace_guest` feature (host-side crates only)
+7. **Examples** — metrics, logging, tracing (with and without `function_call_metrics`), guest-debugging (with `gdb` feature)
+
+Each phase uses `--no-report` to accumulate raw profiling data. The report recipes (`coverage`, `coverage-html`, `coverage-lcov`) then generate the desired output format from the collected data. `coverage-ci` combines both steps into a single command.
+
+Coverage is collected for the host-side workspace crates (`hyperlight_common`, `hyperlight_host`, `hyperlight_testing`, `hyperlight_component_util`, `hyperlight_component_macro`). Guest crates (`hyperlight-guest`, `hyperlight-guest-bin`, `hyperlight-guest-capi`, `hyperlight-guest-tracing`) and the `fuzz` crate are excluded because guest crates are `no_std` and cannot be compiled for the host target under coverage instrumentation.