From a1f5bb60e46d60e701760e257e37163f400e5f4c Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Sat, 25 Apr 2026 21:16:20 +0100 Subject: [PATCH 1/4] Fix test_tests_collected_once collecting nothing Use a pytest plugin to record collected node IDs directly instead of parsing captured stdout, and disable pytest-retry in the nested run so it does not exit with INTERNAL_ERROR. Also remove a stray breakpoint and assert that the baseline collection is non-empty. Co-Authored-By: Claude Opus 4.7 (1M context) --- ci/test_custom_linters.py | 59 +++++++++++++++++++++------------------ pyproject.toml | 1 + 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/ci/test_custom_linters.py b/ci/test_custom_linters.py index c02fcacc6..4911e8fd9 100644 --- a/ci/test_custom_linters.py +++ b/ci/test_custom_linters.py @@ -1,15 +1,11 @@ """Custom lint tests.""" from pathlib import Path -from typing import TYPE_CHECKING import pytest import yaml from beartype import beartype -if TYPE_CHECKING: - from collections.abc import Iterable - @beartype def _ci_patterns(*, repository_root: Path) -> set[str]: @@ -23,32 +19,44 @@ def _ci_patterns(*, repository_root: Path) -> set[str]: return ci_patterns +class _CollectPlugin: + """Pytest plugin that records the node IDs of collected items.""" + + def __init__(self) -> None: + """Start with an empty set of collected node IDs.""" + self.collected: set[str] = set() + + def pytest_itemcollected(self, item: pytest.Item) -> None: + """Record each collected item's node ID.""" + self.collected.add(item.nodeid) + + @beartype -def _tests_from_pattern( - *, - ci_pattern: str, - capsys: pytest.CaptureFixture[str], -) -> set[str]: +def _tests_from_pattern(*, ci_pattern: str) -> set[str]: """From a CI pattern, get all tests ``pytest`` would collect.""" - # Clear the captured output. - capsys.readouterr() - tests: Iterable[str] = set() + plugin = _CollectPlugin() pytest.main( args=[ "-q", "--collect-only", - # If there are any warnings, these obscure the output. + # Disable pytest-retry to avoid: + # ``` + # ValueError: no option named 'filtered_exceptions' + # ``` + # which causes the nested run to exit with INTERNAL_ERROR + # before any items are collected. + "-p", + "no:pytest-retry", + # Disable warnings to avoid many instances of: + # ``` + # Unknown config option: retry_delay + # ``` "--disable-warnings", ci_pattern, ], + plugins=[plugin], ) - data = capsys.readouterr().out - for line in data.splitlines(): - # We filter empty lines and lines which look like - # "9 tests collected in 0.01s". - if line and "collected in" not in line: - tests = {*tests, line} - return set(tests) + return plugin.collected def test_ci_patterns_valid(request: pytest.FixtureRequest) -> None: @@ -82,20 +90,18 @@ def test_ci_patterns_valid(request: pytest.FixtureRequest) -> None: assert collect_only_result == 0, message -def test_tests_collected_once( - *, - capsys: pytest.CaptureFixture[str], - request: pytest.FixtureRequest, -) -> None: +def test_tests_collected_once(request: pytest.FixtureRequest) -> None: """Each test in the test suite is collected exactly once. This does not necessarily mean that they are run - they may be skipped. """ ci_patterns = _ci_patterns(repository_root=request.config.rootpath) + all_tests = _tests_from_pattern(ci_pattern=".") + assert all_tests tests_to_patterns: dict[str, set[str]] = {} for pattern in ci_patterns: - tests = _tests_from_pattern(ci_pattern=pattern, capsys=capsys) + tests = _tests_from_pattern(ci_pattern=pattern) for test in tests: if test in tests_to_patterns: tests_to_patterns[test].add(pattern) @@ -110,6 +116,5 @@ def test_tests_collected_once( ) assert len(patterns) == 1, message - all_tests = _tests_from_pattern(ci_pattern=".", capsys=capsys) assert tests_to_patterns.keys() - all_tests == set() assert all_tests - tests_to_patterns.keys() == set() diff --git a/pyproject.toml b/pyproject.toml index eba475b4e..4ef7618a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -417,6 +417,7 @@ ignore_names = [ # pytest configuration "pytest_collect_file", "pytest_collection_modifyitems", + "pytest_itemcollected", "pytest_plugins", "pytest_set_filtered_exceptions", "pytest_addoption", From 9a9e460fc9bfaf48d28e7b1b61c891d7ab6e840e Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Sat, 25 Apr 2026 21:20:28 +0100 Subject: [PATCH 2/4] Add missing test files to CI matrix The matrix in test.yml was missing entries for several test files and docs, so those tests were never run in CI. Add them now that test_tests_collected_once actually verifies coverage. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 323819b55..f8706f211 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -113,11 +113,15 @@ jobs: - tests/mock_vws/test_update_target.py::TestWidth - tests/mock_vws/test_update_target.py::TestInactiveProject - tests/mock_vws/test_requests_mock_usage.py + - tests/mock_vws/test_respx_mock_usage.py - tests/mock_vws/test_flask_app_usage.py - tests/mock_vws/test_vumark_generation_api.py + - tests/mock_vws/test_target_validators.py - tests/mock_vws/test_docker.py + - ci/test_custom_linters.py - README.rst - docs/source/basic-example.rst + - docs/source/httpx-example.rst steps: - uses: actions/checkout@v6 From 00f78e659436949fadaa12348aa183b18c1c8a6f Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Sat, 25 Apr 2026 21:34:17 +0100 Subject: [PATCH 3/4] Use a single **/*.rst entry in the CI matrix Replaces the per-file rst entries with a glob. Enable bash globstar in the workflow run step so the pattern expands, and expand globs in the linter helper since pytest.main is invoked without a shell. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/test.yml | 5 ++--- ci/test_custom_linters.py | 21 +++++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f8706f211..213177eeb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -119,9 +119,7 @@ jobs: - tests/mock_vws/test_target_validators.py - tests/mock_vws/test_docker.py - ci/test_custom_linters.py - - README.rst - - docs/source/basic-example.rst - - docs/source/httpx-example.rst + - '**/*.rst' steps: - uses: actions/checkout@v6 @@ -163,6 +161,7 @@ jobs: - name: Run tests run: | + shopt -s globstar uv run --extra=dev \ coverage run -m pytest \ -s \ diff --git a/ci/test_custom_linters.py b/ci/test_custom_linters.py index 4911e8fd9..83cd8a330 100644 --- a/ci/test_custom_linters.py +++ b/ci/test_custom_linters.py @@ -7,6 +7,23 @@ from beartype import beartype +@beartype +def _expand_pattern(*, ci_pattern: str) -> list[str]: + """Expand a CI pattern, treating shell-style globs as recursive globs. + + Hidden directories (such as ``.venv``) are skipped to match bash's + default globbing behaviour. Patterns without glob metacharacters are + returned unchanged. + """ + if not any(char in ci_pattern for char in "*?["): + return [ci_pattern] + return [ + path.as_posix() + for path in Path().glob(pattern=ci_pattern) + if not any(part.startswith(".") for part in path.parts) + ] + + @beartype def _ci_patterns(*, repository_root: Path) -> set[str]: """Return the CI patterns given in the CI configuration file.""" @@ -52,7 +69,7 @@ def _tests_from_pattern(*, ci_pattern: str) -> set[str]: # Unknown config option: retry_delay # ``` "--disable-warnings", - ci_pattern, + *_expand_pattern(ci_pattern=ci_pattern), ], plugins=[plugin], ) @@ -71,7 +88,7 @@ def test_ci_patterns_valid(request: pytest.FixtureRequest) -> None: collect_only_result = pytest.main( args=[ "--collect-only", - ci_pattern, + *_expand_pattern(ci_pattern=ci_pattern), # Disable pytest-retry to avoid: # ``` # ValueError: no option named 'filtered_exceptions' From 413ab1a5901ef5cf14114f1f9cdf623c329f0c0c Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Sat, 25 Apr 2026 21:41:23 +0100 Subject: [PATCH 4/4] Use docs/ as a single CI matrix entry instead of per-rst entries Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/test.yml | 4 ++-- ci/test_custom_linters.py | 21 ++------------------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 213177eeb..1b61c7ff8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -119,7 +119,8 @@ jobs: - tests/mock_vws/test_target_validators.py - tests/mock_vws/test_docker.py - ci/test_custom_linters.py - - '**/*.rst' + - README.rst + - docs/ steps: - uses: actions/checkout@v6 @@ -161,7 +162,6 @@ jobs: - name: Run tests run: | - shopt -s globstar uv run --extra=dev \ coverage run -m pytest \ -s \ diff --git a/ci/test_custom_linters.py b/ci/test_custom_linters.py index 83cd8a330..4911e8fd9 100644 --- a/ci/test_custom_linters.py +++ b/ci/test_custom_linters.py @@ -7,23 +7,6 @@ from beartype import beartype -@beartype -def _expand_pattern(*, ci_pattern: str) -> list[str]: - """Expand a CI pattern, treating shell-style globs as recursive globs. - - Hidden directories (such as ``.venv``) are skipped to match bash's - default globbing behaviour. Patterns without glob metacharacters are - returned unchanged. - """ - if not any(char in ci_pattern for char in "*?["): - return [ci_pattern] - return [ - path.as_posix() - for path in Path().glob(pattern=ci_pattern) - if not any(part.startswith(".") for part in path.parts) - ] - - @beartype def _ci_patterns(*, repository_root: Path) -> set[str]: """Return the CI patterns given in the CI configuration file.""" @@ -69,7 +52,7 @@ def _tests_from_pattern(*, ci_pattern: str) -> set[str]: # Unknown config option: retry_delay # ``` "--disable-warnings", - *_expand_pattern(ci_pattern=ci_pattern), + ci_pattern, ], plugins=[plugin], ) @@ -88,7 +71,7 @@ def test_ci_patterns_valid(request: pytest.FixtureRequest) -> None: collect_only_result = pytest.main( args=[ "--collect-only", - *_expand_pattern(ci_pattern=ci_pattern), + ci_pattern, # Disable pytest-retry to avoid: # ``` # ValueError: no option named 'filtered_exceptions'