From c516aee0811371619430b63b5d3a17a2c0518c3d Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Sat, 25 Apr 2026 07:10:21 +0100 Subject: [PATCH 1/4] Fix vacuous meta-test and add missing CI matrix entries The in-process pytest.main() in ci/test_custom_linters.py installs its own capture, so capsys saw an empty string and tests passed vacuously. Switch to a collection-hook plugin that reads node IDs directly. This surfaces three test files missing from the ci-tests matrix (test_respx_mock_usage.py, test_target_validators.py, docs/source/httpx-example.rst) plus the meta-tests themselves; add them. Closes #3128. --- .github/workflows/test.yml | 4 ++ ci/test_custom_linters.py | 83 ++++++++++++++++++-------------------- 2 files changed, 43 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 323819b55..b64ba7faf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -113,11 +113,15 @@ jobs: - tests/mock_vws/test_update_target.py::TestWidth - tests/mock_vws/test_update_target.py::TestInactiveProject - tests/mock_vws/test_requests_mock_usage.py + - tests/mock_vws/test_respx_mock_usage.py + - tests/mock_vws/test_target_validators.py - tests/mock_vws/test_flask_app_usage.py - tests/mock_vws/test_vumark_generation_api.py - tests/mock_vws/test_docker.py - README.rst - docs/source/basic-example.rst + - docs/source/httpx-example.rst + - ci/ steps: - uses: actions/checkout@v6 diff --git a/ci/test_custom_linters.py b/ci/test_custom_linters.py index c02fcacc6..991aa4165 100644 --- a/ci/test_custom_linters.py +++ b/ci/test_custom_linters.py @@ -1,15 +1,11 @@ """Custom lint tests.""" from pathlib import Path -from typing import TYPE_CHECKING import pytest import yaml from beartype import beartype -if TYPE_CHECKING: - from collections.abc import Iterable - @beartype def _ci_patterns(*, repository_root: Path) -> set[str]: @@ -23,32 +19,49 @@ def _ci_patterns(*, repository_root: Path) -> set[str]: return ci_patterns +class _CollectPlugin: + """Pytest plugin that records collected node IDs.""" + + def __init__(self) -> None: + """Initialize an empty set of collected node IDs.""" + self.nodeids: set[str] = set() + + def pytest_collection_modifyitems( + self, + items: list[pytest.Item], + ) -> None: + """Record the node IDs of all collected items.""" + self.nodeids.update(item.nodeid for item in items) + + @beartype -def _tests_from_pattern( - *, - ci_pattern: str, - capsys: pytest.CaptureFixture[str], -) -> set[str]: - """From a CI pattern, get all tests ``pytest`` would collect.""" - # Clear the captured output. - capsys.readouterr() - tests: Iterable[str] = set() +def _tests_from_pattern(*, ci_pattern: str) -> set[str]: + """From a CI pattern, get all tests ``pytest`` would collect. + + Uses a collection-hook plugin instead of parsing stdout: an in-process + ``pytest.main()`` installs its own output capture, so reading from + ``capsys`` would see an empty string and the test would pass vacuously. + """ + plugin = _CollectPlugin() pytest.main( args=[ - "-q", "--collect-only", - # If there are any warnings, these obscure the output. + # Disable pytest-retry to avoid: + # ``` + # ValueError: no option named 'filtered_exceptions' + # ``` + "-p", + "no:pytest-retry", + # Disable warnings to avoid many instances of: + # ``` + # Unknown config option: retry_delay + # ``` "--disable-warnings", ci_pattern, ], + plugins=[plugin], ) - data = capsys.readouterr().out - for line in data.splitlines(): - # We filter empty lines and lines which look like - # "9 tests collected in 0.01s". - if line and "collected in" not in line: - tests = {*tests, line} - return set(tests) + return plugin.nodeids def test_ci_patterns_valid(request: pytest.FixtureRequest) -> None: @@ -60,31 +73,13 @@ def test_ci_patterns_valid(request: pytest.FixtureRequest) -> None: ci_patterns = _ci_patterns(repository_root=request.config.rootpath) for ci_pattern in ci_patterns: - collect_only_result = pytest.main( - args=[ - "--collect-only", - ci_pattern, - # Disable pytest-retry to avoid: - # ``` - # ValueError: no option named 'filtered_exceptions' - # ```` - "-p", - "no:pytest-retry", - # Disable warnings to avoid many instances of: - # ``` - # Unknown config option: retry_delay - # ``` - "--disable-warnings", - ], - ) - + tests = _tests_from_pattern(ci_pattern=ci_pattern) message = f'"{ci_pattern}" does not match any tests.' - assert collect_only_result == 0, message + assert tests, message def test_tests_collected_once( *, - capsys: pytest.CaptureFixture[str], request: pytest.FixtureRequest, ) -> None: """Each test in the test suite is collected exactly once. @@ -95,7 +90,7 @@ def test_tests_collected_once( tests_to_patterns: dict[str, set[str]] = {} for pattern in ci_patterns: - tests = _tests_from_pattern(ci_pattern=pattern, capsys=capsys) + tests = _tests_from_pattern(ci_pattern=pattern) for test in tests: if test in tests_to_patterns: tests_to_patterns[test].add(pattern) @@ -110,6 +105,6 @@ def test_tests_collected_once( ) assert len(patterns) == 1, message - all_tests = _tests_from_pattern(ci_pattern=".", capsys=capsys) + all_tests = _tests_from_pattern(ci_pattern=".") assert tests_to_patterns.keys() - all_tests == set() assert all_tests - tests_to_patterns.keys() == set() From 65a6322492aa4ccf1a3c7f8714b64b24d403695f Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Sat, 25 Apr 2026 07:32:20 +0100 Subject: [PATCH 2/4] Speed up custom linter tests by collecting once Previously ran pytest.main() per pattern (~120 invocations, 89s). Now do a single in-process collection and check each pattern against the cached node IDs in pure Python via a small boundary-aware prefix matcher. Runtime drops from 89s to under 1s. --- ci/test_custom_linters.py | 51 +++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/ci/test_custom_linters.py b/ci/test_custom_linters.py index 991aa4165..bc05a3c00 100644 --- a/ci/test_custom_linters.py +++ b/ci/test_custom_linters.py @@ -34,9 +34,9 @@ def pytest_collection_modifyitems( self.nodeids.update(item.nodeid for item in items) -@beartype -def _tests_from_pattern(*, ci_pattern: str) -> set[str]: - """From a CI pattern, get all tests ``pytest`` would collect. +@pytest.fixture(scope="module") +def all_tests() -> frozenset[str]: + """Collect every test node ID in the suite, exactly once. Uses a collection-hook plugin instead of parsing stdout: an in-process ``pytest.main()`` installs its own output capture, so reading from @@ -57,14 +57,36 @@ def _tests_from_pattern(*, ci_pattern: str) -> set[str]: # Unknown config option: retry_delay # ``` "--disable-warnings", - ci_pattern, + ".", ], plugins=[plugin], ) - return plugin.nodeids + return frozenset(plugin.nodeids) + +@beartype +def _matches(*, nodeid: str, ci_pattern: str) -> bool: + """Whether ``pytest `` would have collected ``nodeid``. + + The patterns in the CI matrix are all of the form ``path[/]`` or + ``path::Class[::method]``. A node ID matches if it equals the pattern, + is a directory child of a pattern ending with ``/``, or extends the + pattern at a ``::`` (sub-item), ``/`` (path), or ``[`` (parametrize) + boundary. + """ + if nodeid == ci_pattern: + return True + if not nodeid.startswith(ci_pattern): + return False + if ci_pattern.endswith("/"): + return True + return nodeid[len(ci_pattern)] in {":", "/", "["} -def test_ci_patterns_valid(request: pytest.FixtureRequest) -> None: + +def test_ci_patterns_valid( + request: pytest.FixtureRequest, + all_tests: frozenset[str], +) -> None: """ All of the CI patterns in the CI configuration match at least one test in @@ -73,14 +95,17 @@ def test_ci_patterns_valid(request: pytest.FixtureRequest) -> None: ci_patterns = _ci_patterns(repository_root=request.config.rootpath) for ci_pattern in ci_patterns: - tests = _tests_from_pattern(ci_pattern=ci_pattern) + matched = { + n for n in all_tests if _matches(nodeid=n, ci_pattern=ci_pattern) + } message = f'"{ci_pattern}" does not match any tests.' - assert tests, message + assert matched, message def test_tests_collected_once( *, request: pytest.FixtureRequest, + all_tests: frozenset[str], ) -> None: """Each test in the test suite is collected exactly once. @@ -90,12 +115,9 @@ def test_tests_collected_once( tests_to_patterns: dict[str, set[str]] = {} for pattern in ci_patterns: - tests = _tests_from_pattern(ci_pattern=pattern) - for test in tests: - if test in tests_to_patterns: - tests_to_patterns[test].add(pattern) - else: - tests_to_patterns[test] = {pattern} + for test in all_tests: + if _matches(nodeid=test, ci_pattern=pattern): + tests_to_patterns.setdefault(test, set()).add(pattern) for test_name, patterns in tests_to_patterns.items(): message = ( @@ -105,6 +127,5 @@ def test_tests_collected_once( ) assert len(patterns) == 1, message - all_tests = _tests_from_pattern(ci_pattern=".") assert tests_to_patterns.keys() - all_tests == set() assert all_tests - tests_to_patterns.keys() == set() From fa7f56d13162075b01a718698a97a4d10d406be7 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Sat, 25 Apr 2026 08:31:29 +0100 Subject: [PATCH 3/4] Assert pytest collection succeeded in custom linter fixture Previously the return code of pytest.main() was discarded, so a collection error (import failure, syntax error) would silently leave the fixture with whatever items were captured before the crash. Assert the exit code is OK so collection errors fail the meta-tests loudly. --- ci/test_custom_linters.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/test_custom_linters.py b/ci/test_custom_linters.py index bc05a3c00..d893b18b6 100644 --- a/ci/test_custom_linters.py +++ b/ci/test_custom_linters.py @@ -43,7 +43,7 @@ def all_tests() -> frozenset[str]: ``capsys`` would see an empty string and the test would pass vacuously. """ plugin = _CollectPlugin() - pytest.main( + exit_code = pytest.main( args=[ "--collect-only", # Disable pytest-retry to avoid: @@ -61,6 +61,12 @@ def all_tests() -> frozenset[str]: ], plugins=[plugin], ) + # Fail loudly on collection errors (import failures, syntax errors, etc.) + # rather than silently using whatever items were captured before the + # crash. + assert exit_code == pytest.ExitCode.OK, ( + f"Collection failed with exit code {exit_code}." + ) return frozenset(plugin.nodeids) From d907796d16f5b1925d6984b3c6a1dc37097af4d2 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Sat, 25 Apr 2026 08:52:12 +0100 Subject: [PATCH 4/4] Revert linter speedup; use authoritative pytest selection The prefix-matching shortcut was a hand-rolled approximation of pytest's selector. Go back to invoking pytest.main() per pattern so selection matches CI exactly, even though it's slower (~89s). Keep the exit-code assertion so collection errors fail loudly. --- ci/test_custom_linters.py | 50 ++++++++++----------------------------- 1 file changed, 13 insertions(+), 37 deletions(-) diff --git a/ci/test_custom_linters.py b/ci/test_custom_linters.py index d893b18b6..0817a0896 100644 --- a/ci/test_custom_linters.py +++ b/ci/test_custom_linters.py @@ -34,9 +34,9 @@ def pytest_collection_modifyitems( self.nodeids.update(item.nodeid for item in items) -@pytest.fixture(scope="module") -def all_tests() -> frozenset[str]: - """Collect every test node ID in the suite, exactly once. +@beartype +def _tests_from_pattern(*, ci_pattern: str) -> set[str]: + """From a CI pattern, get all tests ``pytest`` would collect. Uses a collection-hook plugin instead of parsing stdout: an in-process ``pytest.main()`` installs its own output capture, so reading from @@ -57,7 +57,7 @@ def all_tests() -> frozenset[str]: # Unknown config option: retry_delay # ``` "--disable-warnings", - ".", + ci_pattern, ], plugins=[plugin], ) @@ -65,34 +65,12 @@ def all_tests() -> frozenset[str]: # rather than silently using whatever items were captured before the # crash. assert exit_code == pytest.ExitCode.OK, ( - f"Collection failed with exit code {exit_code}." + f"Collection for {ci_pattern!r} failed with exit code {exit_code}." ) - return frozenset(plugin.nodeids) - + return plugin.nodeids -@beartype -def _matches(*, nodeid: str, ci_pattern: str) -> bool: - """Whether ``pytest `` would have collected ``nodeid``. - - The patterns in the CI matrix are all of the form ``path[/]`` or - ``path::Class[::method]``. A node ID matches if it equals the pattern, - is a directory child of a pattern ending with ``/``, or extends the - pattern at a ``::`` (sub-item), ``/`` (path), or ``[`` (parametrize) - boundary. - """ - if nodeid == ci_pattern: - return True - if not nodeid.startswith(ci_pattern): - return False - if ci_pattern.endswith("/"): - return True - return nodeid[len(ci_pattern)] in {":", "/", "["} - -def test_ci_patterns_valid( - request: pytest.FixtureRequest, - all_tests: frozenset[str], -) -> None: +def test_ci_patterns_valid(request: pytest.FixtureRequest) -> None: """ All of the CI patterns in the CI configuration match at least one test in @@ -101,17 +79,14 @@ def test_ci_patterns_valid( ci_patterns = _ci_patterns(repository_root=request.config.rootpath) for ci_pattern in ci_patterns: - matched = { - n for n in all_tests if _matches(nodeid=n, ci_pattern=ci_pattern) - } + tests = _tests_from_pattern(ci_pattern=ci_pattern) message = f'"{ci_pattern}" does not match any tests.' - assert matched, message + assert tests, message def test_tests_collected_once( *, request: pytest.FixtureRequest, - all_tests: frozenset[str], ) -> None: """Each test in the test suite is collected exactly once. @@ -121,9 +96,9 @@ def test_tests_collected_once( tests_to_patterns: dict[str, set[str]] = {} for pattern in ci_patterns: - for test in all_tests: - if _matches(nodeid=test, ci_pattern=pattern): - tests_to_patterns.setdefault(test, set()).add(pattern) + tests = _tests_from_pattern(ci_pattern=pattern) + for test in tests: + tests_to_patterns.setdefault(test, set()).add(pattern) for test_name, patterns in tests_to_patterns.items(): message = ( @@ -133,5 +108,6 @@ def test_tests_collected_once( ) assert len(patterns) == 1, message + all_tests = _tests_from_pattern(ci_pattern=".") assert tests_to_patterns.keys() - all_tests == set() assert all_tests - tests_to_patterns.keys() == set()