From 884512271031142cfa9f3f690ccb099218036d6c Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 13:34:58 +0200 Subject: [PATCH] Add experimental _pytest.scenario API: nested configs + in-memory collection pytest's own testsuite is heavily integration-based: most tests write files via pytester and run a full session, then glob-match terminal output. This adds an experimental internal API to test pytest with pytest hermetically instead: * ConfigSpec/configured() build a parsed+configured nested Config from declarative data through the real parse phases -- no rootdir discovery, config files, conftests, plugin autoload, or env vars. Config.parse() is split into behavior-preserving phase methods so the programmatic path shares code with command line parsing, and ArgsSource.SPEC marks verbatim args. * fake_module()/collect_tests()/run_tests() collect test items from in-memory objects (functions, classes, module namespaces) through the standard pytest_collection flow via a ScenarioModule whose _getobj serves the preset object; nodeids derive from synthetic rootdir-relative paths that are never touched on disk. * RunRecord/ItemRunRecord provide typed structured results derived from real report objects via pytest_report_teststatus, with a pytester-compatible assert_outcomes(). The default scenario plugin set deliberately excludes capture, terminal, assertion, cacheprovider and the process-global-state plugins; capture nesting is the documented follow-up. Supporting core fixes: * Class.from_parent() no longer silently discards a passed obj. * get_config() accepts an explicit invocation dir=. * debugging/unittest guard cross-plugin option reads (trace/usepdb) so they work when the debugging plugin is not registered. Validated by 33 self-tests (0.3s, hermeticity asserted) and by converting seven existing tests in testing/python/ to the new API. Co-Authored-By: Claude Fable 5 --- changelog/14809.feature.rst | 13 + src/_pytest/config/__init__.py | 156 +++++++---- src/_pytest/debugging.py | 2 +- src/_pytest/python.py | 16 +- src/_pytest/scenario/__init__.py | 147 ++++++++++ src/_pytest/scenario/collection.py | 166 ++++++++++++ src/_pytest/scenario/config.py | 161 +++++++++++ src/_pytest/scenario/results.py | 216 +++++++++++++++ src/_pytest/unittest.py | 2 +- testing/python/collect.py | 109 ++++---- testing/python/fixtures.py | 61 ++--- testing/python/metafunc.py | 39 ++- testing/test_scenario.py | 412 +++++++++++++++++++++++++++++ testing/test_warnings.py | 7 +- 14 files changed, 1338 insertions(+), 169 deletions(-) create mode 100644 changelog/14809.feature.rst create mode 100644 src/_pytest/scenario/__init__.py create mode 100644 src/_pytest/scenario/collection.py create mode 100644 src/_pytest/scenario/config.py create mode 100644 src/_pytest/scenario/results.py create mode 100644 testing/test_scenario.py diff --git a/changelog/14809.feature.rst b/changelog/14809.feature.rst new file mode 100644 index 00000000000..223005789b8 --- /dev/null +++ b/changelog/14809.feature.rst @@ -0,0 +1,13 @@ +Added the experimental internal ``_pytest.scenario`` API for testing pytest with pytest: +build a hermetic nested configuration from declarative data (``ConfigSpec``/``configured()``), +collect test items from in-memory python objects instead of files on disk +(``fake_module()``, ``collect_tests()``), run them through the standard runtest protocol, +and assert on typed report objects (``RunRecord``) instead of glob-matching terminal output. + +In support of this: + +* ``Config.parse()`` was split into behavior-preserving phase methods, so programmatic + config construction shares real code with command line parsing. +* ``Config.ArgsSource.SPEC`` marks args taken verbatim from a programmatic specification. +* ``pytest.Class.from_parent()`` no longer silently discards a passed ``obj``. +* ``_pytest.config.get_config()`` accepts an explicit invocation ``dir=``. diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 688cc8a9054..34f565c62bf 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -356,13 +356,14 @@ def get_config( plugins: Sequence[str | _PluggyPlugin] | None = None, *, prog: str | None = None, + dir: pathlib.Path | None = None, ) -> Config: # Subsequent calls to main will create a fresh instance. pluginmanager = PytestPluginManager() invocation_params = Config.InvocationParams( args=args or (), plugins=plugins, - dir=pathlib.Path.cwd(), + dir=dir if dir is not None else pathlib.Path.cwd(), ) config = Config(pluginmanager, invocation_params=invocation_params, prog=prog) @@ -881,7 +882,7 @@ def consider_module(self, mod: types.ModuleType) -> None: self._import_plugin_specs(getattr(mod, "pytest_plugins", [])) def _import_plugin_specs( - self, spec: None | types.ModuleType | str | Sequence[str] + self, spec: types.ModuleType | str | Sequence[str] | None ) -> None: plugins = _get_plugin_specs_as_list(spec) for import_spec in plugins: @@ -931,7 +932,7 @@ def import_plugin(self, modname: str, consider_entry_points: bool = False) -> No def _get_plugin_specs_as_list( - specs: None | types.ModuleType | str | Sequence[str], + specs: types.ModuleType | str | Sequence[str] | None, ) -> list[str]: """Parse a plugins specification into a list of plugin names.""" # None means empty. @@ -1100,10 +1101,17 @@ class ArgsSource(enum.Enum): INCOVATION_DIR = INVOCATION_DIR # backwards compatibility alias #: 'testpaths' configuration value. TESTPATHS = enum.auto() + #: Programmatic specification; args are taken verbatim, without + #: filesystem-based fallbacks (experimental). + SPEC = enum.auto() # Set by cacheprovider plugin. cache: Cache + #: The parsed command line namespace, including only options known at + #: the time of parsing. Set during :meth:`parse`. + known_args_namespace: argparse.Namespace + def __init__( self, pluginmanager: PytestPluginManager, @@ -1546,34 +1554,28 @@ def _get_unknown_ini_keys(self) -> set[str]: known_keys = self._parser._inidict.keys() | self._parser._ini_aliases.keys() return self._inicfg.keys() - known_keys - def parse(self, args: list[str], addopts: bool = True) -> None: - # Parse given cmdline arguments into this config object. - assert self.args == [], ( - "can only parse cmdline args at most once per Config object" - ) - - self.hook.pytest_addhooks.call_historic( - kwargs=dict(pluginmanager=self.pluginmanager) - ) + def _preparse_addopts(self, args: list[str]) -> None: + """Prepend options from the PYTEST_ADDOPTS environment variable (in place).""" + env_addopts = os.environ.get("PYTEST_ADDOPTS", "") + if len(env_addopts): + args[:] = ( + self._validate_args(shlex.split(env_addopts), "via PYTEST_ADDOPTS") + + args + ) - if addopts: - env_addopts = os.environ.get("PYTEST_ADDOPTS", "") - if len(env_addopts): - args[:] = ( - self._validate_args(shlex.split(env_addopts), "via PYTEST_ADDOPTS") - + args - ) + def _apply_rootdir( + self, + *, + rootpath: pathlib.Path, + inipath: pathlib.Path | None, + inicfg: ConfigDict, + ignored_config_files: Sequence[str], + ) -> None: + """Apply an already-determined rootdir/inifile setup to this config. - # At this point, self.option contains only defaults from the _processopt - # callback. - ns = self._parser.parse_known_args(args, namespace=copy.copy(self.option)) - rootpath, inipath, inicfg, ignored_config_files = determine_setup( - inifile=ns.inifilename, - override_ini=ns.override_ini, - args=ns.file_or_dir, - rootdir_cmd_arg=ns.rootdir or None, - invocation_dir=self.invocation_params.dir, - ) + Normally fed by :func:`determine_setup`, but usable with explicitly + constructed values as well (experimental). + """ self._rootpath = rootpath self._inipath = inipath self._ignored_config_files = ignored_config_files @@ -1581,6 +1583,8 @@ def parse(self, args: list[str], addopts: bool = True) -> None: self._parser.extra_info["rootdir"] = str(self.rootpath) self._parser.extra_info["inifile"] = str(self.inipath) + def _register_core_ini_options(self) -> None: + """Register the ini options which parse() itself consumes.""" self._parser.addini("addopts", "Extra command line options", "args") self._parser.addini("minversion", "Minimally required pytest version") self._parser.addini( @@ -1593,20 +1597,8 @@ def parse(self, args: list[str], addopts: bool = True) -> None: default=[], ) - if addopts: - args[:] = ( - self._validate_args(self.getini("addopts"), "via addopts config") + args - ) - - self.known_args_namespace = self._parser.parse_known_args( - args, namespace=copy.copy(self.option) - ) - if addopts: - # addopts may have added overrides (especially via OverrideIniAction). - # The thing can be endlessly circular but we only do one level (#14442). - if overrides := parse_override_ini(self.known_args_namespace.override_ini): - self._inicfg.update(overrides) - self._inicache.clear() + def _load_plugins_phase(self, args: list[str]) -> None: + """Load plugins from args, entry points and environment; reparse known args.""" self._checkversion() self._consider_importhook() self._configure_python_path() @@ -1632,6 +1624,8 @@ def parse(self, args: list[str], addopts: bool = True) -> None: self._validate_plugins() self._warn_about_skipped_plugins() + def _load_initial_conftests_phase(self, args: list[str]) -> None: + """Default confcutdir and fire the pytest_load_initial_conftests hook.""" if self.known_args_namespace.confcutdir is None: if self.inipath is not None: confcutdir = str(self.inipath.parent) @@ -1653,19 +1647,83 @@ def parse(self, args: list[str], addopts: bool = True) -> None: else: raise + def _finalize_parse(self, args: list[str], *, decide_args: bool = True) -> None: + """Fully parse args into self.option and decide the initial args. + + With ``decide_args=False`` (experimental), the positional args are + taken verbatim without the testpaths/invocation-dir fallbacks and + ``args_source`` is set to :attr:`ArgsSource.SPEC`. + """ + if not hasattr(self, "known_args_namespace"): + self.known_args_namespace = self._parser.parse_known_args( + args, namespace=copy.copy(self.option) + ) try: self._parser.parse(args, namespace=self.option) except PrintHelp: return - self.args, self.args_source = self._decide_args( - args=getattr(self.option, FILE_OR_DIR), - pyargs=self.option.pyargs, - testpaths=self.getini("testpaths"), + if decide_args: + self.args, self.args_source = self._decide_args( + args=getattr(self.option, FILE_OR_DIR), + pyargs=self.option.pyargs, + testpaths=self.getini("testpaths"), + invocation_dir=self.invocation_params.dir, + rootpath=self.rootpath, + warn=True, + ) + else: + self.args = list(getattr(self.option, FILE_OR_DIR)) + self.args_source = Config.ArgsSource.SPEC + + def parse(self, args: list[str], addopts: bool = True) -> None: + # Parse given cmdline arguments into this config object. + assert self.args == [], ( + "can only parse cmdline args at most once per Config object" + ) + + self.hook.pytest_addhooks.call_historic( + kwargs=dict(pluginmanager=self.pluginmanager) + ) + + if addopts: + self._preparse_addopts(args) + + # At this point, self.option contains only defaults from the _processopt + # callback. + ns = self._parser.parse_known_args(args, namespace=copy.copy(self.option)) + rootpath, inipath, inicfg, ignored_config_files = determine_setup( + inifile=ns.inifilename, + override_ini=ns.override_ini, + args=ns.file_or_dir, + rootdir_cmd_arg=ns.rootdir or None, invocation_dir=self.invocation_params.dir, - rootpath=self.rootpath, - warn=True, ) + self._apply_rootdir( + rootpath=rootpath, + inipath=inipath, + inicfg=inicfg, + ignored_config_files=ignored_config_files, + ) + self._register_core_ini_options() + + if addopts: + args[:] = ( + self._validate_args(self.getini("addopts"), "via addopts config") + args + ) + + self.known_args_namespace = self._parser.parse_known_args( + args, namespace=copy.copy(self.option) + ) + if addopts: + # addopts may have added overrides (especially via OverrideIniAction). + # The thing can be endlessly circular but we only do one level (#14442). + if overrides := parse_override_ini(self.known_args_namespace.override_ini): + self._inicfg.update(overrides) + self._inicache.clear() + self._load_plugins_phase(args) + self._load_initial_conftests_phase(args) + self._finalize_parse(args) def issue_config_time_warning(self, warning: Warning, stacklevel: int) -> None: """Issue and handle a warning during the "configure" stage. diff --git a/src/_pytest/debugging.py b/src/_pytest/debugging.py index b256f83c8bf..399ed8987a3 100644 --- a/src/_pytest/debugging.py +++ b/src/_pytest/debugging.py @@ -330,7 +330,7 @@ def wrapper(*args, **kwargs) -> None: def maybe_wrap_pytest_function_for_tracing(pyfuncitem) -> None: """Wrap the given pytestfunct item for tracing support if --trace was given in the command line.""" - if pyfuncitem.config.getvalue("trace"): + if pyfuncitem.config.getoption("trace", False): wrap_pytest_function_for_tracing(pyfuncitem) diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 4f168012c36..f1e0f5e8585 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -238,7 +238,7 @@ def pytest_pycollect_makemodule(module_path: Path, parent) -> Module: @hookimpl(trylast=True) def pytest_pycollect_makeitem( collector: Module | Class, name: str, obj: object -) -> None | nodes.Item | nodes.Collector | list[nodes.Item | nodes.Collector]: +) -> nodes.Item | nodes.Collector | list[nodes.Item | nodes.Collector] | None: assert isinstance(collector, Class | Module), type(collector) # Nothing was collected elsewhere, let's do it here. if safe_isclass(obj): @@ -765,10 +765,22 @@ def _get_first_non_fixture_func(obj: object, names: Iterable[str]) -> object | N class Class(PyCollector): """Collector for test methods (and nested classes) in a Python class.""" + #: Explicitly provided class object, taking precedence over looking up + #: ``name`` on the parent's object (experimental). + _given_obj: type | None = None + @classmethod def from_parent(cls, parent, *, name, obj=None, **kw) -> Self: # type: ignore[override] """The public constructor.""" - return super().from_parent(name=name, parent=parent, **kw) + node: Self = super().from_parent(name=name, parent=parent, **kw) + if obj is not None: + node._given_obj = obj + return node + + def _getobj(self): + if self._given_obj is not None: + return self._given_obj + return super()._getobj() def newinstance(self): return self.obj() diff --git a/src/_pytest/scenario/__init__.py b/src/_pytest/scenario/__init__.py new file mode 100644 index 00000000000..01b4eddc31f --- /dev/null +++ b/src/_pytest/scenario/__init__.py @@ -0,0 +1,147 @@ +"""Nested configs and in-memory collection for pytest-under-pytest testing. + +EXPERIMENTAL: internal API, no backwards-compatibility guarantees. + +This package lets a test build a hermetic nested pytest configuration from +declarative data (:class:`ConfigSpec`), collect test items from in-memory +python objects instead of files on disk, run them through the standard +runtest protocol, and assert on typed report objects (:class:`RunRecord`) +instead of glob-matching rendered terminal output. + +Known limitations (by design, for now): + +* Scenario configs never load conftest files; pass plugin objects via + ``ConfigSpec.extra_plugins`` instead. +* The ``capture`` and ``terminal`` plugins are not loaded by default: + ``capsys``/``capfd`` are unavailable inside scenario tests and no + terminal output exists. Capture nesting is a planned follow-up. +* Assertion rewriting is not applied to scenario sources; sources defined + in the host test suite's own files are already rewritten by the host. +* Sources without real code objects (``exec``'d, lambdas) degrade + ``reportinfo``/traceback quality. +* Process-global warning filters active around the scenario (e.g. the + host suite's ``filterwarnings = error``) are inherited; a scenario's + own ``inicfg={"filterwarnings": [...]}`` takes precedence over them. +""" + +from __future__ import annotations + +from collections.abc import Iterator +import contextlib +import pathlib + +from _pytest.config import Config +from _pytest.main import Session +from _pytest.nodes import Item +from _pytest.scenario.collection import collect_objects +from _pytest.scenario.collection import fake_module +from _pytest.scenario.collection import running_session +from _pytest.scenario.collection import ScenarioModule +from _pytest.scenario.collection import Source +from _pytest.scenario.config import ConfigSpec +from _pytest.scenario.config import configured +from _pytest.scenario.config import DEFAULT_SCENARIO_PLUGINS +from _pytest.scenario.results import ItemRunRecord +from _pytest.scenario.results import run_items +from _pytest.scenario.results import RunRecord + + +__all__ = [ + "DEFAULT_SCENARIO_PLUGINS", + "ConfigSpec", + "ItemRunRecord", + "RunRecord", + "Scenario", + "ScenarioModule", + "Source", + "collect_objects", + "collect_tests", + "configured", + "fake_module", + "run_items", + "run_tests", + "running_session", + "scenario", +] + + +def _resolve_spec(spec: ConfigSpec | None, rootpath: pathlib.Path | None) -> ConfigSpec: + if spec is None: + spec = ConfigSpec(rootpath=rootpath) + elif rootpath is not None and spec.rootpath is None: + spec = spec.replace(rootpath=rootpath) + return spec + + +def run_tests( + *sources: Source, + rootpath: pathlib.Path | None = None, + spec: ConfigSpec | None = None, + name: str = "scenario_module", +) -> RunRecord: + """Collect and run the given in-memory sources in a nested config; + return the structured results.""" + with scenario(*sources, rootpath=rootpath, spec=spec, name=name) as s: + s.collect() + return s.run() + + +def collect_tests( + *sources: Source, + rootpath: pathlib.Path | None = None, + spec: ConfigSpec | None = None, + name: str = "scenario_module", +) -> list[Item]: + """Collect (only) the given in-memory sources; return the items. + + The nested config and session are torn down before returning; the + items remain usable for structural assertions (names, nodeids, marks). + """ + with scenario(*sources, rootpath=rootpath, spec=spec, name=name) as s: + return s.collect() + + +class Scenario: + """An entered scenario: a configured nested config plus a running + session, over which sources can be collected and run stepwise.""" + + def __init__( + self, config: Config, session: Session, sources: tuple[Source, ...], name: str + ) -> None: + self.config = config + self.session = session + self._sources = sources + self._name = name + self._collected = False + + def collect(self, *sources: Source) -> list[Item]: + """Collect the scenario's sources (plus any given extra ones).""" + self._collected = True + return collect_objects(self.session, *self._sources, *sources, name=self._name) + + def run(self, items: list[Item] | None = None) -> RunRecord: + """Run the collected items (collecting first if needed).""" + if not self._collected: + self.collect() + return run_items(self.session, items) + + +@contextlib.contextmanager +def scenario( + *sources: Source, + rootpath: pathlib.Path | None = None, + spec: ConfigSpec | None = None, + name: str = "scenario_module", +) -> Iterator[Scenario]: + """Enter a nested config and running session for the given sources. + + Usage:: + + with scenario(test_fn, SomeTestClass, rootpath=tmp_path) as s: + items = s.collect() + record = s.run() + record.assert_outcomes(passed=2) + """ + resolved = _resolve_spec(spec, rootpath) + with configured(resolved) as config, running_session(config) as session: + yield Scenario(config, session, sources, name) diff --git a/src/_pytest/scenario/collection.py b/src/_pytest/scenario/collection.py new file mode 100644 index 00000000000..0edb4af890e --- /dev/null +++ b/src/_pytest/scenario/collection.py @@ -0,0 +1,166 @@ +"""In-memory collection for scenarios (EXPERIMENTAL).""" + +from __future__ import annotations + +from collections.abc import Callable +from collections.abc import Iterator +import contextlib +import types + +from _pytest.config import Config +from _pytest.config import ExitCode +from _pytest.main import Session +from _pytest.nodes import Collector +from _pytest.nodes import Item +from _pytest.python import Module +from _pytest.reports import CollectReport +from _pytest.scenario.results import ensure_recorder + + +#: A test source: a module-like object collected as a virtual module, or a +#: loose class/callable wrapped into a synthesized one. +Source = types.ModuleType | type | Callable[..., object] + + +class ScenarioModule(Module): + """A Module collector backed by an in-memory python object. + + The synthetic ``path`` is rootdir-relative (giving well-formed nodeids) + but never touched on disk; the standard import chokepoint is bypassed + by serving the preset object from ``_getobj``. + """ + + _scenario_obj: types.ModuleType + + @classmethod + def from_parent( # type: ignore[override] + cls, + parent: Session, + *, + obj: types.ModuleType, + name: str | None = None, + ) -> ScenarioModule: + """The public constructor.""" + if name is None: + name = obj.__name__ + path = parent.config.rootpath / f"{name}.py" + node: ScenarioModule = super().from_parent(parent, path=path) + node._scenario_obj = obj + return node + + def _getobj(self) -> types.ModuleType: + return self._scenario_obj + + +class ScenarioCollection: + """Plugin serving a preset list of collectors as the session's + collection tree, instead of walking filesystem paths.""" + + name = "scenario-collection" + + def __init__(self, session: Session) -> None: + self.session = session + self.collectors: list[Collector] = [] + + def pytest_make_collect_report(self, collector: Collector) -> CollectReport | None: + if collector is self.session: + return CollectReport( + collector.nodeid, "passed", None, list(self.collectors) + ) + return None + + +def scenario_collection(session: Session) -> ScenarioCollection: + """Get the preset-collection plugin for the session, installing it if needed.""" + plugin: ScenarioCollection | None = session.config.pluginmanager.get_plugin( + ScenarioCollection.name + ) + if plugin is None: + plugin = ScenarioCollection(session) + session.config.pluginmanager.register(plugin, ScenarioCollection.name) + assert plugin.session is session + return plugin + + +@contextlib.contextmanager +def running_session(config: Config) -> Iterator[Session]: + """A started :class:`Session` for a configured scenario config. + + ``pytest_sessionstart`` runs on enter (installing ``SetupState`` and the + ``FixtureManager``); ``pytest_sessionfinish`` runs on exit. + """ + session = Session.from_config(config) + ensure_recorder(config) + scenario_collection(session) + config.hook.pytest_sessionstart(session=session) + exitstatus: int | ExitCode = ExitCode.OK + try: + yield session + except BaseException: + exitstatus = ExitCode.INTERNAL_ERROR + raise + finally: + config.hook.pytest_sessionfinish(session=session, exitstatus=exitstatus) + + +def fake_module( + name: str, *members: object, **named_members: object +) -> types.ModuleType: + """Create an in-memory module with an explicit name, collecting the + given members together. + + Positional members are stored under their ``__name__``; keyword + members under the given keyword (e.g. ``pytestmark=...``). + """ + module = types.ModuleType(name) + for member in members: + member_name = getattr(member, "__name__", None) + if member_name is None: + raise ValueError( + f"member {member!r} has no __name__; pass it as a keyword instead" + ) + setattr(module, member_name, member) + for member_name, member in named_members.items(): + setattr(module, member_name, member) + return module + + +def collect_objects( + session: Session, + *sources: Source, + name: str = "scenario_module", +) -> list[Item]: + """Collect test items from in-memory objects through the standard + collection protocol. + + Module-like sources each become a :class:`ScenarioModule`; loose + classes and callables are wrapped into one synthesized module named + ``name``. The regular ``pytest_collection`` hook then runs, so + ``-k``/``-m`` deselection, ``pytest_collection_modifyitems`` and + ``pytest_collection_finish`` all apply. + """ + config = session.config + collection = scenario_collection(session) + loose: list[object] = [] + for source in sources: + if isinstance(source, types.ModuleType): + collection.collectors.append( + ScenarioModule.from_parent(session, obj=source) + ) + else: + loose.append(source) + if loose: + module = fake_module(name, *loose) + if not config.getini("collect_imported_tests"): + # python.py drops objects whose __module__ differs from the + # containing module; synthesized namespaces always differ. + raise ValueError( + "collect_imported_tests=False would silently drop loose " + "scenario sources; pass a real module object instead" + ) + collection.collectors.append( + ScenarioModule.from_parent(session, obj=module, name=name) + ) + + config.hook.pytest_collection(session=session) + return session.items diff --git a/src/_pytest/scenario/config.py b/src/_pytest/scenario/config.py new file mode 100644 index 00000000000..f11de5b4259 --- /dev/null +++ b/src/_pytest/scenario/config.py @@ -0,0 +1,161 @@ +"""Nested config construction for scenarios (EXPERIMENTAL).""" + +from __future__ import annotations + +from collections.abc import Iterator +from collections.abc import Mapping +import contextlib +import dataclasses +import pathlib +from typing import Final + +from _pytest.config import Config +from _pytest.config import essential_plugins +from _pytest.config import PytestPluginManager +from _pytest.config.findpaths import ConfigValue + + +#: Plugins loaded into a scenario config by default: the essential core +#: plus the plugins that give tests their usual semantics, deliberately +#: excluding everything that renders output, captures io, or installs +#: process-global state (terminal, capture, cacheprovider, assertion, +#: debugging, faulthandler, logging, threadexception, unraisableexception, ...). +DEFAULT_SCENARIO_PLUGINS: Final[tuple[str, ...]] = ( + *essential_plugins, # mark, main, runner, fixtures, helpconfig + "python", + "skipping", + "warnings", + "reports", + "unittest", + "monkeypatch", + "recwarn", + "tmpdir", +) + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class ConfigSpec: + """Declarative description of a nested pytest configuration. + + A spec is plain data: build it in a fixture, parametrize it, or derive + variants via :meth:`replace`/:meth:`with_plugins`. State is only + acquired when the spec is passed to :func:`configured`. + """ + + #: Root directory anchoring nodeids and synthetic module paths. + #: Must be an existing directory; it is never read from or written to. + rootpath: pathlib.Path | None = None + + #: Command line arguments, taken verbatim (never coerced to paths). + args: tuple[str, ...] = () + + #: Ini configuration values, authoritative (no config file is read). + #: Values are plain ``str``/``list[str]`` (ini mode) or preconstructed + #: :class:`ConfigValue` instances. + inicfg: Mapping[str, object] = dataclasses.field(default_factory=dict) + + #: Built-in plugin names to load. + plugins: tuple[str, ...] = DEFAULT_SCENARIO_PLUGINS + + #: Additional plugins: importable module names or plugin objects. + #: Plugin objects are the scenario replacement for conftest files. + extra_plugins: tuple[str | object, ...] = () + + #: Invocation directory; defaults to ``rootpath``, never the implicit cwd. + invocation_dir: pathlib.Path | None = None + + #: Not supported yet; scenario configs never load conftest files. + load_conftests: bool = False + + def replace(self, **kw: object) -> ConfigSpec: + return dataclasses.replace(self, **kw) # type: ignore[arg-type] + + def with_plugins(self, *plugins: str | object) -> ConfigSpec: + """Return a spec with the given plugins added. + + Built-in plugin names extend :attr:`plugins`; anything else is + appended to :attr:`extra_plugins`. + """ + from _pytest.config import builtin_plugins + + names = tuple(p for p in plugins if isinstance(p, str) and p in builtin_plugins) + extras = tuple( + p for p in plugins if not (isinstance(p, str) and p in builtin_plugins) + ) + return self.replace( + plugins=self.plugins + names, + extra_plugins=self.extra_plugins + extras, + ) + + def without_plugins(self, *names: str) -> ConfigSpec: + """Return a spec with the given built-in plugin names removed.""" + return self.replace(plugins=tuple(p for p in self.plugins if p not in names)) + + +@contextlib.contextmanager +def configured(spec: ConfigSpec) -> Iterator[Config]: + """Build a parsed *and* configured :class:`Config` from a spec. + + The config is constructed from the spec's explicit values through the + same parse phases a command line invocation uses, but without rootdir + discovery, config file reading, conftest loading, plugin autoloading, + or environment variable consultation. + + On exit, ``pytest_unconfigure`` and the config cleanup stack run. + """ + if spec.rootpath is None: + raise ValueError("ConfigSpec.rootpath is required to build a config") + if spec.load_conftests: + raise NotImplementedError( + "loading conftest files is not supported in scenario configs yet" + ) + if not spec.rootpath.is_dir(): + raise ValueError(f"ConfigSpec.rootpath is not a directory: {spec.rootpath}") + missing = [name for name in essential_plugins if name not in spec.plugins] + if missing: + raise ValueError( + f"ConfigSpec.plugins must include the essential plugins, missing: {missing}" + ) + + invocation_dir = ( + spec.invocation_dir if spec.invocation_dir is not None else spec.rootpath + ) + pluginmanager = PytestPluginManager() + config = Config( + pluginmanager, + invocation_params=Config.InvocationParams( + args=spec.args, + plugins=spec.extra_plugins or None, + dir=invocation_dir, + ), + ) + try: + for name in spec.plugins: + pluginmanager.import_plugin(name) + for plugin in spec.extra_plugins: + if isinstance(plugin, str): + pluginmanager.import_plugin(plugin) + else: + pluginmanager.register(plugin) + + config.hook.pytest_addhooks.call_historic( + kwargs=dict(pluginmanager=pluginmanager) + ) + inicfg = { + name: value + if isinstance(value, ConfigValue) + else ConfigValue(value, origin="file", mode="ini") + for name, value in spec.inicfg.items() + } + config._apply_rootdir( + rootpath=spec.rootpath, + inipath=None, + inicfg=inicfg, + ignored_config_files=(), + ) + config._register_core_ini_options() + config._finalize_parse(list(spec.args), decide_args=False) + config._do_configure() + yield config + finally: + config._ensure_unconfigure() diff --git a/src/_pytest/scenario/results.py b/src/_pytest/scenario/results.py new file mode 100644 index 00000000000..092945738e5 --- /dev/null +++ b/src/_pytest/scenario/results.py @@ -0,0 +1,216 @@ +"""Structured results for scenario runs (EXPERIMENTAL).""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Sequence +import dataclasses +import warnings as warnings_module + +from _pytest.config import Config +from _pytest.main import Session +from _pytest.nodes import Item +from _pytest.reports import CollectReport +from _pytest.reports import TestReport + + +class ScenarioRecorder: + """Plugin recording reports/warnings/deselections on a scenario config.""" + + name = "scenario-recorder" + + def __init__(self) -> None: + self.test_reports: list[TestReport] = [] + self.collect_reports: list[CollectReport] = [] + self.warnings: list[warnings_module.WarningMessage] = [] + self.deselected: int = 0 + + def pytest_runtest_logreport(self, report: TestReport) -> None: + self.test_reports.append(report) + + def pytest_collectreport(self, report: CollectReport) -> None: + self.collect_reports.append(report) + + def pytest_warning_recorded( + self, warning_message: warnings_module.WarningMessage + ) -> None: + self.warnings.append(warning_message) + + def pytest_deselected(self, items: Sequence[Item]) -> None: + self.deselected += len(items) + + +def ensure_recorder(config: Config) -> ScenarioRecorder: + """Get the recorder registered on the config, installing it if needed.""" + recorder: ScenarioRecorder | None = config.pluginmanager.get_plugin( + ScenarioRecorder.name + ) + if recorder is None: + recorder = ScenarioRecorder() + config.pluginmanager.register(recorder, ScenarioRecorder.name) + return recorder + + +@dataclasses.dataclass(frozen=True) +class ItemRunRecord: + """The reports of one test item's run, by phase.""" + + nodeid: str + setup: TestReport | None + call: TestReport | None + teardown: TestReport | None + #: Aggregate category as reported by ``pytest_report_teststatus`` + #: ("passed", "failed", "skipped", "error", "xfailed", "xpassed", ...). + outcome: str + + @property + def reports(self) -> list[TestReport]: + return [r for r in (self.setup, self.call, self.teardown) if r is not None] + + @property + def passed(self) -> bool: + return self.outcome == "passed" + + @property + def failed(self) -> bool: + return self.outcome in ("failed", "error") + + @property + def skipped(self) -> bool: + return self.outcome == "skipped" + + +@dataclasses.dataclass(frozen=True) +class RunRecord: + """Typed, structured results of a scenario run. + + Everything is derived from real report objects — nothing is scraped + from rendered output. + """ + + reports: list[TestReport] + collect_reports: list[CollectReport] + warnings: list[warnings_module.WarningMessage] + deselected: int + by_test: dict[str, ItemRunRecord] + _by_name: dict[str, ItemRunRecord] + _counts: dict[str, int] + + @classmethod + def from_recorder(cls, recorder: ScenarioRecorder, *, config: Config) -> RunRecord: + counts: Counter[str] = Counter() + phases: dict[str, dict[str, TestReport]] = {} + categories: dict[str, list[str]] = {} + for report in recorder.test_reports: + phases.setdefault(report.nodeid, {})[report.when] = report + status = config.hook.pytest_report_teststatus(report=report, config=config) + if status is not None: + category = status[0] + else: + # The catch-all status impl lives in the terminal plugin, + # which scenario configs exclude; mirror its categorization. + category = report.outcome + if category: + counts[category] += 1 + categories.setdefault(report.nodeid, []).append(category) + for collect_report in recorder.collect_reports: + if collect_report.failed: + counts["error"] += 1 + + by_test: dict[str, ItemRunRecord] = {} + for nodeid, by_when in phases.items(): + cats = categories.get(nodeid, []) + if "error" in cats: + outcome = "error" + elif cats: + outcome = cats[-1] + else: + outcome = "" + by_test[nodeid] = ItemRunRecord( + nodeid=nodeid, + setup=by_when.get("setup"), + call=by_when.get("call"), + teardown=by_when.get("teardown"), + outcome=outcome, + ) + + name_map: dict[str, ItemRunRecord | None] = {} + for nodeid, record in by_test.items(): + name = nodeid.rpartition("::")[2] + # Ambiguous bare names are poisoned rather than guessed. + name_map[name] = None if name in name_map else record + by_name = {name: rec for name, rec in name_map.items() if rec is not None} + + return cls( + reports=list(recorder.test_reports), + collect_reports=list(recorder.collect_reports), + warnings=list(recorder.warnings), + deselected=recorder.deselected, + by_test=by_test, + _by_name=by_name, + _counts=dict(counts), + ) + + def __getitem__(self, name: str) -> ItemRunRecord: + if name in self.by_test: + return self.by_test[name] + if name in self._by_name: + return self._by_name[name] + raise KeyError( + f"no unambiguous test named {name!r}; known: {sorted(self.by_test)}" + ) + + def outcomes(self) -> dict[str, int]: + """Outcome category counts, keyed like the terminal summary + ("passed", "failed", "errors", "skipped", "xfailed", "xpassed").""" + counts = dict(self._counts) + if "error" in counts: + counts["errors"] = counts.pop("error") + return counts + + def assert_outcomes( + self, + *, + passed: int = 0, + skipped: int = 0, + failed: int = 0, + errors: int = 0, + xpassed: int = 0, + xfailed: int = 0, + warnings: int | None = None, + deselected: int | None = None, + ) -> None: + """Assert the run produced exactly the given outcome counts. + + Signature-compatible with :meth:`RunResult.assert_outcomes + <_pytest.pytester.RunResult.assert_outcomes>`. + """ + __tracebackhide__ = True + from _pytest.pytester_assertions import assert_outcomes + + outcomes = self.outcomes() + outcomes["warnings"] = len(self.warnings) + outcomes["deselected"] = self.deselected + assert_outcomes( + outcomes, + passed=passed, + skipped=skipped, + failed=failed, + errors=errors, + xpassed=xpassed, + xfailed=xfailed, + warnings=warnings, + deselected=deselected, + ) + + +def run_items(session: Session, items: Sequence[Item] | None = None) -> RunRecord: + """Run items through the standard runtest protocol hook and return + the structured results recorded so far on this config.""" + if items is None: + items = list(session.items) + recorder = ensure_recorder(session.config) + for i, item in enumerate(items): + nextitem = items[i + 1] if i + 1 < len(items) else None + item.ihook.pytest_runtest_protocol(item=item, nextitem=nextitem) + return RunRecord.from_recorder(recorder, config=session.config) diff --git a/src/_pytest/unittest.py b/src/_pytest/unittest.py index d5286af1470..002196069e0 100644 --- a/src/_pytest/unittest.py +++ b/src/_pytest/unittest.py @@ -399,7 +399,7 @@ def runtest(self) -> None: # We need to consider if the test itself is skipped, or the whole class. assert isinstance(self.parent, UnitTestCase) skipped = _is_skipped(self.obj) or _is_skipped(self.parent.obj) - if self.config.getoption("usepdb") and not skipped: + if self.config.getoption("usepdb", False) and not skipped: self._explicit_tearDown = testcase.tearDown setattr(testcase, "tearDown", lambda *args: None) diff --git a/testing/python/collect.py b/testing/python/collect.py index c9023f98595..2e977090cda 100644 --- a/testing/python/collect.py +++ b/testing/python/collect.py @@ -2,18 +2,20 @@ from __future__ import annotations import os +from pathlib import Path import sys import textwrap from typing import Any import _pytest._code from _pytest.config import ExitCode -from _pytest.main import Session from _pytest.monkeypatch import MonkeyPatch from _pytest.nodes import Collector from _pytest.pytester import Pytester from _pytest.python import Class from _pytest.python import Function +from _pytest.scenario import collect_tests +from _pytest.scenario import run_tests import pytest @@ -180,54 +182,49 @@ def __new__(self): ] ) - def test_class_subclassobject(self, pytester: Pytester) -> None: - pytester.getmodulecol( - """ - class test(object): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*collected 0*"]) + def test_class_subclassobject(self, tmp_path: Path) -> None: + class test: + pass + + assert collect_tests(test, rootpath=tmp_path) == [] - def test_static_method(self, pytester: Pytester) -> None: + def test_static_method(self, tmp_path: Path) -> None: """Support for collecting staticmethod tests (#2528, #2699)""" - pytester.getmodulecol( - """ - import pytest - class Test(object): - @staticmethod - def test_something(): - pass - @pytest.fixture - def fix(self): - return 1 + class Test: + @staticmethod + def test_something(): + pass - @staticmethod - def test_fix(fix): - assert fix == 1 - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*collected 2 items*", "*2 passed in*"]) + @pytest.fixture + def fix(self): + return 1 - def test_setup_teardown_class_as_classmethod(self, pytester: Pytester) -> None: - pytester.makepyfile( - test_mod1=""" - class TestClassMethod(object): - @classmethod - def setup_class(cls): - pass - def test_1(self): - pass - @classmethod - def teardown_class(cls): - pass - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 passed*"]) + @staticmethod + def test_fix(fix): + assert fix == 1 + + record = run_tests(Test, rootpath=tmp_path) + record.assert_outcomes(passed=2) + + def test_setup_teardown_class_as_classmethod(self, tmp_path: Path) -> None: + events = [] + + class TestClassMethod: + @classmethod + def setup_class(cls): + events.append("setup") + + def test_1(self): + pass + + @classmethod + def teardown_class(cls): + events.append("teardown") + + record = run_tests(TestClassMethod, rootpath=tmp_path) + record.assert_outcomes(passed=1) + assert events == ["setup", "teardown"] def test_issue1035_obj_has_getattr(self, pytester: Pytester) -> None: modcol = pytester.getmodulecol( @@ -384,32 +381,32 @@ def __call__(self, tmp_path): ) @staticmethod - def make_function(pytester: Pytester, **kwargs: Any) -> Any: - from _pytest.fixtures import FixtureManager - - config = pytester.parseconfigure() - session = Session.from_config(config) - session._fixturemanager = FixtureManager(session) + def make_function(tmp_path: Path, **kwargs: Any) -> Any: + from _pytest.scenario import ConfigSpec + from _pytest.scenario import configured + from _pytest.scenario import running_session - return pytest.Function.from_parent(parent=session, **kwargs) + with configured(ConfigSpec(rootpath=tmp_path)) as config: + with running_session(config) as session: + return pytest.Function.from_parent(parent=session, **kwargs) - def test_function_equality(self, pytester: Pytester) -> None: + def test_function_equality(self, tmp_path: Path) -> None: def func1(): pass def func2(): pass - f1 = self.make_function(pytester, name="name", callobj=func1) + f1 = self.make_function(tmp_path, name="name", callobj=func1) assert f1 == f1 f2 = self.make_function( - pytester, name="name", callobj=func2, originalname="foobar" + tmp_path, name="name", callobj=func2, originalname="foobar" ) assert f1 != f2 - def test_repr_produces_actual_test_id(self, pytester: Pytester) -> None: + def test_repr_produces_actual_test_id(self, tmp_path: Path) -> None: f = self.make_function( - pytester, name=r"test[\xe5]", callobj=self.test_repr_produces_actual_test_id + tmp_path, name=r"test[\xe5]", callobj=self.test_repr_produces_actual_test_id ) assert repr(f) == r"" diff --git a/testing/python/fixtures.py b/testing/python/fixtures.py index 69044226b6d..ef21cb46e34 100644 --- a/testing/python/fixtures.py +++ b/testing/python/fixtures.py @@ -15,6 +15,7 @@ from _pytest.pytester import get_public_names from _pytest.pytester import Pytester from _pytest.python import Function +from _pytest.scenario import run_tests import pytest @@ -1181,50 +1182,38 @@ def test_request_fixturenames_dynamic_fixture(self, pytester: Pytester) -> None: result = pytester.runpytest("-vv") result.stdout.fnmatch_lines(["*1 passed*"]) - def test_setupdecorator_and_xunit(self, pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest - - values = [] + def test_setupdecorator_and_xunit(self, tmp_path: Path) -> None: + values: list[str] = [] - @pytest.fixture(scope='module', autouse=True) - def setup_module(): - values.append("module") + @pytest.fixture(scope="module", autouse=True) + def setup_module(): + values.append("module") - @pytest.fixture(autouse=True) - def setup_function(): - values.append("function") + @pytest.fixture(autouse=True) + def setup_function(): + values.append("function") - def test_func(): - pass + def test_func(): + pass - class TestClass: - @pytest.fixture(scope="class", autouse=True) - @classmethod - def setup_class(cls): - values.append("class") + class TestClass: + @pytest.fixture(scope="class", autouse=True) + @classmethod + def setup_class(cls): + values.append("class") - @pytest.fixture(autouse=True) - def setup_method(self): - values.append("method") + @pytest.fixture(autouse=True) + def setup_method(self): + values.append("method") - def test_method(self): - pass + def test_method(self): + pass - def test_all(): - assert values == [ - "module", - "function", - "class", - "function", - "method", - "function", - ] - """ + record = run_tests( + setup_module, setup_function, test_func, TestClass, rootpath=tmp_path ) - reprec = pytester.inline_run("-v") - reprec.assertoutcome(passed=3) + record.assert_outcomes(passed=2) + assert values == ["module", "function", "class", "function", "method"] def test_fixtures_sub_subdir_normalize_sep(self, pytester: Pytester) -> None: # this tests that normalization of nodeids takes place diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index 7dfc8b9986b..98fcd241590 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -5,6 +5,7 @@ from collections.abc import Sequence import dataclasses import itertools +from pathlib import Path import re import sys import textwrap @@ -24,6 +25,8 @@ from _pytest.pytester import Pytester from _pytest.python import Function from _pytest.python import IdMaker +from _pytest.scenario import fake_module +from _pytest.scenario import run_tests from _pytest.scope import Scope import pytest @@ -175,7 +178,7 @@ class Exc(Exception): def __repr__(self): return "Exc(from_gen)" - def gen() -> Iterator[int | None | Exc]: + def gen() -> Iterator[int | Exc | None]: yield 0 yield None yield Exc() @@ -1300,29 +1303,23 @@ def test_method(self, metafunc, pytestconfig): result = pytester.runpytest(p, "-v") result.assert_outcomes(passed=2) - def test_two_functions(self, pytester: Pytester) -> None: - p = pytester.makepyfile( - """ - def pytest_generate_tests(metafunc): - metafunc.parametrize('arg1', [10, 20], ids=['0', '1']) + def test_two_functions(self, tmp_path: Path) -> None: + def pytest_generate_tests(metafunc): + metafunc.parametrize("arg1", [10, 20], ids=["0", "1"]) - def test_func1(arg1): - assert arg1 == 10 + def test_func1(arg1): + assert arg1 == 10 - def test_func2(arg1): - assert arg1 in (10, 20) - """ - ) - result = pytester.runpytest("-v", p) - result.stdout.fnmatch_lines( - [ - "*test_func1*0*PASS*", - "*test_func1*1*FAIL*", - "*test_func2*PASS*", - "*test_func2*PASS*", - "*1 failed, 3 passed*", - ] + def test_func2(arg1): + assert arg1 in (10, 20) + + module = fake_module( + "test_two_functions", pytest_generate_tests, test_func1, test_func2 ) + record = run_tests(module, rootpath=tmp_path) + record.assert_outcomes(passed=3, failed=1) + assert record["test_two_functions.py::test_func1[0]"].passed + assert record["test_two_functions.py::test_func1[1]"].failed def test_noself_in_method(self, pytester: Pytester) -> None: p = pytester.makepyfile( diff --git a/testing/test_scenario.py b/testing/test_scenario.py new file mode 100644 index 00000000000..dafff3452b1 --- /dev/null +++ b/testing/test_scenario.py @@ -0,0 +1,412 @@ +"""Tests for the experimental _pytest.scenario API.""" + +from __future__ import annotations + +from collections.abc import Generator +import os +from pathlib import Path +import sys +import types +import unittest +import warnings + +from _pytest.config import Config +from _pytest.pytester import Pytester +from _pytest.scenario import collect_tests +from _pytest.scenario import ConfigSpec +from _pytest.scenario import configured +from _pytest.scenario import fake_module +from _pytest.scenario import run_tests +from _pytest.scenario import running_session +from _pytest.scenario import scenario +from _pytest.scenario import ScenarioModule +import pytest + + +class TestConfigSpec: + def test_rootpath_required(self) -> None: + with pytest.raises(ValueError, match="rootpath is required"): + with configured(ConfigSpec()): + pass + + def test_rootpath_must_be_dir(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="not a directory"): + with configured(ConfigSpec(rootpath=tmp_path / "missing")): + pass + + def test_essential_plugins_validated(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match=r"essential plugins.*runner"): + with configured(ConfigSpec(rootpath=tmp_path, plugins=("python", "mark"))): + pass + + def test_load_conftests_unsupported(self, tmp_path: Path) -> None: + with pytest.raises(NotImplementedError, match="conftest"): + with configured(ConfigSpec(rootpath=tmp_path, load_conftests=True)): + pass + + def test_configured_basics(self, tmp_path: Path) -> None: + spec = ConfigSpec(rootpath=tmp_path, args=("-k", "nothing")) + with configured(spec) as config: + assert config.rootpath == tmp_path + assert config.inipath is None + assert config.args == [] + assert config.args_source is Config.ArgsSource.SPEC + assert config.getoption("keyword") == "nothing" + assert config.invocation_params.dir == tmp_path + # paired teardown ran + assert not config._configured + + def test_inicfg_is_authoritative(self, tmp_path: Path) -> None: + spec = ConfigSpec(rootpath=tmp_path, inicfg={"usefixtures": ["myfix"]}) + with configured(spec) as config: + assert config.getini("usefixtures") == ["myfix"] + + def test_excluded_plugins_absent(self, tmp_path: Path) -> None: + with configured(ConfigSpec(rootpath=tmp_path)) as config: + assert config.pluginmanager.get_plugin("capturemanager") is None + assert config.pluginmanager.get_plugin("terminalreporter") is None + assert not config.pluginmanager.hasplugin("capture") + assert not config.pluginmanager.hasplugin("terminal") + assert not config.pluginmanager.hasplugin("cacheprovider") + + def test_spec_derivation_helpers(self) -> None: + spec = ConfigSpec() + derived = spec.with_plugins("capture").without_plugins("unittest") + assert "capture" in derived.plugins + assert "unittest" not in derived.plugins + # frozen: original unchanged + assert "capture" not in spec.plugins + + +class TestCollection: + def test_collect_loose_functions(self, tmp_path: Path) -> None: + def test_one() -> None: + pass + + def test_two() -> None: + pass + + items = collect_tests(test_one, test_two, rootpath=tmp_path) + assert [item.nodeid for item in items] == [ + "scenario_module.py::test_one", + "scenario_module.py::test_two", + ] + + def test_collect_class(self, tmp_path: Path) -> None: + class TestGroup: + def test_method(self) -> None: + pass + + @staticmethod + def test_static() -> None: + pass + + items = collect_tests(TestGroup, rootpath=tmp_path) + assert [item.name for item in items] == ["test_method", "test_static"] + assert items[0].nodeid == "scenario_module.py::TestGroup::test_method" + + def test_collect_module_object(self, tmp_path: Path) -> None: + def test_in_module() -> None: + pass + + module = fake_module("my_virtual", test_in_module) + items = collect_tests(module, rootpath=tmp_path) + assert [item.nodeid for item in items] == ["my_virtual.py::test_in_module"] + + def test_module_pytestmark_applies(self, tmp_path: Path) -> None: + def test_marked() -> None: + pass + + module = fake_module( + "marked_mod", + test_marked, + pytestmark=[pytest.mark.skip(reason="module-wide")], + ) + record = run_tests(module, rootpath=tmp_path) + record.assert_outcomes(skipped=1) + + def test_parametrize(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("x", [1, 2, 3]) + def test_param(x: int) -> None: + assert x < 3 + + items = collect_tests(test_param, rootpath=tmp_path) + assert [item.name for item in items] == [ + "test_param[1]", + "test_param[2]", + "test_param[3]", + ] + record = run_tests(test_param, rootpath=tmp_path) + record.assert_outcomes(passed=2, failed=1) + + def test_keyword_deselection(self, tmp_path: Path) -> None: + def test_alpha() -> None: + pass + + def test_beta() -> None: + pass + + spec = ConfigSpec(rootpath=tmp_path, args=("-k", "alpha")) + record = run_tests(test_alpha, test_beta, spec=spec) + record.assert_outcomes(passed=1, deselected=1) + + def test_lowercase_class_not_collected(self, tmp_path: Path) -> None: + class test: + pass + + assert collect_tests(test, rootpath=tmp_path) == [] + + def test_unittest_testcase(self, tmp_path: Path) -> None: + class MyCase(unittest.TestCase): + def test_method(self) -> None: + self.assertEqual(1, 1) + + record = run_tests(MyCase, rootpath=tmp_path) + record.assert_outcomes(passed=1) + + def test_two_module_sources(self, tmp_path: Path) -> None: + def test_a() -> None: + pass + + def test_b() -> None: + pass + + mod_a = fake_module("mod_a", test_a) + mod_b = fake_module("mod_b", test_b) + items = collect_tests(mod_a, mod_b, rootpath=tmp_path) + assert [item.nodeid for item in items] == [ + "mod_a.py::test_a", + "mod_b.py::test_b", + ] + + +class TestRunning: + def test_outcome_categories(self, tmp_path: Path) -> None: + def test_passes() -> None: + pass + + @pytest.mark.skipif("True", reason="nope") + def test_skips() -> None: + pass + + def test_fails() -> None: + left = 1 + assert left == 2 + + @pytest.mark.xfail(reason="known") + def test_xfails() -> None: + raise AssertionError("boom") + + record = run_tests( + test_passes, test_skips, test_fails, test_xfails, rootpath=tmp_path + ) + record.assert_outcomes(passed=1, skipped=1, failed=1, xfailed=1) + assert record["test_passes"].passed + assert record["test_fails"].failed + assert record["test_skips"].skipped + assert "nope" in record["test_skips"].setup.longreprtext # type: ignore[union-attr] + + def test_setup_error_is_error(self, tmp_path: Path) -> None: + @pytest.fixture + def broken() -> None: + raise RuntimeError("bad setup") + + def test_uses_broken(broken: None) -> None: + pass + + record = run_tests(broken, test_uses_broken, rootpath=tmp_path) + record.assert_outcomes(errors=1) + assert record["test_uses_broken"].outcome == "error" + + def test_warning_recorded(self, tmp_path: Path) -> None: + def test_warns() -> None: + warnings.warn(UserWarning("boo")) + + # The host suite runs with filterwarnings=error, which is process + # state the nested run inherits; the scenario's own ini filters + # take precedence over it. + spec = ConfigSpec(rootpath=tmp_path, inicfg={"filterwarnings": ["always"]}) + record = run_tests(test_warns, spec=spec) + record.assert_outcomes(passed=1, warnings=1) + assert "boo" in str(record.warnings[0].message) + + def test_getitem_ambiguity(self, tmp_path: Path) -> None: + def test_same() -> None: + pass + + mod_a = fake_module("dup_a", test_same) + mod_b = fake_module("dup_b", test_same) + record = run_tests(mod_a, mod_b, rootpath=tmp_path) + record.assert_outcomes(passed=2) + assert record["dup_a.py::test_same"].passed + with pytest.raises(KeyError, match="no unambiguous test"): + record["test_same"] + + def test_stepwise_scenario(self, tmp_path: Path) -> None: + def test_one() -> None: + pass + + with scenario(test_one, rootpath=tmp_path) as s: + items = s.collect() + assert len(items) == 1 + record = s.run() + record.assert_outcomes(passed=1) + + def test_sequential_scenarios(self, tmp_path: Path) -> None: + def test_first() -> None: + pass + + def test_second() -> None: + assert False + + run_tests(test_first, rootpath=tmp_path).assert_outcomes(passed=1) + run_tests(test_second, rootpath=tmp_path).assert_outcomes(failed=1) + + +class TestFixtures: + def test_function_fixture(self, tmp_path: Path) -> None: + @pytest.fixture + def value() -> int: + return 41 + + def test_uses_value(value: int) -> None: + assert value == 41 + + record = run_tests(value, test_uses_value, rootpath=tmp_path) + record.assert_outcomes(passed=1) + + def test_class_scoped_fixture_and_teardown_order(self, tmp_path: Path) -> None: + events: list[str] = [] + + class TestGroup: + @pytest.fixture(scope="class") + def resource(self) -> Generator[str]: + events.append("setup") + yield "res" + events.append("teardown") + + def test_one(self, resource: str) -> None: + events.append("one") + + def test_two(self, resource: str) -> None: + events.append("two") + + record = run_tests(TestGroup, rootpath=tmp_path) + record.assert_outcomes(passed=2) + assert events == ["setup", "one", "two", "teardown"] + + def test_module_scoped_fixture(self, tmp_path: Path) -> None: + events: list[str] = [] + + @pytest.fixture(scope="module") + def modres() -> Generator[int]: + events.append("setup") + yield 1 + events.append("teardown") + + def test_a(modres: int) -> None: + events.append("a") + + def test_b(modres: int) -> None: + events.append("b") + + record = run_tests(modres, test_a, test_b, rootpath=tmp_path) + record.assert_outcomes(passed=2) + assert events == ["setup", "a", "b", "teardown"] + + def test_request_module_is_synthesized_module(self, tmp_path: Path) -> None: + seen: list[types.ModuleType] = [] + + def test_introspect(request: pytest.FixtureRequest) -> None: + seen.append(request.module) + + run_tests(test_introspect, rootpath=tmp_path).assert_outcomes(passed=1) + (module,) = seen + assert isinstance(module, types.ModuleType) + assert module.__name__ == "scenario_module" + + def test_fixture_from_extra_plugin(self, tmp_path: Path) -> None: + class FixturePlugin: + @pytest.fixture + def injected(self) -> str: + return "from-plugin" + + def test_uses_injected(injected: str) -> None: + assert injected == "from-plugin" + + spec = ConfigSpec(rootpath=tmp_path, extra_plugins=(FixturePlugin(),)) + run_tests(test_uses_injected, spec=spec).assert_outcomes(passed=1) + + def test_monkeypatch_fixture_available(self, tmp_path: Path) -> None: + def test_uses_monkeypatch(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SCENARIO_PROBE", "1") + assert os.environ["SCENARIO_PROBE"] == "1" + + run_tests(test_uses_monkeypatch, rootpath=tmp_path).assert_outcomes(passed=1) + assert "SCENARIO_PROBE" not in os.environ + + +class TestNodeConstruction: + def test_class_from_parent_keeps_obj(self, tmp_path: Path) -> None: + """Class.from_parent(obj=...) takes precedence over name lookup.""" + + class Hidden: + def test_method(self) -> None: + pass + + empty = fake_module("holder") + with configured(ConfigSpec(rootpath=tmp_path)) as config: + with running_session(config) as session: + module = ScenarioModule.from_parent(session, obj=empty, name="holder") + cls = pytest.Class.from_parent( + module, name="NotAnAttribute", obj=Hidden + ) + assert cls.obj is Hidden + assert [item.name for item in cls.collect()] == ["test_method"] + + +class TestHermeticity: + def test_no_process_state_leaked(self, tmp_path: Path) -> None: + def test_noop() -> None: + pass + + # Warm-up: let lazy imports happen before snapshotting. + run_tests(test_noop, rootpath=tmp_path).assert_outcomes(passed=1) + + cwd = os.getcwd() + sys_path = list(sys.path) + modules = set(sys.modules) + environ = dict(os.environ) + + run_tests(test_noop, rootpath=tmp_path).assert_outcomes(passed=1) + + assert os.getcwd() == cwd + assert sys.path == sys_path + assert set(sys.modules) == modules + assert dict(os.environ) == environ + + def test_no_files_created(self, tmp_path: Path) -> None: + def test_noop() -> None: + pass + + run_tests(test_noop, rootpath=tmp_path).assert_outcomes(passed=1) + assert list(tmp_path.iterdir()) == [] + + +class TestPytesterInterplay: + def test_scenario_inside_full_pytest_run(self, pytester: Pytester) -> None: + """The scenario API works inside a captured, terminal-full pytest run.""" + pytester.makepyfile( + """ + from _pytest.scenario import run_tests + + def test_host(tmp_path): + def test_inner(): + assert 1 + 1 == 2 + + record = run_tests(test_inner, rootpath=tmp_path) + record.assert_outcomes(passed=1) + """ + ) + result = pytester.runpytest_inprocess() + result.assert_outcomes(passed=1) diff --git a/testing/test_warnings.py b/testing/test_warnings.py index 6b439eb4b33..7280b1c44ba 100644 --- a/testing/test_warnings.py +++ b/testing/test_warnings.py @@ -784,15 +784,16 @@ def test_issue4445_initial_conftest(self, pytester: Pytester, capwarn) -> None: ) pytester.parseconfig("--help") - # with stacklevel=2 the warning should originate from config._preparse and is - # thrown by an erroneous conftest.py + # with stacklevel=2 the warning should originate from the conftest + # loading phase of config.parse and is thrown by an erroneous + # conftest.py assert len(capwarn.captured) == 1 warning, location = capwarn.captured.pop() file, _, func = location assert "could not load initial conftests" in str(warning.message) assert f"config{os.sep}__init__.py" in file - assert func == "parse" + assert func == "_load_initial_conftests_phase" @pytest.mark.filterwarnings("default") def test_conftest_warning_captured(self, pytester: Pytester) -> None: