From d011e352fd4bcbdde429ffacef9a3bf952dc0d2d Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 11:17:01 +0200 Subject: [PATCH 1/2] Add opt-in FunctionDefinition collection tree node (#3926) Introduce the ``collect_function_definition`` ini option to promote the internal ``FunctionDefinition`` from a transient parametrization helper to a real collector node in the collection tree. The option is tri-state: - ``hidden`` (default): unchanged flat layout; the definition drives parametrization and is kept out of the tree. - ``pedantic``: insert the definition node between the Module/Class and its test functions; function-level markers are scoped to the definition and each invocation owns only its callspec markers. - ``messy``: migration stopgap -- insert the node but transfer the function-level markers back onto each invocation to preserve the legacy marker layout for code not yet prepared for the new scope; emits a header warning. FunctionDefinition becomes a PyCollector (no longer a Function/Item), so it is collected rather than run. Invocation node ids stay flat and identical across all modes, so selection, caching and reporting are unaffected. obj and instance resolution and getmodpath skip an interposed definition node. Co-Authored-By: Claude Opus 4.8 --- changelog/3926.improvement.rst | 15 ++ doc/en/reference/reference.rst | 81 +++++++ src/_pytest/main.py | 11 + src/_pytest/python.py | 255 +++++++++++++++----- testing/test_collect_function_definition.py | 193 +++++++++++++++ 5 files changed, 492 insertions(+), 63 deletions(-) create mode 100644 changelog/3926.improvement.rst create mode 100644 testing/test_collect_function_definition.py diff --git a/changelog/3926.improvement.rst b/changelog/3926.improvement.rst new file mode 100644 index 00000000000..98f5503663d --- /dev/null +++ b/changelog/3926.improvement.rst @@ -0,0 +1,15 @@ +Added the :confval:`collect_function_definition` option, which can promote the +internal ``FunctionDefinition`` to a real node in the collection tree. + +When set to ``pedantic``, a ``FunctionDefinition`` collector node is inserted +between a :class:`~pytest.Module`/:class:`~pytest.Class` and its test functions, +so that the (possibly parametrized) invocations of each test are collected +underneath a shared definition node instead of as flat siblings. Function-level +markers are scoped to the definition node in this mode. + +The default ``hidden`` keeps the previous flat layout, and the transitional +``messy`` value keeps the definition node out of marker resolution for code that +is not yet prepared for the new scope. + +The node ids of the individual test invocations are unchanged in all modes, so +test selection, caching and reporting behave exactly as before. diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 4f2e0b70850..b7235d58a99 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -1378,6 +1378,79 @@ passed multiple times. The expected format is ``name=value``. For example:: variables, that will be expanded. For more information about cache plugin please refer to :ref:`cache_provider`. +.. confval:: collect_function_definition + :type: ``string`` + :default: ``hidden`` + + .. versionadded:: 9.2 + + Controls whether a ``FunctionDefinition`` collector node is inserted into the + collection tree between a :class:`~pytest.Module`/:class:`~pytest.Class` and + its test functions, grouping the (possibly parametrized) invocations of a test + function *underneath* the definition node instead of as flat siblings. + + It accepts three values: + + ``hidden`` (default) + Keep the flat layout. The definition node is used only internally to drive + parametrization and is kept out of the collection tree. + + ``pedantic`` + Insert the definition node with correct marker scoping: function-level + markers (such as ``@pytest.mark.parametrize`` or custom marks) belong to + the definition node, and each invocation only owns its own parameter-set + markers. This is the intended target layout. + + ``messy`` + A stopgap for migration only. It inserts the definition node but transfers + the function-level markers back down onto each invocation, reproducing the + old (duplicated) marker layout so that code which is not yet prepared for + the definition scope keeps working. It emits a warning in the session + header and exists purely to buy time; fix the offending code and move to + ``pedantic``. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + collect_function_definition = "pedantic" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + collect_function_definition = pedantic + + For example, given a parametrized test: + + .. code-block:: python + + import pytest + + + @pytest.mark.parametrize("x", [1, 2]) + def test_it(x): ... + + the default (``hidden``) layout collects the invocations directly under the + module:: + + + + + + while ``pedantic`` and ``messy`` group them under a definition node:: + + + + + + + This is an opt-in structural change. The node ids of the individual test + invocations are unchanged (for example ``test_it.py::test_it[1]``), so test + selection, caching and reporting behave exactly as before. + .. confval:: collect_imported_tests :type: ``bool`` :default: ``true`` @@ -3677,6 +3750,14 @@ All the command-line flags can also be obtained by running ``pytest --help``:: collect_imported_tests (bool): Whether to collect tests in imported modules outside `testpaths` + collect_function_definition (string): + How the function-definition collector node + participates in the collection tree. - hidden + (default): keep the flat layout, no node in the tree + - pedantic: insert the node and scope function-level + markers to it - messy: insert the node but transfer + markers down to each invocation to preserve the + legacy marker layout (emits a warning) consider_namespace_packages (bool): Consider namespace packages when resolving module names during import diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 1b337e20c7e..364f27b7d2d 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -251,6 +251,17 @@ def pytest_addoption(parser: Parser) -> None: type="bool", default=True, ) + parser.addini( + "collect_function_definition", + "How the function-definition collector node participates in the " + "collection tree.\n" + "- hidden (default): keep the flat layout, no node in the tree\n" + "- pedantic: insert the node and scope function-level markers to it\n" + "- messy: insert the node but transfer markers down to each invocation " + "to preserve the legacy marker layout (emits a warning)", + type="string", + default="hidden", + ) parser.addini( "consider_namespace_packages", type="bool", diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 4f168012c36..7b2d26669ea 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -91,6 +91,29 @@ get_args(LongStrIdStrategy) ) +# Modes for the ``collect_function_definition`` option. +# - "hidden": legacy flat layout; the FunctionDefinition is used transiently to +# drive parametrization and kept out of ("hidden" from) the tree. +# - "pedantic": insert the FunctionDefinition node; function-level markers are +# scoped to it and each invocation only owns its callspec markers. +# - "messy": insert the node, but transfer the function-level markers down onto +# each invocation to preserve the legacy marker layout. Emits a +# warning, as this defeats the purpose of the definition scope. +FunctionDefinitionMode = Literal["hidden", "pedantic", "messy"] +_FUNCTION_DEFINITION_MODES: frozenset[FunctionDefinitionMode] = frozenset( + get_args(FunctionDefinitionMode) +) + + +def _collect_function_definition_mode(config: Config) -> FunctionDefinitionMode: + value = config.getini("collect_function_definition") + if value not in _FUNCTION_DEFINITION_MODES: + raise UsageError( + f"Unknown collect_function_definition: {value!r}. " + f"Valid values: {', '.join(sorted(_FUNCTION_DEFINITION_MODES))}" + ) + return cast(FunctionDefinitionMode, value) + def pytest_addoption(parser: Parser) -> None: parser.addini( @@ -163,6 +186,16 @@ def pytest_configure(config: Config) -> None: ) +def pytest_report_header(config: Config) -> list[str] | None: + if _collect_function_definition_mode(config) == "messy": + return [ + "warning: collect_function_definition=messy transfers markers to the " + "individual invocations to preserve the legacy layout; prefer " + "'pedantic' once your plugins handle the definition scope", + ] + return None + + def async_fail(nodeid: str) -> None: msg = ( "async def functions are not natively supported.\n" @@ -331,6 +364,11 @@ def getmodpath(self, stopatmodule: bool = True, includemodule: bool = False) -> """Return Python path relative to the containing module.""" parts = [] for node in self.iter_parents(): + # A FunctionDefinition parent contributes the same name as the + # Function collected under it, so skip it to avoid duplication + # (but keep it when it is the node itself). + if node is not self and isinstance(node, FunctionDefinition): + continue name = node.name if isinstance(node, Module): name = os.path.splitext(name)[0] @@ -467,54 +505,19 @@ def collect(self) -> Iterable[nodes.Item | nodes.Collector]: result.extend(values) return result - def _genfunctions(self, name: str, funcobj) -> Iterator[Function]: - modulecol = self.getparent(Module) - assert modulecol is not None - module = modulecol.obj - clscol = self.getparent(Class) - cls = (clscol and clscol.obj) or None - + def _genfunctions( + self, name: str, funcobj + ) -> Iterator[Function | FunctionDefinition]: definition = FunctionDefinition.from_parent(self, name=name, callobj=funcobj) - fixtureinfo = definition._fixtureinfo - - # pytest_generate_tests impls call metafunc.parametrize() which fills - # metafunc._calls, the outcome of the hook. - metafunc = Metafunc( - definition=definition, - fixtureinfo=fixtureinfo, - config=self.config, - cls=cls, - module=module, - _ispytest=True, - ) - methods = [] - if hasattr(module, "pytest_generate_tests"): - methods.append(module.pytest_generate_tests) - if cls is not None and hasattr(cls, "pytest_generate_tests"): - methods.append(cls().pytest_generate_tests) - self.ihook.pytest_generate_tests.call_extra(methods, dict(metafunc=metafunc)) - - if not metafunc._calls: - yield Function.from_parent(self, name=name, fixtureinfo=fixtureinfo) + if _collect_function_definition_mode(self.config) == "hidden": + # Legacy flat layout: the definition is used only to drive + # parametrization and is discarded ("hidden"), the invocations are + # collected directly under this collector. + yield from definition._generate_functions(self) else: - metafunc._recompute_direct_params_indices() - # Direct parametrizations taking place in module/class-specific - # `metafunc.parametrize` calls may have shadowed some fixtures, so make sure - # we update what the function really needs a.k.a its fixture closure. Note that - # direct parametrizations using `@pytest.mark.parametrize` have already been considered - # into making the closure using `ignore_args` arg to `getfixtureclosure`. - fixtureinfo.prune_dependency_tree() - - for callspec in metafunc._calls: - subname = f"{name}[{callspec.id}]" if callspec._idlist else name - yield Function.from_parent( - self, - name=subname, - callspec=callspec, - fixtureinfo=fixtureinfo, - keywords={callspec.id: True}, - originalname=name, - ) + # Insert the function definition as a collector node into the tree; + # its ``collect()`` yields the (possibly parametrized) invocations. + yield definition def importtestmodule( @@ -1688,8 +1691,9 @@ def __init__( session: Session | None = None, fixtureinfo: FuncFixtureInfo | None = None, originalname: str | None = None, + nodeid: str | None = None, ) -> None: - super().__init__(name, parent, config=config, session=session) + super().__init__(name, parent, config=config, session=session, nodeid=nodeid) if callobj is not NOTSET: self._obj = callobj @@ -1706,7 +1710,12 @@ def __init__( # Note: when FunctionDefinition is introduced, we should change ``originalname`` # to a readonly property that returns FunctionDefinition.name. - self.own_markers.extend(get_unpacked_marks(self.obj)) + # Function-level markers are owned by the FunctionDefinition scope when + # that node is part of the collection tree; otherwise (flat layout) the + # Function owns them itself. In "messy" mode FunctionDefinition.collect() + # transfers them back onto each invocation for legacy compatibility. + if not isinstance(self.parent, FunctionDefinition): + self.own_markers.extend(get_unpacked_marks(self.obj)) if callspec: self.callspec = callspec self.own_markers.extend(callspec.marks) @@ -1748,17 +1757,16 @@ def instance(self): try: return self._instance except AttributeError: - if isinstance(self.parent, Class): - # Each Function gets a fresh class instance. - self._instance = self._getinstance() - else: - self._instance = None + self._instance = self._getinstance() return self._instance def _getinstance(self): - if isinstance(self.parent, Class): + # The containing class, if any -- skipping over an interposed + # FunctionDefinition node (see the ``collect_function_definition`` option). + cls = self.getparent(Class) + if cls is not None: # Each Function gets a fresh class instance. - return self.parent.newinstance() + return cls.newinstance() else: return None @@ -1767,8 +1775,13 @@ def _getobj(self): if instance is not None: parent_obj = instance else: - assert self.parent is not None - parent_obj = self.parent.obj # type: ignore[attr-defined] + # The namespace this function was collected from -- a Module (or a + # Class handled above), skipping over an interposed FunctionDefinition. + parent = self.parent + while isinstance(parent, FunctionDefinition): + parent = parent.parent + assert parent is not None + parent_obj = parent.obj # type: ignore[attr-defined] return getattr(parent_obj, self.originalname) @property @@ -1823,14 +1836,130 @@ def repr_failure( # type: ignore[override] return self._repr_failure_py(excinfo, style=style) -class FunctionDefinition(Function): - """This class is a stop gap solution until we evolve to have actual function - definition nodes and manage to get rid of ``metafunc``.""" +class FunctionDefinition(PyCollector): + """Collector node for a single test function definition. - def runtest(self) -> None: - raise RuntimeError("function definitions are not supposed to be run as tests") + Its children are the (possibly parametrized) :class:`Function` invocations + generated from the definition via :hook:`pytest_generate_tests`. + + This node is only inserted into the collection tree when the + :confval:`collect_function_definition` option is enabled. Otherwise it is + created transiently to drive parametrization (backing :class:`Metafunc`) + and then discarded, with the resulting :class:`Function` items collected + directly under the containing :class:`Class`/:class:`Module`. + """ - setup = runtest + # Markers are handled explicitly below, mirroring Function. + _ALLOW_MARKERS = False + + def __init__( + self, + name: str, + parent, + callobj, + config: Config | None = None, + session: Session | None = None, + ) -> None: + super().__init__(name, parent, config=config, session=session) + + # The definition always stands for a concrete function object; unlike + # Function it never looks the object up from its parent lazily. + self._obj = callobj + + self.own_markers.extend(get_unpacked_marks(self.obj)) + self.keywords.update((mark.name, mark) for mark in self.own_markers) + self.keywords.update(self.obj.__dict__) + + def collect(self) -> Iterable[nodes.Item | nodes.Collector]: + children = list(self._generate_functions(self)) + if _collect_function_definition_mode(self.config) == "messy": + # Legacy marker layout: transfer this scope's markers back onto each + # invocation and drop them here, so marker resolution matches the flat + # layout exactly (no duplication when walking parents in iter_markers). + marks = self.own_markers + for child in children: + child.own_markers[:0] = marks + child.keywords.update((mark.name, mark) for mark in marks) + self.own_markers = [] + return children + + def _generate_functions(self, parent: nodes.Collector) -> Iterator[Function]: + """Run :hook:`pytest_generate_tests` and yield the resulting + :class:`Function` invocations under ``parent``. + + ``parent`` is this definition node when it is part of the tree, or the + containing :class:`Class`/:class:`Module` in the legacy flat layout. + """ + name = self.name + modulecol = self.getparent(Module) + assert modulecol is not None + module = modulecol.obj + clscol = self.getparent(Class) + cls = (clscol and clscol.obj) or None + + # Compute the function's fixture closure. This drives parametrization and + # is shared with the generated invocations; it is intentionally *not* + # stored on the definition -- fixtures belong to the executed items, not + # to this collector. + # TODO(#3926): getfixtureinfo() is item-scoped, but here the definition + # (a collector) stands in for the not-yet-created invocations. Resolve by + # giving fixture-closure computation a node-level entry point instead of + # casting a collector to an item. + fixtureinfo = self.session._fixturemanager.getfixtureinfo( + cast(nodes.Item, self), self.obj, cls + ) + + # pytest_generate_tests impls call metafunc.parametrize() which fills + # metafunc._calls, the outcome of the hook. + metafunc = Metafunc( + definition=self, + fixtureinfo=fixtureinfo, + config=self.config, + cls=cls, + module=module, + _ispytest=True, + ) + methods = [] + if hasattr(module, "pytest_generate_tests"): + methods.append(module.pytest_generate_tests) + if cls is not None and hasattr(cls, "pytest_generate_tests"): + methods.append(cls().pytest_generate_tests) + self.ihook.pytest_generate_tests.call_extra(methods, dict(metafunc=metafunc)) + + # The invocations keep a flat nodeid anchored at the collector containing + # the definition, regardless of whether the definition itself is part of + # the tree, so nodeids are stable across the ``collect_function_definition`` + # option. + assert self.parent is not None + base_nodeid = self.parent.nodeid + + if not metafunc._calls: + yield Function.from_parent( + parent, + name=name, + fixtureinfo=fixtureinfo, + nodeid=f"{base_nodeid}::{name}", + ) + else: + metafunc._recompute_direct_params_indices() + # Direct parametrizations taking place in module/class-specific + # `metafunc.parametrize` calls may have shadowed some fixtures, so make sure + # we update what the function really needs a.k.a its fixture closure. Note that + # direct parametrizations using `@pytest.mark.parametrize` have already been considered + # into making the closure using `ignore_args` arg to `getfixtureclosure`. + fixtureinfo.prune_dependency_tree() + + for callspec in metafunc._calls: + subname = f"{name}[{callspec.id}]" if callspec._idlist else name + yield Function.from_parent( + parent, + name=subname, + callspec=callspec, + fixtureinfo=fixtureinfo, + keywords={callspec.id: True}, + originalname=name, + nodeid=f"{base_nodeid}::{subname}", + ) def __getattr__(name: str) -> object: diff --git a/testing/test_collect_function_definition.py b/testing/test_collect_function_definition.py new file mode 100644 index 00000000000..5860bd2c356 --- /dev/null +++ b/testing/test_collect_function_definition.py @@ -0,0 +1,193 @@ +"""Tests for the `collect_function_definition` configuration value.""" + +from __future__ import annotations + +from _pytest.pytester import Pytester +import pytest + + +@pytest.fixture +def sample(pytester: Pytester) -> Pytester: + pytester.makepyfile( + """ + import pytest + + def test_plain(): + pass + + @pytest.mark.parametrize("x", [1, 2]) + def test_param(x): + assert x + + class TestCls: + @pytest.mark.parametrize("y", [3, 4]) + def test_method(self, y): + assert y + + def test_single(self): + pass + """ + ) + return pytester + + +def _set_mode(pytester: Pytester, mode: str) -> None: + pytester.makeini(f"[pytest]\ncollect_function_definition = {mode}\n") + + +FLAT_IDS = [ + "test_plain", + "test_param[1]", + "test_param[2]", + "TestCls::test_method[3]", + "TestCls::test_method[4]", + "TestCls::test_single", +] + +TREE_MODES = ["pedantic", "messy"] +ALL_MODES = ["hidden", *TREE_MODES] + + +def test_hidden_is_default(sample: Pytester) -> None: + """Without the option the definition node is not part of the tree.""" + items, _ = sample.inline_genitems() + for item in items: + assert type(item.parent).__name__ in ("Module", "Class") + + +@pytest.mark.parametrize("mode", TREE_MODES) +def test_definition_node_inserted(sample: Pytester, mode: str) -> None: + """In tree modes a FunctionDefinition collector wraps each test.""" + from _pytest.python import Function + from _pytest.python import FunctionDefinition + + _set_mode(sample, mode) + items, _ = sample.inline_genitems() + assert len(items) == len(FLAT_IDS) + for item in items: + assert isinstance(item, Function) + # Every invocation, including non-parametrized ones, is wrapped. + assert isinstance(item.parent, FunctionDefinition) + + +@pytest.mark.parametrize("mode", ALL_MODES) +def test_nodeids_are_stable(sample: Pytester, mode: str) -> None: + """Invocation nodeids stay flat regardless of the mode.""" + _set_mode(sample, mode) + items, _ = sample.inline_genitems() + suffixes = [item.nodeid.split("::", 1)[1] for item in items] + assert suffixes == FLAT_IDS + + +@pytest.mark.parametrize("mode", ALL_MODES) +def test_tests_run(sample: Pytester, mode: str) -> None: + _set_mode(sample, mode) + result = sample.runpytest() + result.assert_outcomes(passed=6) + + +def test_selection_and_reporting(sample: Pytester) -> None: + """-k selection and failure reporting address the flat nodeid.""" + _set_mode(sample, "pedantic") + result = sample.runpytest("-k", "test_param and 1", "-v") + assert "test_param[1] PASSED" in result.stdout.str() + result.assert_outcomes(passed=1, deselected=5) + + +def test_collect_only_tree(sample: Pytester) -> None: + _set_mode(sample, "pedantic") + result = sample.runpytest("--collect-only") + out = result.stdout.str() + for expected in [ + "", + "", + "", + "", + "", + "", + ]: + assert expected in out + + +@pytest.mark.parametrize("mode", ALL_MODES) +def test_markers_are_not_duplicated(pytester: Pytester, mode: str) -> None: + """A function-level marker resolves exactly once in every mode. + + Regression guard: with the definition node in the tree both it and the + invocation carry the same function-level marks, which must not double up + when walking parents in ``iter_markers``. + """ + pytester.makeini(f"[pytest]\ncollect_function_definition = {mode}\n") + pytester.makepyfile( + """ + import pytest + + @pytest.mark.foo + @pytest.mark.parametrize("x", [1]) + def test_it(x): + pass + """ + ) + items, _ = pytester.inline_genitems() + (item,) = items + assert len(list(item.iter_markers(name="foo"))) == 1 + + +def test_pedantic_scopes_markers_to_definition(pytester: Pytester) -> None: + """In pedantic mode function-level markers live on the definition node.""" + from _pytest.python import FunctionDefinition + + pytester.makeini("[pytest]\ncollect_function_definition = pedantic\n") + pytester.makepyfile( + """ + import pytest + + @pytest.mark.foo + def test_it(): + pass + """ + ) + (item,) = pytester.inline_genitems()[0] + assert isinstance(item.parent, FunctionDefinition) + assert [m.name for m in item.own_markers] == [] + assert [m.name for m in item.parent.own_markers] == ["foo"] + + +def test_messy_transfers_markers_to_invocation(pytester: Pytester) -> None: + """In messy mode markers are transferred back onto the invocation.""" + from _pytest.python import FunctionDefinition + + pytester.makeini("[pytest]\ncollect_function_definition = messy\n") + pytester.makepyfile( + """ + import pytest + + @pytest.mark.foo + def test_it(): + pass + """ + ) + (item,) = pytester.inline_genitems()[0] + assert isinstance(item.parent, FunctionDefinition) + assert [m.name for m in item.own_markers] == ["foo"] + # Transferred away from the definition scope to avoid duplication. + assert [m.name for m in item.parent.own_markers] == [] + + +def test_messy_emits_header_warning(sample: Pytester) -> None: + _set_mode(sample, "messy") + result = sample.runpytest() + result.stdout.fnmatch_lines(["*collect_function_definition=messy*"]) + + +def test_pedantic_has_no_header_warning(sample: Pytester) -> None: + _set_mode(sample, "pedantic") + result = sample.runpytest() + result.stdout.no_fnmatch_line("*collect_function_definition=messy*") + + +def test_invalid_value_errors(sample: Pytester) -> None: + _set_mode(sample, "bogus") + result = sample.runpytest() + result.stderr.fnmatch_lines(["*Unknown collect_function_definition: 'bogus'*"]) + assert result.ret != 0 From 1642d95a4755d022c4a3a2c0ca2ebaba57463905 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 30 Jul 2026 17:44:30 +0200 Subject: [PATCH 2/2] Add a definition scope and a generic ItemDefinition node (#3926) Build on the opt-in FunctionDefinition collection node in two directions: a fixture scope which targets that node, and a collector-agnostic base so non-Python test definitions can be parametrized through the same hook. Scope.Definition sits between Function and Class and is anchored on the definition node, so one fixture instance is shared by all invocations generated from a single definition: @pytest.fixture(scope="definition") def resource(): ... @pytest.mark.parametrize("n", [1, 2, 3]) def test_it(resource, n): ... It requires the definition to actually be in the tree, i.e. collect_function_definition=pedantic. Without it, both a definition-scoped fixture and parametrize(scope="definition") fail with an explicit message rather than silently degrading to function scope. --setup-show spells the scope "D" and keeps the existing indentation for every other scope. nodes.ItemDefinition is the generic counterpart of FunctionDefinition: a collector standing for one test definition, whose collect() runs the generate-tests protocol and turns each callspec into an item. A non-Python collector declares parametrize_argnames and implements make_item(); the chosen values arrive on item.callspec.params, with ids, pytest.param marks and nodeid selection working as they do for Python tests. There is no fixture resolution on that path -- doc/en/proposals/generic_fixture_closures writes up what that would take. The parametrization core moves out of python.py into the new _pytest.parametrize module: CallSpec, IdMaker and the new ParametrizeContext, of which Metafunc is now the Python-specific subclass adding the fixture closure, signature validation and direct-param desugaring. _pytest.python.CallSpec2 keeps warning and resolving as before. Two bugs fixed along the way, both reachable only once a definition node is in the tree: node-id selection of a parametrized test looked at the definition's bare name and failed to match, and skipping an item whose reportinfo() has no line number crashed with an INTERNALERROR. Co-Authored-By: Claude Opus 5 (1M context) --- changelog/14769.bugfix.rst | 11 + changelog/3926.feature.rst | 32 + changelog/3926.misc.rst | 7 + doc/en/example/nonpython.rst | 62 ++ doc/en/how-to/fixtures.rst | 11 +- doc/en/proposals/generic_fixture_closures.rst | 143 +++ doc/en/reference/reference.rst | 20 + src/_pytest/fixtures.py | 112 ++- src/_pytest/main.py | 11 +- src/_pytest/nodes.py | 138 +++ src/_pytest/parametrize.py | 809 +++++++++++++++++ src/_pytest/python.py | 852 +++--------------- src/_pytest/reports.py | 6 +- src/_pytest/scope.py | 10 +- src/_pytest/setuponly.py | 18 +- testing/deprecated_test.py | 2 +- testing/python/metafunc.py | 4 +- testing/test_collect_function_definition.py | 26 + testing/test_definition_scope.py | 344 +++++++ testing/test_item_definition.py | 199 ++++ testing/test_scope.py | 9 +- 21 files changed, 2058 insertions(+), 768 deletions(-) create mode 100644 changelog/14769.bugfix.rst create mode 100644 changelog/3926.feature.rst create mode 100644 changelog/3926.misc.rst create mode 100644 doc/en/proposals/generic_fixture_closures.rst create mode 100644 src/_pytest/parametrize.py create mode 100644 testing/test_definition_scope.py create mode 100644 testing/test_item_definition.py diff --git a/changelog/14769.bugfix.rst b/changelog/14769.bugfix.rst new file mode 100644 index 00000000000..4478b40285b --- /dev/null +++ b/changelog/14769.bugfix.rst @@ -0,0 +1,11 @@ +Selecting a parametrized test by its node id (``pytest test_x.py::test_a[1]``) +now works when the test definition is a node of the collection tree +(:confval:`collect_function_definition` set to ``pedantic`` or ``messy``). The +argument matcher compares names level by level, and the definition node carries +the bare name while the parametrization lives on the items below it; it now +looks through the definition instead of failing to match. + +Relatedly, skipping a non-Python item whose ``reportinfo()`` reports no line +number no longer crashes pytest with an ``INTERNALERROR``; the skip location is +reported without a line instead. This became easy to hit now that non-Python +definitions can carry ``pytest.param(..., marks=pytest.mark.skip(...))``. diff --git a/changelog/3926.feature.rst b/changelog/3926.feature.rst new file mode 100644 index 00000000000..ca44dc5c59e --- /dev/null +++ b/changelog/3926.feature.rst @@ -0,0 +1,32 @@ +Added the ``"definition"`` fixture scope, which shares one fixture instance +across all invocations generated from a single test definition -- that is, +across the parameter sets of one parametrized test, but not between two +different tests: + +.. code-block:: python + + @pytest.fixture(scope="definition") + def resource(): ... + + + @pytest.mark.parametrize("n", [1, 2, 3]) + def test_it(resource, n): + # one `resource` for all three parameter sets + ... + +The scope needs the test definition to be a node of the collection tree, so it +requires :confval:`collect_function_definition` to be set to ``pedantic`` (or +``messy``). Using it without that fails with an explicit error rather than +silently degrading to function scope. + +``@pytest.mark.parametrize(..., scope="definition")`` is supported as well and +groups the invocations of one definition by parameter value. + +Added :class:`_pytest.nodes.ItemDefinition`, the collector-agnostic base for "a +definition which generates items". Subclassing it lets a non-Python collector +have its test definitions parametrized through :hook:`pytest_generate_tests`, +exactly like Python test functions: declare the accepted names in +``parametrize_argnames``, build one item per parameter set in ``make_item()``, +and the chosen values arrive on ``item.callspec.params``. Parameter ids, +``pytest.param(...)`` marks and node-id selection all work as usual; there is no +fixture resolution on that path. See :ref:`non-python parametrization`. diff --git a/changelog/3926.misc.rst b/changelog/3926.misc.rst new file mode 100644 index 00000000000..31cce0a7e6a --- /dev/null +++ b/changelog/3926.misc.rst @@ -0,0 +1,7 @@ +Moved the parametrization core out of ``_pytest.python`` into the new +``_pytest.parametrize`` module: ``CallSpec``, ``IdMaker`` and the new +``ParametrizeContext``, of which :class:`pytest.Metafunc` is now the +Python-specific subclass adding the fixture closure, signature validation and +direct-parameter desugaring. + +``_pytest.python.CallSpec2`` keeps warning and resolving as before. diff --git a/doc/en/example/nonpython.rst b/doc/en/example/nonpython.rst index 9ac2b508ceb..06341428e9e 100644 --- a/doc/en/example/nonpython.rst +++ b/doc/en/example/nonpython.rst @@ -105,3 +105,65 @@ interesting to look at the collection tree: ======================== 2 tests collected in 0.12s ======================== + +.. _`non-python parametrization`: + +Letting non-python tests take part in parametrization +----------------------------------------------------- + +.. versionadded:: 9.2 + +A collector which yields items directly, as ``YamlFile`` above does, produces a +fixed set of tests. To let a test *definition* be parametrized -- by +:hook:`pytest_generate_tests`, exactly like a Python test function -- collect +:class:`~_pytest.nodes.ItemDefinition` nodes instead of items. + +Such a node declares the names it accepts in +:attr:`~_pytest.nodes.ItemDefinition.parametrize_argnames` and builds one item +per parameter set in :meth:`~_pytest.nodes.ItemDefinition.make_item`. The values +chosen for a given item arrive on ``item.callspec.params``: + +.. code-block:: python + + # content of conftest.py + from _pytest import nodes + import pytest + + + class YamlItem(pytest.Item): + def __init__(self, *, spec, **kwargs): + super().__init__(**kwargs) + self.spec = spec + + def runtest(self): ... + + def reportinfo(self): + return self.path, 0, self.name + + + class YamlDefinition(nodes.ItemDefinition): + parametrize_argnames = ("value",) + + def make_item(self, parent, *, name, callspec, nodeid, context): + params = dict(callspec.params) if callspec is not None else {} + return YamlItem.from_parent(parent, name=name, nodeid=nodeid, spec=params) + +Any ``pytest_generate_tests`` implementation now applies: + +.. code-block:: python + + def pytest_generate_tests(metafunc): + if "value" in metafunc.fixturenames: + metafunc.parametrize("value", ["good", "bad"]) + +which collects ``test_simple.yaml::hello[good]`` and +``test_simple.yaml::hello[bad]`` under a single ```` node. +Parameter ids, ``pytest.param(...)`` marks and node-id selection all work as +they do for Python tests. + +.. note:: + + This path covers parametrization only: there is no fixture resolution for + non-Python items, so ``indirect=True`` and requesting fixtures from such an + item are not supported. A ``parametrize()`` call naming anything outside + ``parametrize_argnames`` is rejected at collection time. diff --git a/doc/en/how-to/fixtures.rst b/doc/en/how-to/fixtures.rst index 0ffc0778f9f..073c78b405e 100644 --- a/doc/en/how-to/fixtures.rst +++ b/doc/en/how-to/fixtures.rst @@ -389,7 +389,16 @@ to cause a ``smtp_connection`` fixture function, responsible to create a connect once per test *module* (the default is to invoke once per test *function*). Multiple test functions in a test module will thus each receive the same ``smtp_connection`` fixture instance, thus saving time. -Possible values for ``scope`` are: ``function``, ``class``, ``module``, ``package`` or ``session``. +Possible values for ``scope`` are: ``function``, ``definition``, ``class``, +``module``, ``package`` or ``session``. + +.. note:: + + ``definition`` shares one fixture instance across all the invocations + generated from a single test definition -- that is, across the parameter + sets of one parametrized test, but not between two different tests. It + requires the definition to be a node of the collection tree, see + :confval:`collect_function_definition`. The next example puts the fixture function into a separate ``conftest.py`` file so that tests from multiple test modules in the directory can diff --git a/doc/en/proposals/generic_fixture_closures.rst b/doc/en/proposals/generic_fixture_closures.rst new file mode 100644 index 00000000000..e116117334e --- /dev/null +++ b/doc/en/proposals/generic_fixture_closures.rst @@ -0,0 +1,143 @@ +:orphan: + +============================================ +PROPOSAL: Generic fixture closures for items +============================================ + +.. warning:: + + This document is the follow-up half of the ``ItemDefinition`` work: it + outlines what is needed for *non-Python* items to request fixtures. The + parametrization half has shipped, see :ref:`non-python parametrization`. + +What shipped +------------ + +:class:`~_pytest.nodes.ItemDefinition` lets any collector take part in +:hook:`pytest_generate_tests`. A definition declares the names it accepts up +front: + +.. code-block:: python + + class YamlDefinition(nodes.ItemDefinition): + parametrize_argnames = ("value",) + + def make_item(self, parent, *, name, callspec, nodeid, context): + return YamlItem.from_parent( + parent, name=name, nodeid=nodeid, spec=dict(callspec.params) + ) + +and the values chosen for each parameter set arrive on ``item.callspec.params``. + +``ParametrizeContext`` deliberately has no fixture machinery: ``_infer_scope()`` +returns ``Scope.Function``, ``_register_direct_params()`` is a no-op, and +``_validate_argnames()`` checks against ``parametrize_argnames`` rather than a +fixture closure. ``FixtureManager.pytest_generate_tests`` returns early for a +context which is not a ``Metafunc``. + +That covers parametrization. It does **not** cover: + +- ``indirect=True`` -- there is no fixture to hand ``request.param`` to. +- scope inference from fixture scopes -- an unspecified ``scope=`` is always + function scope. +- a non-Python item requesting a fixture at all, including one at the new + ``"definition"`` scope. + +Why the rest is not a small step +-------------------------------- + +Fixture resolution is written against ``Function``, not against ``Item``. The +concrete couplings: + +**TopRequest assumes a Python function item.** +It is constructed as ``TopRequest(pyfuncitem=item)`` and reads +``item._fixtureinfo``, ``item.funcargs``, ``item.fixturenames`` and +``item.obj``. ``_fillfixtures()`` writes into ``item.funcargs``. +``request.instance`` walks to a :class:`~pytest.Class` node and calls +``newinstance()``. + +**getfixtureinfo() is item-scoped and signature-driven.** +Its closure starts from the function's argument names +(``getfuncargnames(func)``) plus autouse names. A non-Python definition has no +signature; its "argnames" are whatever it declares. ``FunctionDefinition`` +already has to ``cast(nodes.Item, self)`` to call it -- the same wart, one level +up. + +**Direct parametrization desugars into DirectParamFixtureDef.** +That is how a direct param reaches the test at setup time. Without a fixture +system there is nothing to desugar into, which is why the generic path delivers +params on the callspec instead. + +**Function.setup() drives everything.** +``nodes.Item.setup()`` is a no-op; the fixture lifecycle only runs because +``Function.setup()`` calls ``self._request._fillfixtures()``. + +Proposed shape +-------------- + +Three steps, each independently useful. + +1. Split a node-level closure entry point out of ``getfixtureinfo()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Today ``getfixtureinfo(node, func, cls, ignore_args)`` derives ``initialnames`` +from ``func``'s signature. Proposed: + +.. code-block:: python + + def getfixtureinfo_for_argnames( + self, node: nodes.Node, argnames: Sequence[str], *, ignore_args=() + ) -> FuncFixtureInfo: ... + +with the current signature-based version becoming a thin wrapper which computes +``argnames`` from the function. This removes the ``cast(nodes.Item, self)`` in +``FunctionDefinition`` as a side effect, and gives ``ItemDefinition`` a closure +without inventing a fake function. + +2. Make the request object item-generic +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Introduce a small protocol for what ``TopRequest`` needs from its item -- +``fixturenames``, somewhere to store the computed values, ``_fixtureinfo``, and +an optional ``instance`` -- and give :class:`~pytest.Item` a default +implementation. :class:`~pytest.Function` keeps its current behaviour +(``funcargs``, bound instance); a generic item gets a plain dict and +``instance is None``. + +The open question here is ``request.instance``, whose meaning is genuinely +Python-specific. Returning ``None`` for non-Python items is the honest answer, +but fixtures in the wild do use ``request.instance`` unconditionally. + +3. Opt in per definition +~~~~~~~~~~~~~~~~~~~~~~~~ + +Fixture support should not be forced on every ``ItemDefinition``; a collector +which only wants parametrization should not pay for closure computation. +Suggested switch: + +.. code-block:: python + + class ItemDefinition(Collector, abc.ABC): + #: Whether items generated from this definition can request fixtures. + supports_fixtures: bool = False + +With ``supports_fixtures = True``, ``make_parametrize_context()`` computes a +closure from ``parametrize_argnames`` via step 1 and returns a fixture-aware +context, so ``indirect=True``, scope inference and ``"definition"`` scoped +fixtures all work uniformly. ``FixtureManager.pytest_generate_tests`` would then +key off the context being fixture-aware rather than off +``isinstance(metafunc, Metafunc)``. + +Open questions +-------------- + +- Does ``Metafunc`` stay a distinct class once the generic context is + fixture-aware, or does it collapse into ``ParametrizeContext`` plus the + ``module``/``cls``/``function`` attributes? +- ``request.instance`` for non-Python items: ``None``, or an error on access? +- Autouse fixtures currently apply to every item in a subtree. Should they apply + to non-Python items which opted into fixtures, or only to those which declare + the corresponding names? +- Should ``parametrize_argnames`` and the fixture closure be the same list, or + should a definition be able to accept a name for parametrization which is not + requestable as a fixture? diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index b7235d58a99..a3fc26c8ffe 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -937,6 +937,13 @@ Function :members: :show-inheritance: +ItemDefinition +~~~~~~~~~~~~~~ + +.. autoclass:: _pytest.nodes.ItemDefinition() + :members: + :show-inheritance: + FunctionDefinition ~~~~~~~~~~~~~~~~~~ @@ -963,6 +970,12 @@ CallInfo .. autoclass:: pytest.CallInfo() :members: +CallSpec +~~~~~~~~ + +.. autoclass:: _pytest.parametrize.CallSpec() + :members: + CollectReport ~~~~~~~~~~~~~ @@ -1036,6 +1049,13 @@ Metafunc .. autoclass:: pytest.Metafunc() :members: + :inherited-members: + +ParametrizeContext +~~~~~~~~~~~~~~~~~~ + +.. autoclass:: _pytest.parametrize.ParametrizeContext() + :members: Parser ~~~~~~ diff --git a/src/_pytest/fixtures.py b/src/_pytest/fixtures.py index 1b1591d1f83..ff0aac0e416 100644 --- a/src/_pytest/fixtures.py +++ b/src/_pytest/fixtures.py @@ -25,7 +25,6 @@ from typing import Final from typing import final from typing import Generic -from typing import Literal from typing import NoReturn from typing import overload from typing import TYPE_CHECKING @@ -70,6 +69,7 @@ from _pytest.outcomes import fail from _pytest.outcomes import skip from _pytest.outcomes import TEST_OUTCOME +from _pytest.parametrize import _resolve_args_directness from _pytest.pathlib import absolutepath from _pytest.pathlib import bestrelpath from _pytest.scope import HIGH_SCOPES @@ -84,9 +84,9 @@ if TYPE_CHECKING: - from _pytest.python import CallSpec + from _pytest.parametrize import CallSpec + from _pytest.parametrize import ParametrizeContext from _pytest.python import Function - from _pytest.python import Metafunc from _pytest.reports import CollectReport @@ -136,6 +136,23 @@ def get_scope_package( return node.session +def fail_definition_scope_unavailable(nodeid: str, what: str) -> NoReturn: + """Fail because the ``"definition"`` scope was used where no + :class:`~_pytest.nodes.ItemDefinition` node exists to anchor it on. + + Unlike the other scopes, ``"definition"`` needs the test definition to be a + node of the collection tree; there is nothing to fall back to that would + preserve its meaning of "shared by all invocations of one definition". + """ + fail( + f"ScopeUnavailable: {what} needs the 'definition' scope, but {nodeid} " + f"has no definition node in the collection tree.\n" + f"For Python tests, set the collect_function_definition ini option to " + f"'pedantic' to make function definitions part of the tree.", + pytrace=False, + ) + + def is_visibility_more_specific( candidate: FixtureDef[Any], other: FixtureDef[Any] ) -> bool: @@ -174,6 +191,9 @@ def get_scope_node(node: nodes.Node, scope: Scope) -> nodes.Node | None: # Type ignored because this is actually safe, see: # https://github.com/python/mypy/issues/4717 return node.getparent(nodes.Item) # type: ignore[type-abstract] + elif scope is Scope.Definition: + # Type ignored for the same reason as Scope.Function above. + return node.getparent(nodes.ItemDefinition) # type: ignore[type-abstract] elif scope is Scope.Class: return node.getparent(_pytest.python.Class) elif scope is Scope.Module: @@ -277,6 +297,9 @@ class ParamArgKey: scoped_item_path: Path | None #: For Class scope, the class where the item is defined. item_cls: type | None + #: For Definition scope, the nodeid of the definition the item was + #: generated from. + scoped_definition_id: str | None = None _V = TypeVar("_V") @@ -293,6 +316,7 @@ def get_param_argkeys(item: nodes.Item, scope: Scope) -> Iterator[ParamArgKey]: return item_cls = None + scoped_definition_id = None if scope is Scope.Session: scoped_item_path = None elif scope is Scope.Package: @@ -302,15 +326,28 @@ def get_param_argkeys(item: nodes.Item, scope: Scope) -> Iterator[ParamArgKey]: scoped_item_path = item.path elif scope is Scope.Class: scoped_item_path = item.path - item_cls = item.cls # type: ignore[attr-defined] + # Items which are not Python test functions have no class. + item_cls = getattr(item, "cls", None) + elif scope is Scope.Definition: + scoped_item_path = item.path else: assert_never(scope) for argname, param in callspec.params.items(): if callspec._arg2scope[argname] != scope: continue + if scope is Scope.Definition and scoped_definition_id is None: + # Resolved only once a definition-scoped param is actually found: + # items without one need not have a definition node at all. A param + # at this scope can only have been created while the definition was + # a node, so the lookup cannot come up empty here. + definition = get_scope_node(item, Scope.Definition) + assert definition is not None + scoped_definition_id = definition.nodeid param_key = ParamValueKey(param, callspec.indices[argname]) - yield ParamArgKey(argname, param_key, scoped_item_path, item_cls) + yield ParamArgKey( + argname, param_key, scoped_item_path, item_cls, scoped_definition_id + ) def reorder_items(items: Sequence[nodes.Item]) -> list[nodes.Item]: @@ -546,7 +583,8 @@ def _scope(self) -> Scope: @property def scope(self) -> ScopeName: - """Scope string, one of "function", "class", "module", "package", "session".""" + """Scope string, one of "function", "definition", "class", "module", + "package", "session".""" return self._scope.value @abc.abstractmethod @@ -908,6 +946,10 @@ def __init__( if node is None and scope is Scope.Class: # Fallback to function item itself. node = self._pyfuncitem + if node is None and scope is Scope.Definition: + fail_definition_scope_unavailable( + self._pyfuncitem.nodeid, f"fixture {fixturedef.argname!r}" + ) assert node, ( f'Could not obtain a node for scope "{scope}" for function {self._pyfuncitem!r}' ) @@ -1193,7 +1235,8 @@ def __init__( @property def scope(self) -> ScopeName: - """Scope string, one of "function", "class", "module", "package", "session".""" + """Scope string, one of "function", "definition", "class", "module", + "package", "session".""" return self._scope.value @property @@ -1556,7 +1599,13 @@ def fixture( :param scope: The scope for which this fixture is shared; one of ``"function"`` - (default), ``"class"``, ``"module"``, ``"package"`` or ``"session"``. + (default), ``"definition"``, ``"class"``, ``"module"``, ``"package"`` + or ``"session"``. + + ``"definition"`` shares the fixture across all invocations generated + from one test definition, i.e. across the parameter sets of a + parametrized test. It requires the definition to be a node of the + collection tree, see :confval:`collect_function_definition`. This parameter may also be a callable which receives ``(fixture_name, config)`` as parameters, and must return a ``str`` with one of the values mentioned above. @@ -1683,45 +1732,6 @@ def pytest_cmdline_main(config: Config) -> int | ExitCode | None: return None -def _resolve_args_directness( - argnames: Sequence[str], - indirect: bool | Sequence[str], - nodeid: str, -) -> dict[str, Literal["indirect", "direct"]]: - """Resolve if each parametrized argument must be considered an indirect - parameter to a fixture of the same name, or a direct parameter to the - parametrized function, based on the ``indirect`` parameter of the - parametrize() call. - - :param argnames: - List of argument names passed to ``parametrize()``. - :param indirect: - Same as the ``indirect`` parameter of ``parametrize()``. - :param nodeid: - Node ID to which the parametrization is applied. - :returns: - A dict mapping each arg name to either "indirect" or "direct". - """ - arg_directness: dict[str, Literal["indirect", "direct"]] - if isinstance(indirect, bool): - arg_directness = dict.fromkeys(argnames, "indirect" if indirect else "direct") - elif isinstance(indirect, Sequence): - arg_directness = dict.fromkeys(argnames, "direct") - for arg in indirect: - if arg not in argnames: - fail( - f"In {nodeid}: indirect fixture '{arg}' doesn't exist", - pytrace=False, - ) - arg_directness[arg] = "indirect" - else: - fail( - f"In {nodeid}: expected Sequence or boolean for indirect, got {type(indirect).__name__}", - pytrace=False, - ) - return arg_directness - - def _get_direct_parametrize_args(node: nodes.Node) -> set[str]: """Return all direct parametrization arguments of a node, so we don't mistake them for fixtures. @@ -1995,8 +2005,14 @@ def sort_by_scope(arg_name: str) -> Scope: return fixturenames_closure, arg2fixturedefs - def pytest_generate_tests(self, metafunc: Metafunc) -> None: + def pytest_generate_tests(self, metafunc: ParametrizeContext) -> None: """Generate new tests based on parametrized fixtures used by the given metafunc""" + from _pytest.python import Metafunc + + if not isinstance(metafunc, Metafunc): + # The definition's parametrization is not backed by fixtures, so + # there are no parametrized fixtures to expand. + return def get_parametrize_mark_argnames(mark: Mark) -> Sequence[str]: args, _ = ParameterSet._parse_parametrize_args(*mark.args, **mark.kwargs) diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 364f27b7d2d..08218f07ecc 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -1018,6 +1018,7 @@ def collect(self) -> Iterator[nodes.Item | nodes.Collector]: # Prune this level. any_matched_in_collector = False for node in reversed(subnodes): + remaining = matchparts[1:] # Path part e.g. `/a/b/` in `/a/b/test_file.py::TestIt::test_it`. if isinstance(matchparts[0], Path): is_match = node.path == matchparts[0] @@ -1032,7 +1033,13 @@ def collect(self) -> Iterator[nodes.Item | nodes.Collector]: else: if len(matchparts) == 1: # This the last part, one parametrization goes. - if parametrization is not None: + if isinstance(node, nodes.ItemDefinition): + # A definition node carries the bare name, while + # the parametrization is on the items below it. + # Descend and match the same part again there. + is_match = node.name == matchparts[0] + remaining = matchparts + elif parametrization is not None: # A parametrized arg must match exactly. is_match = node.name == matchparts[0] + parametrization else: @@ -1043,7 +1050,7 @@ def collect(self) -> Iterator[nodes.Item | nodes.Collector]: else: is_match = node.name == matchparts[0] if is_match: - work.append((node, matchparts[1:])) + work.append((node, remaining)) any_matched_in_collector = True if not any_matched_in_collector: diff --git a/src/_pytest/nodes.py b/src/_pytest/nodes.py index f0629c2daf7..49e1df379e4 100644 --- a/src/_pytest/nodes.py +++ b/src/_pytest/nodes.py @@ -6,6 +6,7 @@ from collections.abc import Iterable from collections.abc import Iterator from collections.abc import MutableMapping +from collections.abc import Sequence from functools import cached_property from functools import lru_cache import os @@ -35,6 +36,8 @@ from _pytest.mark.structures import MarkDecorator from _pytest.mark.structures import NodeKeywords from _pytest.outcomes import fail +from _pytest.parametrize import CallSpec +from _pytest.parametrize import ParametrizeContext from _pytest.pathlib import absolutepath from _pytest.stash import Stash from _pytest.warning_types import PytestWarning @@ -762,3 +765,138 @@ def location(self) -> tuple[str, int | None, str]: relfspath = self.session._node_location_to_relpath(path) assert type(location[2]) is str return (relfspath, location[1], location[2]) + + +class ItemDefinition(Collector, abc.ABC): + """Base class for collectors standing for a single test *definition*. + + Its children are the (possibly parametrized) items generated from that one + definition. Giving the definition a node of its own makes it addressable: + it owns the definition's markers, it is what :hook:`pytest_generate_tests` + parametrizes, and it is the node ``"definition"`` scoped fixtures are + cached on, shared by every invocation of the definition. + + :class:`~_pytest.python.FunctionDefinition` is the implementation for + Python test functions. Other collectors can subclass this to let their test + definitions be parametrized through the same hook: declare the names that + may be parametrized in :attr:`parametrize_argnames`, and turn one callspec + into an item in :meth:`make_item`. There is no fixture resolution on that + path -- the chosen values arrive on ``item.callspec.params``. + + .. versionadded:: 9.2 + + :ref:`non-python tests`. + """ + + #: The argument names this definition may be parametrized on. + #: + #: A ``parametrize()`` call naming anything else is rejected, mirroring the + #: "uses no argument" error raised for Python test functions. + parametrize_argnames: Sequence[str] = () + + @abc.abstractmethod + def make_item( + self, + parent: Collector, + *, + name: str, + callspec: CallSpec | None, + nodeid: str, + context: ParametrizeContext, + ) -> Item: + """Build the item for a single invocation of this definition. + + :param parent: + The collector to attach the item to. Usually the definition itself, + but not when the definition is kept out of the collection tree (see + :confval:`collect_function_definition`). + :param name: + Name for the item: the definition's name, with the callspec id + appended when parametrized. + :param callspec: + The parametrization of this invocation, or ``None`` when the + definition is not parametrized. + :param nodeid: + The nodeid to give the item. Passed explicitly so that item nodeids + stay the same whether or not the definition is part of the tree. + :param context: + The context the parametrization was collected in. + """ + raise NotImplementedError + + def make_parametrize_context(self) -> ParametrizeContext: + """Create the object to hand to :hook:`pytest_generate_tests`.""" + return ParametrizeContext( + self, + self.config, + argnames=self.parametrize_argnames, + _ispytest=True, + ) + + def parametrize_hook_extras(self) -> Sequence[Callable[..., object]]: + """Extra :hook:`pytest_generate_tests` implementations to call. + + Used by the Python collector to pick up ``pytest_generate_tests`` + functions defined in the test's own module or class. + """ + return () + + def finalize_parametrization(self, context: ParametrizeContext) -> None: + """Called once the hook has run, before any item is built. + + Only called when the definition ended up parametrized. + """ + context._recompute_direct_params_indices() + + def collect(self) -> Iterable[Item | Collector]: + return list(self.generate_items(self)) + + def generate_items(self, parent: Collector) -> Iterator[Item]: + """Run the generate-tests protocol, yielding the items under ``parent``. + + ``parent`` is the definition itself when it is part of the collection + tree, and the collector containing it otherwise. + """ + context = self.make_parametrize_context() + self.ihook.pytest_generate_tests.call_extra( + list(self.parametrize_hook_extras()), dict(metafunc=context) + ) + + # The items keep a flat nodeid anchored at the collector containing the + # definition, whether or not the definition itself is part of the tree, + # so that nodeids -- and with them selection, caching and reporting -- + # do not depend on the shape of the tree. + assert self.parent is not None + base_nodeid = self.parent.nodeid + name = self.name + + if not context._calls: + yield self.make_item( + parent, + name=name, + callspec=None, + nodeid=f"{base_nodeid}::{name}", + context=context, + ) + else: + self.finalize_parametrization(context) + for callspec in context._calls: + subname = f"{name}[{callspec.id}]" if callspec._idlist else name + item = self.make_item( + parent, + name=subname, + callspec=callspec, + nodeid=f"{base_nodeid}::{subname}", + context=context, + ) + if getattr(item, "callspec", None) is None: + # Attaching the parametrization is part of the protocol, not + # of each make_item(): the callspec drives reordering and + # high-scoped param caching, and its marks and id have to + # reach the item for markers and -k to work. Skipped when + # make_item() already did it -- Function does, because it + # can also be constructed directly. + item.callspec = callspec # type: ignore[attr-defined] + item.own_markers.extend(callspec.marks) + item.keywords[callspec.id] = True + yield item diff --git a/src/_pytest/parametrize.py b/src/_pytest/parametrize.py new file mode 100644 index 00000000000..09cbbe58b6e --- /dev/null +++ b/src/_pytest/parametrize.py @@ -0,0 +1,809 @@ +"""The parametrization core: turning ``parametrize()`` calls into callspecs. + +This machinery is deliberately independent of the Python test collector, so +that any :class:`~_pytest.nodes.ItemDefinition` -- not only +:class:`~_pytest.python.FunctionDefinition` -- can be parametrized through +:hook:`pytest_generate_tests`. + +:class:`~_pytest.python.Metafunc` is the Python-specific subclass of +:class:`ParametrizeContext` and remains the object plugins receive for Python +tests. ``CallSpec``, ``IdMaker`` and friends live here rather than in +``python.py`` for the same reason, and are re-exported from there. +""" + +from __future__ import annotations + +from collections import Counter +from collections import defaultdict +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Mapping +from collections.abc import Sequence +import dataclasses +import enum +import hashlib +import itertools +import re +import textwrap +from typing import Any +from typing import cast +from typing import final +from typing import get_args +from typing import Literal +from typing import NoReturn +from typing import TYPE_CHECKING + +from _pytest._io.saferepr import saferepr +from _pytest.compat import ascii_escaped +from _pytest.compat import NOTSET +from _pytest.config import Config +from _pytest.config import UsageError +from _pytest.deprecated import check_ispytest +from _pytest.mark.structures import _HiddenParam +from _pytest.mark.structures import HIDDEN_PARAM +from _pytest.mark.structures import Mark +from _pytest.mark.structures import MarkDecorator +from _pytest.mark.structures import normalize_mark_list +from _pytest.mark.structures import ParameterSet +from _pytest.outcomes import fail +from _pytest.scope import Scope +from _pytest.scope import ScopeName + + +if TYPE_CHECKING: + from _pytest import nodes + from _pytest.fixtures import FixtureDef + + +LongStrIdStrategy = Literal["short", "sha256", "legacy", "disallow"] +_LONG_STR_STRATEGIES: frozenset[LongStrIdStrategy] = frozenset( + get_args(LongStrIdStrategy) +) + + +def _collect_error(msg: str) -> Exception: + """Build a ``Collector.CollectError``. + + ``nodes`` imports this module (for :class:`~_pytest.nodes.ItemDefinition`), + so it can only be imported here lazily. + """ + from _pytest.nodes import Collector + + return Collector.CollectError(msg) + + +@final +@dataclasses.dataclass(frozen=True) +class IdMaker: + """Make IDs for a parametrization.""" + + __slots__ = ( + "argnames", + "config", + "idfn", + "ids", + "nodeid", + "parametersets", + ) + + # The argnames of the parametrization. + argnames: Sequence[str] + # The ParameterSets of the parametrization. + parametersets: Sequence[ParameterSet] + # Optionally, a user-provided callable to make IDs for parameters in a + # ParameterSet. + idfn: Callable[[Any], object | None] | None + # Optionally, explicit IDs for ParameterSets by index. + ids: Sequence[object | None] | None + # Optionally, the pytest config. + # Used for controlling ASCII escaping, determining parametrization ID + # strictness, and for calling the :hook:`pytest_make_parametrize_id` hook. + config: Config | None + # Optionally, the ID of the node being parametrized. + # Used only for clearer error messages. + nodeid: str | None + + def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]: + """Make a unique identifier for each ParameterSet, that may be used to + identify the parametrization in a node ID. + + If strict_parametrization_ids is enabled, and duplicates are detected, + raises CollectError. Otherwise makes the IDs unique as follows: + + Format is -...-[counter], where prm_x_token is + - user-provided id, if given + - else an id derived from the value, applicable for certain types + - else + The counter suffix is appended only in case a string wouldn't be unique + otherwise. + """ + resolved_ids = list(self._resolve_ids()) + # All IDs must be unique! + if len(resolved_ids) != len(set(resolved_ids)): + # Record the number of occurrences of each ID. + id_counts = Counter(resolved_ids) + + if self._strict_parametrization_ids_enabled(): + parameters = ", ".join(self.argnames) + parametersets = ", ".join( + [saferepr(list(param.values)) for param in self.parametersets] + ) + ids = ", ".join( + id if id is not HIDDEN_PARAM else "" for id in resolved_ids + ) + duplicates = ", ".join( + id if id is not HIDDEN_PARAM else "" + for id, count in id_counts.items() + if count > 1 + ) + msg = textwrap.dedent(f""" + Duplicate parametrization IDs detected, but strict_parametrization_ids is set. + + Test name: {self.nodeid} + Parameters: {parameters} + Parameter sets: {parametersets} + IDs: {ids} + Duplicates: {duplicates} + + You can fix this problem using `@pytest.mark.parametrize(..., ids=...)` or `pytest.param(..., id=...)`. + """).strip() # noqa: E501 + raise _collect_error(msg) + + # Map the ID to its next suffix. + id_suffixes: dict[str, int] = defaultdict(int) + # Suffix non-unique IDs to make them unique. + for index, id in enumerate(resolved_ids): + if id_counts[id] > 1: + if id is HIDDEN_PARAM: + self._complain_multiple_hidden_parameter_sets() + suffix = "" + if id and id[-1].isdigit(): + suffix = "_" + new_id = f"{id}{suffix}{id_suffixes[id]}" + while new_id in set(resolved_ids): + id_suffixes[id] += 1 + new_id = f"{id}{suffix}{id_suffixes[id]}" + resolved_ids[index] = new_id + id_suffixes[id] += 1 + assert len(resolved_ids) == len(set(resolved_ids)), ( + f"Internal error: {resolved_ids=}" + ) + return resolved_ids + + def _strict_parametrization_ids_enabled(self) -> bool: + if self.config is None: + return False + strict_parametrization_ids = self.config.getini("strict_parametrization_ids") + if strict_parametrization_ids is None: + strict_parametrization_ids = self.config.getini("strict") + return cast(bool, strict_parametrization_ids) + + def _resolve_ids(self) -> Iterable[str | _HiddenParam]: + """Resolve IDs for all ParameterSets (may contain duplicates).""" + for idx, parameterset in enumerate(self.parametersets): + if parameterset.id is not None: + # ID provided directly - pytest.param(..., id="...") + if parameterset.id is HIDDEN_PARAM: + yield HIDDEN_PARAM + else: + yield _ascii_escaped_by_config(parameterset.id, self.config) + elif self.ids and idx < len(self.ids) and self.ids[idx] is not None: + # ID provided in the IDs list - parametrize(..., ids=[...]). + if self.ids[idx] is HIDDEN_PARAM: + yield HIDDEN_PARAM + else: + yield self._idval_from_value_required(self.ids[idx], idx) + else: + # ID not provided - generate it. + yield "-".join( + self._idval(val, argname, idx) + for val, argname in zip( + parameterset.values, self.argnames, strict=True + ) + ) + + def _idval(self, val: object, argname: str, idx: int) -> str: + """Make an ID for a parameter in a ParameterSet.""" + idval = self._idval_from_function(val, argname, idx) + if idval is not None: + return idval + idval = self._idval_from_hook(val, argname) + if idval is not None: + return idval + if isinstance(val, str | bytes): + idval = self._apply_long_str_strategy(val, argname, idx) + if idval is not None: + return idval + else: + idval = self._idval_from_value(val) + if idval is not None: + return idval + return self._idval_from_argname(argname, idx) + + def _get_long_str_strategy(self) -> LongStrIdStrategy: + if not self.config: + return "short" + value = self.config.getini("parametrize_long_str_id_strategy") + if value not in _LONG_STR_STRATEGIES: + raise UsageError( + f"Unknown parametrize_long_str_id_strategy: {value!r}. " + f"Valid values: {', '.join(sorted(_LONG_STR_STRATEGIES))}" + ) + return cast(LongStrIdStrategy, value) + + def _apply_long_str_strategy( + self, val: str | bytes, argname: str, idx: int + ) -> str | None: + """Apply the configured strategy for long str/bytes parameter values. + + Only used for auto-generated IDs (not explicit ids=[...] or + pytest.param(id=...)). + """ + if len(val) <= 100: + return _ascii_escaped_by_config(val, self.config) + match self._get_long_str_strategy(): + case "legacy": + return _ascii_escaped_by_config(val, self.config) + case "short": + return None + case "sha256": + encoded = val.encode("utf-8") if isinstance(val, str) else val + return hashlib.sha256(encoded).hexdigest() + case "disallow": # pragma: no branch -- fail() raises, confuses coverage + prefix = self._make_error_prefix() + fail( + f"{prefix}parametrize value for '{argname}' at index {idx} " + f"is too long for an auto-generated ID ({len(val)} characters). " + f"Use pytest.param(..., id=...) or parametrize(..., ids=...) " + f"to set an explicit ID, or change parametrize_long_str_id_strategy.", + pytrace=False, + ) + + def _idval_from_function(self, val: object, argname: str, idx: int) -> str | None: + """Try to make an ID for a parameter in a ParameterSet using the + user-provided id callable, if given.""" + if self.idfn is None: + return None + try: + id = self.idfn(val) + except Exception as e: + prefix = f"{self.nodeid}: " if self.nodeid is not None else "" + msg = "error raised while trying to determine id of parameter '{}' at position {}" + msg = prefix + msg.format(argname, idx) + raise ValueError(msg) from e + if id is None: + return None + return self._idval_from_value(id) + + def _idval_from_hook(self, val: object, argname: str) -> str | None: + """Try to make an ID for a parameter in a ParameterSet by calling the + :hook:`pytest_make_parametrize_id` hook.""" + if self.config: + id: str | None = self.config.hook.pytest_make_parametrize_id( + config=self.config, val=val, argname=argname + ) + return id + return None + + def _idval_from_value(self, val: object) -> str | None: + """Try to make an ID for a parameter in a ParameterSet from its value, + if the value type is supported.""" + if isinstance(val, str | bytes): + return _ascii_escaped_by_config(val, self.config) + elif val is None or isinstance(val, float | int | bool | complex): + return str(val) + elif isinstance(val, re.Pattern): + return ascii_escaped(val.pattern) + elif val is NOTSET: + # Fallback to default. Note that NOTSET is an enum.Enum. + pass + elif isinstance(val, enum.Enum): + return str(val) + elif isinstance(getattr(val, "__name__", None), str): + # Name of a class, function, module, etc. + name: str = getattr(val, "__name__") + return name + return None + + def _idval_from_value_required(self, val: object, idx: int) -> str: + """Like _idval_from_value(), but fails if the type is not supported.""" + id = self._idval_from_value(val) + if id is not None: + return id + + # Fail. + prefix = self._make_error_prefix() + msg = ( + f"{prefix}ids contains unsupported value {saferepr(val)} (type: {type(val)!r}) at index {idx}. " + "Supported types are: str, bytes, int, float, complex, bool, enum, regex or anything with a __name__." + ) + fail(msg, pytrace=False) + + @staticmethod + def _idval_from_argname(argname: str, idx: int) -> str: + """Make an ID for a parameter in a ParameterSet from the argument name + and the index of the ParameterSet.""" + return str(argname) + str(idx) + + def _complain_multiple_hidden_parameter_sets(self) -> NoReturn: + fail( + f"{self._make_error_prefix()}multiple instances of HIDDEN_PARAM " + "cannot be used in the same parametrize call, " + "because the tests names need to be unique." + ) + + def _make_error_prefix(self) -> str: + if self.nodeid is not None: + return f"In {self.nodeid}: " + else: + return "" + + +@final +@dataclasses.dataclass(frozen=True) +class CallSpec: + """A planned parameterized invocation of a test function. + + Calculated during collection for a given test function's Metafunc. + Once collection is over, each callspec is turned into a single Item + and stored in item.callspec. + """ + + # arg name -> arg value which will be passed to a fixture of the same name. + params: dict[str, object] = dataclasses.field(default_factory=dict) + # arg name -> arg index. + indices: dict[str, int] = dataclasses.field(default_factory=dict) + # arg name -> parameter scope. + # Used for sorting parametrized resources. + _arg2scope: Mapping[str, Scope] = dataclasses.field(default_factory=dict) + # Parts which will be added to the item's name in `[..]` separated by "-". + _idlist: Sequence[str] = dataclasses.field(default_factory=tuple) + # Marks which will be applied to the item. + marks: list[Mark] = dataclasses.field(default_factory=list) + + def setmulti( + self, + *, + argnames: Iterable[str], + valset: Iterable[object], + id: str | _HiddenParam, + marks: Iterable[Mark | MarkDecorator], + scope: Scope, + param_index: int, + nodeid: str, + ) -> CallSpec: + params = self.params.copy() + indices = self.indices.copy() + arg2scope = dict(self._arg2scope) + for arg, val in zip(argnames, valset, strict=True): + if arg in params: + raise _collect_error(f"{nodeid}: duplicate parametrization of {arg!r}") + params[arg] = val + indices[arg] = param_index + arg2scope[arg] = scope + return CallSpec( + params=params, + indices=indices, + _arg2scope=arg2scope, + _idlist=self._idlist if id is HIDDEN_PARAM else [*self._idlist, id], + marks=[*self.marks, *normalize_mark_list(marks)], + ) + + def getparam(self, name: str) -> object: + try: + return self.params[name] + except KeyError as e: + raise ValueError(name) from e + + @property + def id(self) -> str: + return "-".join(self._idlist) + + +def _ascii_escaped_by_config(val: str | bytes, config: Config | None) -> str: + if config is None: + escape_option = False + else: + escape_option = config.getini( + "disable_test_id_escaping_and_forfeit_all_rights_to_community_support" + ) + # TODO: If escaping is turned off and the user passes bytes, + # will return a bytes. For now we ignore this but the + # code *probably* doesn't handle this case. + return val if escape_option else ascii_escaped(val) # type: ignore + + +def _infer_parametrize_scope( + argnames: Sequence[str], + arg2fixturedefs: Mapping[str, Sequence[FixtureDef[object]]], + indirect: bool | Sequence[str], +) -> Scope: + """Infer the most appropriate scope for a parametrize() call based on its + arguments, for when the scope is not explicitly specified. + + When there's at least one direct argument, always use "function" scope. + + When a test function is parametrized and all its arguments are indirect + (e.g. fixtures), return the most narrow scope based on the fixtures used. + + Related to issue #1832, based on code posted by @Kingdread. + """ + if isinstance(indirect, Sequence): + all_arguments_are_fixtures = len(indirect) == len(argnames) + else: + all_arguments_are_fixtures = bool(indirect) + + if all_arguments_are_fixtures: + # Takes the most narrow scope from used fixtures. + used_scopes = ( + # Higher scope can't request lower scope, so it's OK to only + # look at the first fixturedef in the override chain. + arg2fixturedefs[argname][-1]._scope + for argname in argnames + if argname in arg2fixturedefs + ) + return min(used_scopes, default=Scope.Function) + + return Scope.Function + + +def _resolve_args_directness( + argnames: Sequence[str], + indirect: bool | Sequence[str], + nodeid: str, +) -> dict[str, Literal["indirect", "direct"]]: + """Resolve if each parametrized argument must be considered an indirect + parameter to a fixture of the same name, or a direct parameter to the + parametrized function, based on the ``indirect`` parameter of the + parametrize() call. + + :param argnames: + List of argument names passed to ``parametrize()``. + :param indirect: + Same as the ``indirect`` parameter of ``parametrize()``. + :param nodeid: + Node ID to which the parametrization is applied. + :returns: + A dict mapping each arg name to either "indirect" or "direct". + """ + arg_directness: dict[str, Literal["indirect", "direct"]] + if isinstance(indirect, bool): + arg_directness = dict.fromkeys(argnames, "indirect" if indirect else "direct") + elif isinstance(indirect, Sequence): + arg_directness = dict.fromkeys(argnames, "direct") + for arg in indirect: + if arg not in argnames: + fail( + f"In {nodeid}: indirect fixture '{arg}' doesn't exist", + pytrace=False, + ) + arg_directness[arg] = "indirect" + else: + fail( + f"In {nodeid}: expected Sequence or boolean for indirect, got {type(indirect).__name__}", + pytrace=False, + ) + return arg_directness + + +class ParametrizeContext: + """The object passed to :hook:`pytest_generate_tests`. + + Accumulates :meth:`parametrize` calls into a list of :class:`CallSpec` + objects, which the owning :class:`~_pytest.nodes.ItemDefinition` then turns + into items. + + For Python tests this is subclassed by :class:`pytest.Metafunc`, which adds + the fixture-aware behaviour: scope inference from fixture scopes, indirect + parametrization and validation against the test function's signature. + + A non-Python definition can use this class as-is. It declares the names it + accepts up front (``argnames``), and the values chosen for them are handed + to the generated items on ``item.callspec.params``; there is no fixture + resolution involved. + """ + + def __init__( + self, + definition: nodes.ItemDefinition, + config: Config, + *, + argnames: Sequence[str] = (), + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + + #: The definition being parametrized. + self.definition = definition + + #: Access to the :class:`pytest.Config` object for the test session. + self.config = config + + #: The names this definition can be parametrized on. + #: + #: Named ``fixturenames`` rather than ``argnames`` because that is what + #: :hook:`pytest_generate_tests` implementations in the wild inspect -- + #: keeping the name is what lets them work with non-Python definitions. + #: + #: Stored by reference, so that a caller which later narrows the + #: sequence in place (as the Python fixture closure does) is reflected + #: here. + self.fixturenames: Sequence[str] = argnames + + # Result of parametrize(). + self._calls: list[CallSpec] = [] + + self._params_directness: dict[str, Literal["indirect", "direct"]] = {} + + # Seams for subclasses. The Python implementation (Metafunc) overrides + # these to bring fixtures and the test function's signature into play. + + @property + def _introspection_target(self) -> object: + """The object to introspect for a signature and a source location. + + Used when reporting an empty parameter set. Defaults to the definition + node, for which the source location degrades to "unknown"; Metafunc + returns the test function. + """ + return self.definition + + @property + def _target_name(self) -> str: + """Human readable name of what is being parametrized.""" + return self.definition.name + + def _infer_scope( + self, argnames: Sequence[str], indirect: bool | Sequence[str] + ) -> Scope: + """Determine the scope of a parametrize() call that did not specify one.""" + return Scope.Function + + def _validate_argnames( + self, argnames: Sequence[str], indirect: bool | Sequence[str] + ) -> None: + """Check that the definition actually accepts all of ``argnames``.""" + for argname in argnames: + if argname not in self.fixturenames: + kind = "fixture" if indirect else "argument" + fail( + f"In {self.definition.nodeid}: definition uses no {kind} '{argname}'", + pytrace=False, + ) + + def _register_direct_params( + self, + argnames: Sequence[str], + arg_directness: Mapping[str, Literal["indirect", "direct"]], + scope: Scope, + ) -> None: + """Hook for making the chosen params reachable at setup time. + + Only meaningful where parametrization feeds a fixture system; the + generic path delivers params on ``item.callspec.params`` instead. + """ + + def parametrize( + self, + argnames: str | Sequence[str], + argvalues: Iterable[ParameterSet | Sequence[object] | object], + *, + indirect: bool | Sequence[str] = False, + ids: Iterable[object | None] | Callable[[Any], object | None] | None = None, + scope: ScopeName | None = None, + _param_mark: Mark | None = None, + ) -> None: + """Add new invocations to the underlying test function using the list + of argvalues for the given argnames. Parametrization is performed + during the collection phase. If you need to setup expensive resources + see about setting ``indirect`` to do it at test setup time instead. + + Can be called multiple times per test function (but only on different + argument names), in which case each call parametrizes all previous + parametrizations, e.g. + + :: + + unparametrized: t + parametrize ["x", "y"]: t[x], t[y] + parametrize [1, 2]: t[x-1], t[x-2], t[y-1], t[y-2] + + :param argnames: + A comma-separated string denoting one or more argument names, or + a list/tuple of argument strings. + + :param argvalues: + The list of argvalues determines how often a test is invoked with + different argument values. + + If only one argname was specified argvalues is a list of values. + If N argnames were specified, argvalues must be a list of + N-tuples, where each tuple-element specifies a value for its + respective argname. + + .. versionchanged:: 9.1 + + Passing a non-:class:`~collections.abc.Collection` iterable + (such as a generator or iterator) is deprecated. See + :ref:`parametrize-iterators` for details. + + :param indirect: + A list of arguments' names (subset of argnames) or a boolean. + If True the list contains all names from the argnames. Each + argvalue corresponding to an argname in this list will + be passed as request.param to its respective argname fixture + function so that it can perform more expensive setups during the + setup phase of a test rather than at collection time. + + :param ids: + Sequence of (or generator for) ids for ``argvalues``, + or a callable to return part of the id for each argvalue. + + With sequences (and generators like ``itertools.count()``) the + returned ids should be of type ``string``, ``int``, ``float``, + ``bool``, or ``None``. + They are mapped to the corresponding index in ``argvalues``. + ``None`` means to use the auto-generated id. + + .. versionadded:: 8.4 + :ref:`hidden-param` means to hide the parameter set + from the test name. Can only be used at most 1 time, as + test names need to be unique. + + If it is a callable it will be called for each entry in + ``argvalues``, and the return value is used as part of the + auto-generated id for the whole set (where parts are joined with + dashes ("-")). + This is useful to provide more specific ids for certain items, e.g. + dates. Returning ``None`` will use an auto-generated id. + + If no ids are provided they will be generated automatically from + the argvalues. + + :param scope: + If specified it denotes the scope of the parameters. + The scope is used for grouping tests by parameter instances. + It will also override any fixture-function defined scope, allowing + to set a dynamic scope using test context or configuration. + + .. versionchanged:: 9.1 + + ``indirect``, ``ids`` and ``scope`` are now keyword-only. + """ + nodeid = self.definition.nodeid + + argnames, parametersets = ParameterSet._for_parametrize( + argnames, + argvalues, + self._introspection_target, + self.config, + nodeid=nodeid, + ) + del argvalues + + if "request" in argnames: + fail( + f"{nodeid}: 'request' is a reserved name and cannot be used in @pytest.mark.parametrize", + pytrace=False, + ) + + if scope is not None: + scope_ = Scope.from_user( + scope, descr=f"parametrize() call in {self._target_name}" + ) + else: + scope_ = self._infer_scope(argnames, indirect) + + self._validate_argnames(argnames, indirect) + + # Use any already (possibly) generated ids with parametrize Marks. + generated_ids = None + if _param_mark and _param_mark._param_ids_from: + generated_ids = _param_mark._param_ids_from._param_ids_generated + if generated_ids is not None: + ids = generated_ids + + ids = self._resolve_parameter_set_ids( + argnames, ids, parametersets, nodeid=nodeid + ) + + # Store used (possibly generated) ids with parametrize Marks. + if _param_mark and _param_mark._param_ids_from and generated_ids is None: + object.__setattr__(_param_mark._param_ids_from, "_param_ids_generated", ids) + + # Calculate directness. + arg_directness = _resolve_args_directness(argnames, indirect, nodeid) + self._params_directness.update(arg_directness) + + self._register_direct_params(argnames, arg_directness, scope_) + + # Create the new calls: if we are parametrize() multiple times (by applying the decorator + # more than once) then we accumulate those calls generating the cartesian product + # of all calls. + newcalls = [] + for callspec in self._calls or [CallSpec()]: + for param_index, (param_id, param_set) in enumerate( + zip(ids, parametersets, strict=True) + ): + newcallspec = callspec.setmulti( + argnames=argnames, + valset=param_set.values, + id=param_id, + marks=param_set.marks, + scope=scope_, + param_index=param_index, + nodeid=nodeid, + ) + newcalls.append(newcallspec) + self._calls = newcalls + + def _resolve_parameter_set_ids( + self, + argnames: Sequence[str], + ids: Iterable[object | None] | Callable[[Any], object | None] | None, + parametersets: Sequence[ParameterSet], + nodeid: str, + ) -> list[str | _HiddenParam]: + """Resolve the actual ids for the given parameter sets. + + :param argnames: + Argument names passed to ``parametrize()``. + :param ids: + The `ids` parameter of the ``parametrize()`` call (see docs). + :param parametersets: + The parameter sets, each containing a set of values corresponding + to ``argnames``. + :param nodeid str: + The nodeid of the definition item that generated this + parametrization. + :returns: + List with ids for each parameter set given. + """ + if ids is None: + idfn = None + ids_ = None + elif callable(ids): + idfn = ids + ids_ = None + else: + idfn = None + ids_ = self._validate_ids(ids, parametersets) + id_maker = IdMaker( + argnames, + parametersets, + idfn, + ids_, + self.config, + nodeid=nodeid, + ) + return id_maker.make_unique_parameterset_ids() + + def _validate_ids( + self, + ids: Iterable[object | None], + parametersets: Sequence[ParameterSet], + ) -> list[object | None]: + try: + num_ids = len(ids) # type: ignore[arg-type] + except TypeError: + try: + iter(ids) + except TypeError as e: + raise TypeError("ids must be a callable or an iterable") from e + num_ids = len(parametersets) + + # num_ids == 0 is a special case: https://github.com/pytest-dev/pytest/issues/1849 + if num_ids != len(parametersets) and num_ids != 0: + nodeid = self.definition.nodeid + fail( + f"In {nodeid}: {len(parametersets)} parameter sets specified, with different number of ids: {num_ids}", + pytrace=False, + ) + + return list(itertools.islice(ids, num_ids)) + + def _recompute_direct_params_indices(self) -> None: + for argname, param_type in self._params_directness.items(): + if param_type == "direct": + for i, callspec in enumerate(self._calls): + callspec.indices[argname] = i diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 7b2d26669ea..f71fa55bc97 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -4,32 +4,23 @@ from __future__ import annotations import abc -from collections import Counter -from collections import defaultdict from collections.abc import Callable from collections.abc import Generator from collections.abc import Iterable from collections.abc import Iterator from collections.abc import Mapping from collections.abc import Sequence -import dataclasses -import enum import fnmatch from functools import partial -import hashlib import inspect -import itertools import os from pathlib import Path -import re -import textwrap import types from typing import Any from typing import cast from typing import final from typing import get_args from typing import Literal -from typing import NoReturn from typing import TYPE_CHECKING import warnings @@ -41,8 +32,6 @@ from _pytest._code.code import ExceptionInfo from _pytest._code.code import TerminalRepr from _pytest._code.code import Traceback -from _pytest._io.saferepr import saferepr -from _pytest.compat import ascii_escaped from _pytest.compat import get_default_arg_names from _pytest.compat import get_real_func from _pytest.compat import getimfunc @@ -56,28 +45,23 @@ from _pytest.config.argparsing import Parser from _pytest.deprecated import CALLSPEC2_RENAMED from _pytest.deprecated import check_ispytest -from _pytest.fixtures import _resolve_args_directness from _pytest.fixtures import FixtureDef from _pytest.fixtures import FixtureRequest from _pytest.fixtures import FixtureValue from _pytest.fixtures import FuncFixtureInfo from _pytest.fixtures import get_scope_node from _pytest.main import Session -from _pytest.mark import ParameterSet -from _pytest.mark.structures import _HiddenParam from _pytest.mark.structures import get_unpacked_marks -from _pytest.mark.structures import HIDDEN_PARAM -from _pytest.mark.structures import Mark -from _pytest.mark.structures import MarkDecorator -from _pytest.mark.structures import normalize_mark_list from _pytest.outcomes import fail from _pytest.outcomes import skip +from _pytest.parametrize import _infer_parametrize_scope +from _pytest.parametrize import CallSpec +from _pytest.parametrize import ParametrizeContext from _pytest.pathlib import fnmatch_ex from _pytest.pathlib import import_path from _pytest.pathlib import ImportPathMismatchError from _pytest.pathlib import scandir from _pytest.scope import Scope -from _pytest.scope import ScopeName from _pytest.stash import StashKey from _pytest.warning_types import PytestCollectionWarning from _pytest.warning_types import PytestReturnNotNoneWarning @@ -86,11 +70,6 @@ if TYPE_CHECKING: from typing_extensions import Self -LongStrIdStrategy = Literal["short", "sha256", "legacy", "disallow"] -_LONG_STR_STRATEGIES: frozenset[LongStrIdStrategy] = frozenset( - get_args(LongStrIdStrategy) -) - # Modes for the ``collect_function_definition`` option. # - "hidden": legacy flat layout; the FunctionDefinition is used transiently to # drive parametrization and kept out of ("hidden" from) the tree. @@ -507,13 +486,13 @@ def collect(self) -> Iterable[nodes.Item | nodes.Collector]: def _genfunctions( self, name: str, funcobj - ) -> Iterator[Function | FunctionDefinition]: + ) -> Iterator[nodes.Item | FunctionDefinition]: definition = FunctionDefinition.from_parent(self, name=name, callobj=funcobj) - if _collect_function_definition_mode(self.config) == "hidden": + if not definition.in_collection_tree: # Legacy flat layout: the definition is used only to drive # parametrization and is discarded ("hidden"), the invocations are # collected directly under this collector. - yield from definition._generate_functions(self) + yield from definition.generate_items(self) else: # Insert the function definition as a collector node into the tree; # its ``collect()`` yields the (possibly parametrized) invocations. @@ -887,336 +866,6 @@ def hasnew(obj: object) -> bool: return False -@final -@dataclasses.dataclass(frozen=True) -class IdMaker: - """Make IDs for a parametrization.""" - - __slots__ = ( - "argnames", - "config", - "idfn", - "ids", - "nodeid", - "parametersets", - ) - - # The argnames of the parametrization. - argnames: Sequence[str] - # The ParameterSets of the parametrization. - parametersets: Sequence[ParameterSet] - # Optionally, a user-provided callable to make IDs for parameters in a - # ParameterSet. - idfn: Callable[[Any], object | None] | None - # Optionally, explicit IDs for ParameterSets by index. - ids: Sequence[object | None] | None - # Optionally, the pytest config. - # Used for controlling ASCII escaping, determining parametrization ID - # strictness, and for calling the :hook:`pytest_make_parametrize_id` hook. - config: Config | None - # Optionally, the ID of the node being parametrized. - # Used only for clearer error messages. - nodeid: str | None - - def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]: - """Make a unique identifier for each ParameterSet, that may be used to - identify the parametrization in a node ID. - - If strict_parametrization_ids is enabled, and duplicates are detected, - raises CollectError. Otherwise makes the IDs unique as follows: - - Format is -...-[counter], where prm_x_token is - - user-provided id, if given - - else an id derived from the value, applicable for certain types - - else - The counter suffix is appended only in case a string wouldn't be unique - otherwise. - """ - resolved_ids = list(self._resolve_ids()) - # All IDs must be unique! - if len(resolved_ids) != len(set(resolved_ids)): - # Record the number of occurrences of each ID. - id_counts = Counter(resolved_ids) - - if self._strict_parametrization_ids_enabled(): - parameters = ", ".join(self.argnames) - parametersets = ", ".join( - [saferepr(list(param.values)) for param in self.parametersets] - ) - ids = ", ".join( - id if id is not HIDDEN_PARAM else "" for id in resolved_ids - ) - duplicates = ", ".join( - id if id is not HIDDEN_PARAM else "" - for id, count in id_counts.items() - if count > 1 - ) - msg = textwrap.dedent(f""" - Duplicate parametrization IDs detected, but strict_parametrization_ids is set. - - Test name: {self.nodeid} - Parameters: {parameters} - Parameter sets: {parametersets} - IDs: {ids} - Duplicates: {duplicates} - - You can fix this problem using `@pytest.mark.parametrize(..., ids=...)` or `pytest.param(..., id=...)`. - """).strip() # noqa: E501 - raise nodes.Collector.CollectError(msg) - - # Map the ID to its next suffix. - id_suffixes: dict[str, int] = defaultdict(int) - # Suffix non-unique IDs to make them unique. - for index, id in enumerate(resolved_ids): - if id_counts[id] > 1: - if id is HIDDEN_PARAM: - self._complain_multiple_hidden_parameter_sets() - suffix = "" - if id and id[-1].isdigit(): - suffix = "_" - new_id = f"{id}{suffix}{id_suffixes[id]}" - while new_id in set(resolved_ids): - id_suffixes[id] += 1 - new_id = f"{id}{suffix}{id_suffixes[id]}" - resolved_ids[index] = new_id - id_suffixes[id] += 1 - assert len(resolved_ids) == len(set(resolved_ids)), ( - f"Internal error: {resolved_ids=}" - ) - return resolved_ids - - def _strict_parametrization_ids_enabled(self) -> bool: - if self.config is None: - return False - strict_parametrization_ids = self.config.getini("strict_parametrization_ids") - if strict_parametrization_ids is None: - strict_parametrization_ids = self.config.getini("strict") - return cast(bool, strict_parametrization_ids) - - def _resolve_ids(self) -> Iterable[str | _HiddenParam]: - """Resolve IDs for all ParameterSets (may contain duplicates).""" - for idx, parameterset in enumerate(self.parametersets): - if parameterset.id is not None: - # ID provided directly - pytest.param(..., id="...") - if parameterset.id is HIDDEN_PARAM: - yield HIDDEN_PARAM - else: - yield _ascii_escaped_by_config(parameterset.id, self.config) - elif self.ids and idx < len(self.ids) and self.ids[idx] is not None: - # ID provided in the IDs list - parametrize(..., ids=[...]). - if self.ids[idx] is HIDDEN_PARAM: - yield HIDDEN_PARAM - else: - yield self._idval_from_value_required(self.ids[idx], idx) - else: - # ID not provided - generate it. - yield "-".join( - self._idval(val, argname, idx) - for val, argname in zip( - parameterset.values, self.argnames, strict=True - ) - ) - - def _idval(self, val: object, argname: str, idx: int) -> str: - """Make an ID for a parameter in a ParameterSet.""" - idval = self._idval_from_function(val, argname, idx) - if idval is not None: - return idval - idval = self._idval_from_hook(val, argname) - if idval is not None: - return idval - if isinstance(val, str | bytes): - idval = self._apply_long_str_strategy(val, argname, idx) - if idval is not None: - return idval - else: - idval = self._idval_from_value(val) - if idval is not None: - return idval - return self._idval_from_argname(argname, idx) - - def _get_long_str_strategy(self) -> LongStrIdStrategy: - if not self.config: - return "short" - value = self.config.getini("parametrize_long_str_id_strategy") - if value not in _LONG_STR_STRATEGIES: - raise UsageError( - f"Unknown parametrize_long_str_id_strategy: {value!r}. " - f"Valid values: {', '.join(sorted(_LONG_STR_STRATEGIES))}" - ) - return cast(LongStrIdStrategy, value) - - def _apply_long_str_strategy( - self, val: str | bytes, argname: str, idx: int - ) -> str | None: - """Apply the configured strategy for long str/bytes parameter values. - - Only used for auto-generated IDs (not explicit ids=[...] or - pytest.param(id=...)). - """ - if len(val) <= 100: - return _ascii_escaped_by_config(val, self.config) - match self._get_long_str_strategy(): - case "legacy": - return _ascii_escaped_by_config(val, self.config) - case "short": - return None - case "sha256": - encoded = val.encode("utf-8") if isinstance(val, str) else val - return hashlib.sha256(encoded).hexdigest() - case "disallow": # pragma: no branch -- fail() raises, confuses coverage - prefix = self._make_error_prefix() - fail( - f"{prefix}parametrize value for '{argname}' at index {idx} " - f"is too long for an auto-generated ID ({len(val)} characters). " - f"Use pytest.param(..., id=...) or parametrize(..., ids=...) " - f"to set an explicit ID, or change parametrize_long_str_id_strategy.", - pytrace=False, - ) - - def _idval_from_function(self, val: object, argname: str, idx: int) -> str | None: - """Try to make an ID for a parameter in a ParameterSet using the - user-provided id callable, if given.""" - if self.idfn is None: - return None - try: - id = self.idfn(val) - except Exception as e: - prefix = f"{self.nodeid}: " if self.nodeid is not None else "" - msg = "error raised while trying to determine id of parameter '{}' at position {}" - msg = prefix + msg.format(argname, idx) - raise ValueError(msg) from e - if id is None: - return None - return self._idval_from_value(id) - - def _idval_from_hook(self, val: object, argname: str) -> str | None: - """Try to make an ID for a parameter in a ParameterSet by calling the - :hook:`pytest_make_parametrize_id` hook.""" - if self.config: - id: str | None = self.config.hook.pytest_make_parametrize_id( - config=self.config, val=val, argname=argname - ) - return id - return None - - def _idval_from_value(self, val: object) -> str | None: - """Try to make an ID for a parameter in a ParameterSet from its value, - if the value type is supported.""" - if isinstance(val, str | bytes): - return _ascii_escaped_by_config(val, self.config) - elif val is None or isinstance(val, float | int | bool | complex): - return str(val) - elif isinstance(val, re.Pattern): - return ascii_escaped(val.pattern) - elif val is NOTSET: - # Fallback to default. Note that NOTSET is an enum.Enum. - pass - elif isinstance(val, enum.Enum): - return str(val) - elif isinstance(getattr(val, "__name__", None), str): - # Name of a class, function, module, etc. - name: str = getattr(val, "__name__") - return name - return None - - def _idval_from_value_required(self, val: object, idx: int) -> str: - """Like _idval_from_value(), but fails if the type is not supported.""" - id = self._idval_from_value(val) - if id is not None: - return id - - # Fail. - prefix = self._make_error_prefix() - msg = ( - f"{prefix}ids contains unsupported value {saferepr(val)} (type: {type(val)!r}) at index {idx}. " - "Supported types are: str, bytes, int, float, complex, bool, enum, regex or anything with a __name__." - ) - fail(msg, pytrace=False) - - @staticmethod - def _idval_from_argname(argname: str, idx: int) -> str: - """Make an ID for a parameter in a ParameterSet from the argument name - and the index of the ParameterSet.""" - return str(argname) + str(idx) - - def _complain_multiple_hidden_parameter_sets(self) -> NoReturn: - fail( - f"{self._make_error_prefix()}multiple instances of HIDDEN_PARAM " - "cannot be used in the same parametrize call, " - "because the tests names need to be unique." - ) - - def _make_error_prefix(self) -> str: - if self.nodeid is not None: - return f"In {self.nodeid}: " - else: - return "" - - -@final -@dataclasses.dataclass(frozen=True) -class CallSpec: - """A planned parameterized invocation of a test function. - - Calculated during collection for a given test function's Metafunc. - Once collection is over, each callspec is turned into a single Item - and stored in item.callspec. - """ - - # arg name -> arg value which will be passed to a fixture of the same name. - params: dict[str, object] = dataclasses.field(default_factory=dict) - # arg name -> arg index. - indices: dict[str, int] = dataclasses.field(default_factory=dict) - # arg name -> parameter scope. - # Used for sorting parametrized resources. - _arg2scope: Mapping[str, Scope] = dataclasses.field(default_factory=dict) - # Parts which will be added to the item's name in `[..]` separated by "-". - _idlist: Sequence[str] = dataclasses.field(default_factory=tuple) - # Marks which will be applied to the item. - marks: list[Mark] = dataclasses.field(default_factory=list) - - def setmulti( - self, - *, - argnames: Iterable[str], - valset: Iterable[object], - id: str | _HiddenParam, - marks: Iterable[Mark | MarkDecorator], - scope: Scope, - param_index: int, - nodeid: str, - ) -> CallSpec: - params = self.params.copy() - indices = self.indices.copy() - arg2scope = dict(self._arg2scope) - for arg, val in zip(argnames, valset, strict=True): - if arg in params: - raise nodes.Collector.CollectError( - f"{nodeid}: duplicate parametrization of {arg!r}" - ) - params[arg] = val - indices[arg] = param_index - arg2scope[arg] = scope - return CallSpec( - params=params, - indices=indices, - _arg2scope=arg2scope, - _idlist=self._idlist if id is HIDDEN_PARAM else [*self._idlist, id], - marks=[*self.marks, *normalize_mark_list(marks)], - ) - - def getparam(self, name: str) -> object: - try: - return self.params[name] - except KeyError as e: - raise ValueError(name) from e - - @property - def id(self) -> str: - return "-".join(self._idlist) - - if TYPE_CHECKING: # Deprecated alias kept for type checkers; runtime access goes through __getattr__. CallSpec2 = CallSpec @@ -1254,14 +903,20 @@ def __init__(self, *, node: nodes.Node, argname: str, scope: Scope) -> None: @final -class Metafunc: +class Metafunc(ParametrizeContext): """Objects passed to the :hook:`pytest_generate_tests` hook. They help to inspect a test function and to generate tests according to test configuration or values specified in the class or module where a test function is defined. + + The Python-specific :class:`~_pytest.parametrize.ParametrizeContext`: it + resolves ``argnames`` against the test function's fixture closure and + signature, and desugars direct parametrization into fixtures. """ + definition: FunctionDefinition + def __init__( self, definition: FunctionDefinition, @@ -1273,12 +928,19 @@ def __init__( _ispytest: bool = False, ) -> None: check_ispytest(_ispytest) - #: Access to the underlying :class:`_pytest.python.FunctionDefinition`. - self.definition = definition - - #: Access to the :class:`pytest.Config` object for the test session. - self.config = config + #: + #: Access to the :class:`pytest.Config` object for the test session is + #: available as ``config``, and the set of fixture names required by the + #: test function as ``fixturenames``. + super().__init__( + definition, + config, + # Note: passed through by reference, so that a later + # prune_dependency_tree() is reflected in ``fixturenames``. + argnames=fixtureinfo.names_closure, + _ispytest=True, + ) #: The module object where the test function is defined in. self.module = module @@ -1286,177 +948,53 @@ def __init__( #: Underlying Python test function. self.function = definition.obj - #: Set of fixture names required by the test function. - self.fixturenames = fixtureinfo.names_closure - #: Class object where the test function is defined in or ``None``. self.cls = cls + # The fixture closure driving this parametrization, handed on to the + # generated Functions by FunctionDefinition.make_item(). + self._fixtureinfo = fixtureinfo self._arg2fixturedefs = fixtureinfo.name2fixturedefs - # Result of parametrize(). - self._calls: list[CallSpec] = [] + @property + def _introspection_target(self) -> object: + return self.function - self._params_directness: dict[str, Literal["indirect", "direct"]] = {} + @property + def _target_name(self) -> str: + return cast(str, self.function.__name__) - def parametrize( + def _infer_scope( + self, argnames: Sequence[str], indirect: bool | Sequence[str] + ) -> Scope: + return _infer_parametrize_scope(argnames, self._arg2fixturedefs, indirect) + + def _validate_argnames( self, - argnames: str | Sequence[str], - argvalues: Iterable[ParameterSet | Sequence[object] | object], - *, - indirect: bool | Sequence[str] = False, - ids: Iterable[object | None] | Callable[[Any], object | None] | None = None, - scope: ScopeName | None = None, - _param_mark: Mark | None = None, + argnames: Sequence[str], + indirect: bool | Sequence[str], ) -> None: - """Add new invocations to the underlying test function using the list - of argvalues for the given argnames. Parametrization is performed - during the collection phase. If you need to setup expensive resources - see about setting ``indirect`` to do it at test setup time instead. - - Can be called multiple times per test function (but only on different - argument names), in which case each call parametrizes all previous - parametrizations, e.g. - - :: - - unparametrized: t - parametrize ["x", "y"]: t[x], t[y] - parametrize [1, 2]: t[x-1], t[x-2], t[y-1], t[y-2] - - :param argnames: - A comma-separated string denoting one or more argument names, or - a list/tuple of argument strings. - - :param argvalues: - The list of argvalues determines how often a test is invoked with - different argument values. - - If only one argname was specified argvalues is a list of values. - If N argnames were specified, argvalues must be a list of - N-tuples, where each tuple-element specifies a value for its - respective argname. - - .. versionchanged:: 9.1 - - Passing a non-:class:`~collections.abc.Collection` iterable - (such as a generator or iterator) is deprecated. See - :ref:`parametrize-iterators` for details. - - :param indirect: - A list of arguments' names (subset of argnames) or a boolean. - If True the list contains all names from the argnames. Each - argvalue corresponding to an argname in this list will - be passed as request.param to its respective argname fixture - function so that it can perform more expensive setups during the - setup phase of a test rather than at collection time. - - :param ids: - Sequence of (or generator for) ids for ``argvalues``, - or a callable to return part of the id for each argvalue. - - With sequences (and generators like ``itertools.count()``) the - returned ids should be of type ``string``, ``int``, ``float``, - ``bool``, or ``None``. - They are mapped to the corresponding index in ``argvalues``. - ``None`` means to use the auto-generated id. - - .. versionadded:: 8.4 - :ref:`hidden-param` means to hide the parameter set - from the test name. Can only be used at most 1 time, as - test names need to be unique. - - If it is a callable it will be called for each entry in - ``argvalues``, and the return value is used as part of the - auto-generated id for the whole set (where parts are joined with - dashes ("-")). - This is useful to provide more specific ids for certain items, e.g. - dates. Returning ``None`` will use an auto-generated id. - - If no ids are provided they will be generated automatically from - the argvalues. - - :param scope: - If specified it denotes the scope of the parameters. - The scope is used for grouping tests by parameter instances. - It will also override any fixture-function defined scope, allowing - to set a dynamic scope using test context or configuration. - - .. versionchanged:: 9.1 - - ``indirect``, ``ids`` and ``scope`` are now keyword-only. - """ - nodeid = self.definition.nodeid - - argnames, parametersets = ParameterSet._for_parametrize( - argnames, - argvalues, - self.function, - self.config, - nodeid=self.definition.nodeid, - ) - del argvalues - - if "request" in argnames: - fail( - f"{nodeid}: 'request' is a reserved name and cannot be used in @pytest.mark.parametrize", - pytrace=False, - ) - - if scope is not None: - scope_ = Scope.from_user( - scope, descr=f"parametrize() call in {self.function.__name__}" - ) - else: - scope_ = _infer_parametrize_scope(argnames, self._arg2fixturedefs, indirect) - self._validate_if_using_arg_names(argnames, indirect) - # Use any already (possibly) generated ids with parametrize Marks. - if _param_mark and _param_mark._param_ids_from: - generated_ids = _param_mark._param_ids_from._param_ids_generated - if generated_ids is not None: - ids = generated_ids - - ids = self._resolve_parameter_set_ids( - argnames, ids, parametersets, nodeid=self.definition.nodeid - ) - - # Store used (possibly generated) ids with parametrize Marks. - if _param_mark and _param_mark._param_ids_from and generated_ids is None: - object.__setattr__(_param_mark._param_ids_from, "_param_ids_generated", ids) - - # Calculate directness. - arg_directness = _resolve_args_directness( - argnames, indirect, self.definition.nodeid - ) - self._params_directness.update(arg_directness) + def _register_direct_params( + self, + argnames: Sequence[str], + arg_directness: Mapping[str, Literal["indirect", "direct"]], + scope: Scope, + ) -> None: + """Desugar direct parametrizations into artificial fixturedefs. - # Add direct parametrizations as fixturedefs to arg2fixturedefs by - # registering artificial DirectParamFixtureDef's such that later at test - # setup time we can rely on FixtureDefs to exist for all argnames. - node = None + Registering a :class:`DirectParamFixtureDef` for every direct argname + means that at test setup time we can rely on a FixtureDef existing for + all argnames. + """ # For scopes higher than function, a DirectParamFixtureDef might have # already been created for the scope. We thus store and cache the # DirectParamFixtureDef on the node related to the scope. - if scope_ is Scope.Function: + if scope is Scope.Function: name2directparamfixturedef = None else: - collector = self.definition.parent - assert collector is not None - node = get_scope_node(collector, scope_) - if node is None: - # If used class scope and there is no class, use module-level - # collector (for now). - if scope_ is Scope.Class: - assert isinstance(collector, Module) - node = collector - # If used package scope and there is no package, use session - # (for now). - elif scope_ is Scope.Package: - node = collector.session - else: - assert False, f"Unhandled missing scope: {scope}" + node = self._scope_node_for_direct_params(scope) default: dict[str, DirectParamFixtureDef[object]] = {} name2directparamfixturedef = node.stash.setdefault( name2directparamfixturedef_key, default @@ -1473,96 +1011,39 @@ def parametrize( fixturedef = DirectParamFixtureDef( node=self.definition.session, argname=argname, - scope=scope_, + scope=scope, ) if name2directparamfixturedef is not None: name2directparamfixturedef[argname] = fixturedef self._arg2fixturedefs[argname] = [fixturedef] - # Create the new calls: if we are parametrize() multiple times (by applying the decorator - # more than once) then we accumulate those calls generating the cartesian product - # of all calls. - newcalls = [] - for callspec in self._calls or [CallSpec()]: - for param_index, (param_id, param_set) in enumerate( - zip(ids, parametersets, strict=True) - ): - newcallspec = callspec.setmulti( - argnames=argnames, - valset=param_set.values, - id=param_id, - marks=param_set.marks, - scope=scope_, - param_index=param_index, - nodeid=nodeid, + def _scope_node_for_direct_params(self, scope: Scope) -> nodes.Node: + """The node the DirectParamFixtureDefs for ``scope`` are cached on.""" + if scope is Scope.Definition: + if not self.definition.in_collection_tree: + fixtures.fail_definition_scope_unavailable( + self.definition.nodeid, + f"parametrize(scope='definition') in {self._target_name}", ) - newcalls.append(newcallspec) - self._calls = newcalls - - def _resolve_parameter_set_ids( - self, - argnames: Sequence[str], - ids: Iterable[object | None] | Callable[[Any], object | None] | None, - parametersets: Sequence[ParameterSet], - nodeid: str, - ) -> list[str | _HiddenParam]: - """Resolve the actual ids for the given parameter sets. - - :param argnames: - Argument names passed to ``parametrize()``. - :param ids: - The `ids` parameter of the ``parametrize()`` call (see docs). - :param parametersets: - The parameter sets, each containing a set of values corresponding - to ``argnames``. - :param nodeid str: - The nodeid of the definition item that generated this - parametrization. - :returns: - List with ids for each parameter set given. - """ - if ids is None: - idfn = None - ids_ = None - elif callable(ids): - idfn = ids - ids_ = None - else: - idfn = None - ids_ = self._validate_ids(ids, parametersets) - id_maker = IdMaker( - argnames, - parametersets, - idfn, - ids_, - self.config, - nodeid=nodeid, - ) - return id_maker.make_unique_parameterset_ids() - - def _validate_ids( - self, - ids: Iterable[object | None], - parametersets: Sequence[ParameterSet], - ) -> list[object | None]: - try: - num_ids = len(ids) # type: ignore[arg-type] - except TypeError: - try: - iter(ids) - except TypeError as e: - raise TypeError("ids must be a callable or an iterable") from e - num_ids = len(parametersets) - - # num_ids == 0 is a special case: https://github.com/pytest-dev/pytest/issues/1849 - if num_ids != len(parametersets) and num_ids != 0: - nodeid = self.definition.nodeid - fail( - f"In {nodeid}: {len(parametersets)} parameter sets specified, with different number of ids: {num_ids}", - pytrace=False, - ) - - return list(itertools.islice(ids, num_ids)) + # The definition is the scope node for its own invocations; unlike + # the other scopes it is not found by looking at ancestors. + return self.definition + collector = self.definition.parent + assert collector is not None + node = get_scope_node(collector, scope) + if node is None: + # If used class scope and there is no class, use module-level + # collector (for now). + if scope is Scope.Class: + assert isinstance(collector, Module) + node = collector + # If used package scope and there is no package, use session + # (for now). + elif scope is Scope.Package: + node = collector.session + else: + assert False, f"Unhandled missing scope: {scope}" + return node def _validate_if_using_arg_names( self, @@ -1594,59 +1075,6 @@ def _validate_if_using_arg_names( pytrace=False, ) - def _recompute_direct_params_indices(self) -> None: - for argname, param_type in self._params_directness.items(): - if param_type == "direct": - for i, callspec in enumerate(self._calls): - callspec.indices[argname] = i - - -def _infer_parametrize_scope( - argnames: Sequence[str], - arg2fixturedefs: Mapping[str, Sequence[fixtures.FixtureDef[object]]], - indirect: bool | Sequence[str], -) -> Scope: - """Infer the most appropriate scope for a parametrize() call based on its - arguments, for when the scope is not explicitly specified. - - When there's at least one direct argument, always use "function" scope. - - When a test function is parametrized and all its arguments are indirect - (e.g. fixtures), return the most narrow scope based on the fixtures used. - - Related to issue #1832, based on code posted by @Kingdread. - """ - if isinstance(indirect, Sequence): - all_arguments_are_fixtures = len(indirect) == len(argnames) - else: - all_arguments_are_fixtures = bool(indirect) - - if all_arguments_are_fixtures: - # Takes the most narrow scope from used fixtures. - used_scopes = ( - # Higher scope can't request lower scope, so it's OK to only - # look at the first fixturedef in the override chain. - arg2fixturedefs[argname][-1]._scope - for argname in argnames - if argname in arg2fixturedefs - ) - return min(used_scopes, default=Scope.Function) - - return Scope.Function - - -def _ascii_escaped_by_config(val: str | bytes, config: Config | None) -> str: - if config is None: - escape_option = False - else: - escape_option = config.getini( - "disable_test_id_escaping_and_forfeit_all_rights_to_community_support" - ) - # TODO: If escaping is turned off and the user passes bytes, - # will return a bytes. For now we ignore this but the - # code *probably* doesn't handle this case. - return val if escape_option else ascii_escaped(val) # type: ignore - class Function(PyobjMixin, nodes.Item): """Item responsible for setting up and executing a Python test function. @@ -1836,7 +1264,7 @@ def repr_failure( # type: ignore[override] return self._repr_failure_py(excinfo, style=style) -class FunctionDefinition(PyCollector): +class FunctionDefinition(nodes.ItemDefinition, PyCollector): """Collector node for a single test function definition. Its children are the (possibly parametrized) :class:`Function` invocations @@ -1870,8 +1298,19 @@ def __init__( self.keywords.update((mark.name, mark) for mark in self.own_markers) self.keywords.update(self.obj.__dict__) + @property + def in_collection_tree(self) -> bool: + """Whether this definition is a node of the collection tree. + + False in the default ``hidden`` mode of + :confval:`collect_function_definition`, where the definition only exists + transiently during collection -- and thus cannot anchor anything at run + time, such as a ``"definition"`` scoped fixture. + """ + return _collect_function_definition_mode(self.config) != "hidden" + def collect(self) -> Iterable[nodes.Item | nodes.Collector]: - children = list(self._generate_functions(self)) + children = list(super().collect()) if _collect_function_definition_mode(self.config) == "messy": # Legacy marker layout: transfer this scope's markers back onto each # invocation and drop them here, so marker resolution matches the flat @@ -1883,23 +1322,24 @@ def collect(self) -> Iterable[nodes.Item | nodes.Collector]: self.own_markers = [] return children - def _generate_functions(self, parent: nodes.Collector) -> Iterator[Function]: - """Run :hook:`pytest_generate_tests` and yield the resulting - :class:`Function` invocations under ``parent``. + # The generate-tests protocol, see nodes.ItemDefinition. - ``parent`` is this definition node when it is part of the tree, or the - containing :class:`Class`/:class:`Module` in the legacy flat layout. - """ - name = self.name + @property + def _module_obj(self) -> types.ModuleType: modulecol = self.getparent(Module) assert modulecol is not None - module = modulecol.obj + return cast(types.ModuleType, modulecol.obj) + + @property + def _cls_obj(self) -> type | None: clscol = self.getparent(Class) - cls = (clscol and clscol.obj) or None + return (clscol and clscol.obj) or None + def make_parametrize_context(self) -> Metafunc: + cls = self._cls_obj # Compute the function's fixture closure. This drives parametrization and - # is shared with the generated invocations; it is intentionally *not* - # stored on the definition -- fixtures belong to the executed items, not + # is shared with the generated invocations; it is carried by the Metafunc + # rather than stored here -- fixtures belong to the executed items, not # to this collector. # TODO(#3926): getfixtureinfo() is item-scoped, but here the definition # (a collector) stands in for the not-yet-created invocations. Resolve by @@ -1908,58 +1348,62 @@ def _generate_functions(self, parent: nodes.Collector) -> Iterator[Function]: fixtureinfo = self.session._fixturemanager.getfixtureinfo( cast(nodes.Item, self), self.obj, cls ) - - # pytest_generate_tests impls call metafunc.parametrize() which fills - # metafunc._calls, the outcome of the hook. - metafunc = Metafunc( + return Metafunc( definition=self, fixtureinfo=fixtureinfo, config=self.config, cls=cls, - module=module, + module=self._module_obj, _ispytest=True, ) + + def parametrize_hook_extras(self) -> Sequence[Callable[..., object]]: + module = self._module_obj + cls = self._cls_obj methods = [] if hasattr(module, "pytest_generate_tests"): methods.append(module.pytest_generate_tests) if cls is not None and hasattr(cls, "pytest_generate_tests"): methods.append(cls().pytest_generate_tests) - self.ihook.pytest_generate_tests.call_extra(methods, dict(metafunc=metafunc)) - - # The invocations keep a flat nodeid anchored at the collector containing - # the definition, regardless of whether the definition itself is part of - # the tree, so nodeids are stable across the ``collect_function_definition`` - # option. - assert self.parent is not None - base_nodeid = self.parent.nodeid - - if not metafunc._calls: - yield Function.from_parent( + return methods + + def finalize_parametrization(self, context: ParametrizeContext) -> None: + super().finalize_parametrization(context) + # Direct parametrizations taking place in module/class-specific + # `metafunc.parametrize` calls may have shadowed some fixtures, so make sure + # we update what the function really needs a.k.a its fixture closure. Note that + # direct parametrizations using `@pytest.mark.parametrize` have already been considered + # into making the closure using `ignore_args` arg to `getfixtureclosure`. + assert isinstance(context, Metafunc) + context._fixtureinfo.prune_dependency_tree() + + def make_item( + self, + parent: nodes.Collector, + *, + name: str, + callspec: CallSpec | None, + nodeid: str, + context: ParametrizeContext, + ) -> Function: + assert isinstance(context, Metafunc) + fixtureinfo = context._fixtureinfo + if callspec is None: + return Function.from_parent( parent, name=name, fixtureinfo=fixtureinfo, - nodeid=f"{base_nodeid}::{name}", + nodeid=nodeid, ) - else: - metafunc._recompute_direct_params_indices() - # Direct parametrizations taking place in module/class-specific - # `metafunc.parametrize` calls may have shadowed some fixtures, so make sure - # we update what the function really needs a.k.a its fixture closure. Note that - # direct parametrizations using `@pytest.mark.parametrize` have already been considered - # into making the closure using `ignore_args` arg to `getfixtureclosure`. - fixtureinfo.prune_dependency_tree() - - for callspec in metafunc._calls: - subname = f"{name}[{callspec.id}]" if callspec._idlist else name - yield Function.from_parent( - parent, - name=subname, - callspec=callspec, - fixtureinfo=fixtureinfo, - keywords={callspec.id: True}, - originalname=name, - nodeid=f"{base_nodeid}::{subname}", - ) + return Function.from_parent( + parent, + name=name, + callspec=callspec, + fixtureinfo=fixtureinfo, + keywords={callspec.id: True}, + originalname=self.name, + nodeid=nodeid, + ) def __getattr__(name: str) -> object: diff --git a/src/_pytest/reports.py b/src/_pytest/reports.py index 011a69db001..fb9ce20692a 100644 --- a/src/_pytest/reports.py +++ b/src/_pytest/reports.py @@ -417,8 +417,10 @@ def from_item_and_call(cls, item: Item, call: CallInfo[None]) -> TestReport: ) if excinfo.value._use_item_location: path, line = item.reportinfo()[:2] - assert line is not None - longrepr = (os.fspath(path), line + 1, r.message) + # Items which cannot point at a line -- non-Python items may + # legitimately report None -- get the location without one. + lineno = line + 1 if line is not None else -1 + longrepr = (os.fspath(path), lineno, r.message) else: longrepr = (str(r.path), r.lineno, r.message) elif isinstance(excinfo.value, BaseExceptionGroup) and ( diff --git a/src/_pytest/scope.py b/src/_pytest/scope.py index 68a9da0e6b8..9cc05bc1e90 100644 --- a/src/_pytest/scope.py +++ b/src/_pytest/scope.py @@ -15,7 +15,7 @@ from typing import Literal -ScopeName = Literal["session", "package", "module", "class", "function"] +ScopeName = Literal["session", "package", "module", "class", "definition", "function"] @total_ordering @@ -27,13 +27,19 @@ class Scope(Enum): ->>> higher ->>> - Function < Class < Module < Package < Session + Function < Definition < Class < Module < Package < Session <<<- lower <<<- + + ``Definition`` is the scope of a single test *definition*, shared by all + the (possibly parametrized) invocations generated from it. It requires the + definition to be a node in the collection tree -- see + :confval:`collect_function_definition`. """ # Scopes need to be listed from lower to higher. Function = "function" + Definition = "definition" Class = "class" Module = "module" Package = "package" diff --git a/src/_pytest/setuponly.py b/src/_pytest/setuponly.py index 7e6b46bcdb4..eed5b52e52c 100644 --- a/src/_pytest/setuponly.py +++ b/src/_pytest/setuponly.py @@ -61,6 +61,20 @@ def pytest_fixture_post_finalizer( del fixturedef.cached_param +# Use smaller indentation the higher the scope. Spelled out rather than derived +# from the Scope order, so that adding a scope does not shift the output of the +# existing ones; Definition shares Function's indentation and is told apart by +# the scope letter. +_SCOPE_INDENTS = { + Scope.Session: 0, + Scope.Package: 1, + Scope.Module: 2, + Scope.Class: 3, + Scope.Definition: 4, + Scope.Function: 4, +} + + def _show_fixture_action( fixturedef: FixtureDef[object], config: Config, msg: str ) -> None: @@ -70,9 +84,7 @@ def _show_fixture_action( tw = config.get_terminal_writer() tw.line() - # Use smaller indentation the higher the scope: Session = 0, Package = 1, etc. - scope_indent = list(reversed(Scope)).index(fixturedef._scope) - tw.write(" " * 2 * scope_indent) + tw.write(" " * 2 * _SCOPE_INDENTS[fixturedef._scope]) scopename = fixturedef.scope[0].upper() tw.write(f"{msg:<8} {scopename} {fixturedef.argname}") diff --git a/testing/deprecated_test.py b/testing/deprecated_test.py index 8eed1fb3149..cb36dbe7941 100644 --- a/testing/deprecated_test.py +++ b/testing/deprecated_test.py @@ -316,8 +316,8 @@ def test_it(request): def test_callspec2_renamed() -> None: """Importing/accessing CallSpec2 warns and returns CallSpec.""" + from _pytest.parametrize import CallSpec import _pytest.python as python_mod - from _pytest.python import CallSpec with pytest.warns(pytest.PytestRemovedIn10Warning, match="CallSpec2"): from _pytest.python import CallSpec2 diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index 3dd07a0ffc8..61b0f91f36c 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -21,9 +21,9 @@ from _pytest.compat import NOTSET from _pytest.outcomes import fail from _pytest.outcomes import Failed +from _pytest.parametrize import IdMaker from _pytest.pytester import Pytester from _pytest.python import Function -from _pytest.python import IdMaker from _pytest.scope import Scope import pytest @@ -223,7 +223,7 @@ def func(request): def test_infer_parametrize_scope(self) -> None: """Unit test for _infer_parameterize_scope (#3941).""" - from _pytest.python import _infer_parametrize_scope + from _pytest.parametrize import _infer_parametrize_scope @dataclasses.dataclass class DummyFixtureDef: diff --git a/testing/test_collect_function_definition.py b/testing/test_collect_function_definition.py index 5860bd2c356..68e8a486151 100644 --- a/testing/test_collect_function_definition.py +++ b/testing/test_collect_function_definition.py @@ -94,6 +94,32 @@ def test_selection_and_reporting(sample: Pytester) -> None: result.assert_outcomes(passed=1, deselected=5) +@pytest.mark.parametrize("mode", ALL_MODES) +@pytest.mark.parametrize( + ("selector", "expected"), + [ + ("test_param[1]", 1), + ("test_param", 2), + ("test_plain", 1), + ("TestCls::test_method[3]", 1), + ("TestCls::test_single", 1), + ("TestCls", 3), + ], +) +def test_selection_by_nodeid( + sample: Pytester, mode: str, selector: str, expected: int +) -> None: + """Addressing a test by its (flat) nodeid works in every mode. + + The definition node carries the bare name while the parametrization lives on + the items below it, so the argument matcher has to look through it. + """ + _set_mode(sample, mode) + module = next(sample.path.glob("test_*.py")).name + result = sample.runpytest(f"{module}::{selector}") + result.assert_outcomes(passed=expected) + + def test_collect_only_tree(sample: Pytester) -> None: _set_mode(sample, "pedantic") result = sample.runpytest("--collect-only") diff --git a/testing/test_definition_scope.py b/testing/test_definition_scope.py new file mode 100644 index 00000000000..083a80d1007 --- /dev/null +++ b/testing/test_definition_scope.py @@ -0,0 +1,344 @@ +"""Tests for the ``"definition"`` fixture scope. + +A definition-scoped fixture is shared by all the (possibly parametrized) +invocations generated from one test definition. That needs the definition to be +a node of the collection tree, i.e. :confval:`collect_function_definition` set +to something other than the default ``hidden``. +""" + +from __future__ import annotations + +from _pytest.pytester import Pytester +import pytest + + +def _enable_definition_nodes(pytester: Pytester) -> None: + pytester.makeini("[pytest]\ncollect_function_definition = pedantic\n") + + +def test_shared_between_invocations_of_one_definition(pytester: Pytester) -> None: + """One fixture instance per definition, not per invocation.""" + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + setups = [] + + @pytest.fixture(scope="definition") + def counter(request): + setups.append(request.node.nodeid) + return len(setups) + + @pytest.mark.parametrize("n", [1, 2, 3]) + def test_a(counter, n): + assert counter == 1 + + @pytest.mark.parametrize("n", [1, 2]) + def test_b(counter, n): + assert counter == 2 + + def test_setups_are_per_definition(): + assert setups == [ + "test_shared_between_invocations_of_one_definition.py::test_a", + "test_shared_between_invocations_of_one_definition.py::test_b", + ] + """ + ) + pytester.runpytest().assert_outcomes(passed=6) + + +def test_torn_down_when_definition_is_left(pytester: Pytester) -> None: + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + events = [] + + @pytest.fixture(scope="definition") + def res(request): + events.append(f"setup {request.node.name}") + yield + events.append(f"teardown {request.node.name}") + + @pytest.mark.parametrize("n", [1, 2]) + def test_a(res, n): + events.append(f"test_a{n}") + + @pytest.mark.parametrize("n", [1, 2]) + def test_b(res, n): + events.append(f"test_b{n}") + + def test_order(): + assert events == [ + "setup test_a", "test_a1", "test_a2", "teardown test_a", + "setup test_b", "test_b1", "test_b2", "teardown test_b", + ] + """ + ) + pytester.runpytest().assert_outcomes(passed=5) + + +def test_methods_of_a_class(pytester: Pytester) -> None: + """Each method has its own definition, the class is still shared.""" + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + definition_setups = [] + class_setups = [] + + @pytest.fixture(scope="definition") + def per_definition(): + definition_setups.append(1) + + @pytest.fixture(scope="class") + def per_class(): + class_setups.append(1) + + class TestIt: + @pytest.mark.parametrize("n", [1, 2]) + def test_a(self, per_definition, per_class, n): + pass + + @pytest.mark.parametrize("n", [1, 2]) + def test_b(self, per_definition, per_class, n): + pass + + def test_counts(): + assert len(definition_setups) == 2 + assert len(class_setups) == 1 + """ + ) + pytester.runpytest().assert_outcomes(passed=5) + + +def test_unavailable_without_definition_nodes(pytester: Pytester) -> None: + """In the default ``hidden`` mode there is no node to anchor the scope on.""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="definition") + def res(): + return 1 + + def test_it(res): + pass + """ + ) + result = pytester.runpytest() + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines( + [ + "*ScopeUnavailable: fixture 'res' needs the 'definition' scope, but" + " *::test_it has no definition node in the collection tree.", + "*set the collect_function_definition ini option to 'pedantic'*", + ] + ) + + +def test_may_request_higher_scoped_fixture(pytester: Pytester) -> None: + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="module") + def per_module(): + return 1 + + @pytest.fixture(scope="definition") + def per_definition(per_module): + return per_module + 1 + + def test_it(per_definition): + assert per_definition == 2 + """ + ) + pytester.runpytest().assert_outcomes(passed=1) + + +def test_higher_scope_may_not_request_definition_scope(pytester: Pytester) -> None: + """Class scope is higher than definition scope, so this is a ScopeMismatch.""" + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="definition") + def per_definition(): + return 1 + + @pytest.fixture(scope="class") + def per_class(per_definition): + return per_definition + + def test_it(per_class): + pass + """ + ) + result = pytester.runpytest() + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines( + [ + "*ScopeMismatch: You tried to access the definition scoped fixture" + " per_definition with a class scoped request object*" + ] + ) + + +def test_definition_scope_may_request_function_scope_fails( + pytester: Pytester, +) -> None: + """Function scope is lower than definition scope.""" + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def per_function(): + return 1 + + @pytest.fixture(scope="definition") + def per_definition(per_function): + return per_function + + def test_it(per_definition): + pass + """ + ) + result = pytester.runpytest() + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines( + [ + "*ScopeMismatch: You tried to access the function scoped fixture" + " per_function with a definition scoped request object*" + ] + ) + + +def test_setup_show_marks_the_scope(pytester: Pytester) -> None: + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="definition") + def res(): + return 1 + + @pytest.mark.parametrize("n", [1, 2]) + def test_it(res, n): + pass + """ + ) + result = pytester.runpytest("--setup-show") + result.assert_outcomes(passed=2) + result.stdout.fnmatch_lines( + [ + " SETUP D res", + "*test_it[[]1[]]*", + "*test_it[[]2[]]*", + " TEARDOWN D res", + ] + ) + + +def test_parametrize_with_definition_scope(pytester: Pytester) -> None: + """A definition-scoped param is set up once per value within the definition.""" + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + events = [] + + @pytest.fixture + def outer(request): + events.append(f"setup {request.param}") + yield request.param + events.append(f"teardown {request.param}") + + @pytest.mark.parametrize("outer", ["a", "b"], indirect=True, scope="definition") + @pytest.mark.parametrize("inner", [1, 2]) + def test_it(outer, inner): + events.append(f"{outer}{inner}") + + def test_order(): + assert events == [ + "setup a", "a1", "a2", "teardown a", + "setup b", "b1", "b2", "teardown b", + ] + """ + ) + pytester.runpytest().assert_outcomes(passed=5) + + +def test_parametrize_with_definition_scope_needs_definition_nodes( + pytester: Pytester, +) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize("x", [1, 2], scope="definition") + def test_it(x): + pass + """ + ) + result = pytester.runpytest() + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines( + [ + "*ScopeUnavailable: parametrize(scope='definition') in test_it needs" + " the 'definition' scope*" + ] + ) + + +def test_unknown_scope_still_rejected(pytester: Pytester) -> None: + """Guard against the new scope name loosening validation.""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="definitions") + def res(): + return 1 + + def test_it(res): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*got an unexpected scope value 'definitions'*"]) + + +@pytest.mark.parametrize("scope", ["session", "package", "module", "class"]) +def test_definition_scoped_fixture_may_be_requested_by_tests( + pytester: Pytester, scope: str +) -> None: + """A definition-scoped fixture composes with every higher scope.""" + _enable_definition_nodes(pytester) + pytester.makepyfile( + f""" + import pytest + + @pytest.fixture(scope="{scope}") + def higher(): + return "{scope}" + + @pytest.fixture(scope="definition") + def per_definition(higher): + return higher + + @pytest.mark.parametrize("n", [1, 2]) + def test_it(per_definition, n): + assert per_definition == "{scope}" + """ + ) + pytester.runpytest().assert_outcomes(passed=2) diff --git a/testing/test_item_definition.py b/testing/test_item_definition.py new file mode 100644 index 00000000000..f4c61f5fdcb --- /dev/null +++ b/testing/test_item_definition.py @@ -0,0 +1,199 @@ +"""Tests for :class:`_pytest.nodes.ItemDefinition`. + +The generic definition node lets a non-Python collector take part in +:hook:`pytest_generate_tests`: it declares the names it accepts, the hook +parametrizes it like any Python test, and the chosen values arrive on +``item.callspec.params``. +""" + +from __future__ import annotations + +from _pytest.pytester import Pytester +import pytest + + +# A minimal non-Python collector: every whitespace-separated word in a ``.echo`` +# file is a test definition that can be parametrized on "value". +ECHO_PLUGIN = """ +from _pytest import nodes +import pytest + + +class EchoItem(pytest.Item): + def __init__(self, *, spec, **kw): + super().__init__(**kw) + self.spec = spec + + def runtest(self): + assert self.spec.get("value") != "bad", f"bad value in {self.name}" + + def reportinfo(self): + return self.path, None, self.name + + +class EchoDefinition(nodes.ItemDefinition): + parametrize_argnames = ("value",) + + def make_item(self, parent, *, name, callspec, nodeid, context): + return EchoItem.from_parent( + parent, + name=name, + nodeid=nodeid, + spec=dict(callspec.params) if callspec is not None else {}, + ) + + +class EchoFile(pytest.File): + def collect(self): + for word in self.path.read_text().split(): + yield EchoDefinition.from_parent(self, name=word) + + +def pytest_collect_file(file_path, parent): + if file_path.suffix == ".echo": + return EchoFile.from_parent(parent, path=file_path) +""" + + +@pytest.fixture +def echo(pytester: Pytester) -> Pytester: + pytester.makeconftest(ECHO_PLUGIN) + pytester.makefile(".echo", things="alpha beta") + return pytester + + +def test_definition_without_parametrization(echo: Pytester) -> None: + """No parametrize() call: one item per definition, same name.""" + result = echo.runpytest("--collect-only", "-q") + result.stdout.fnmatch_lines(["things.echo::alpha", "things.echo::beta"]) + echo.runpytest().assert_outcomes(passed=2) + + +def test_generate_tests_hook_parametrizes_the_definition(echo: Pytester) -> None: + """An ordinary pytest_generate_tests impl works on a non-Python definition.""" + echo.makeconftest( + ECHO_PLUGIN + + """ +def pytest_generate_tests(metafunc): + if "value" in metafunc.fixturenames: + metafunc.parametrize("value", ["good", "bad"]) +""" + ) + result = echo.runpytest("--collect-only", "-q") + result.stdout.fnmatch_lines( + [ + "things.echo::alpha[[]good[]]", + "things.echo::alpha[[]bad[]]", + "things.echo::beta[[]good[]]", + "things.echo::beta[[]bad[]]", + ] + ) + result = echo.runpytest() + result.assert_outcomes(passed=2, failed=2) + result.stdout.fnmatch_lines(["*bad value in alpha[[]bad[]]*"]) + + +def test_params_are_delivered_on_the_callspec(echo: Pytester) -> None: + echo.makeconftest( + ECHO_PLUGIN + + """ +def pytest_generate_tests(metafunc): + metafunc.parametrize("value", ["good"]) + +def pytest_collection_modifyitems(items): + for item in items: + assert item.callspec.params == {"value": "good"} + assert item.spec == {"value": "good"} +""" + ) + echo.runpytest().assert_outcomes(passed=2) + + +def test_ids_and_marks_from_pytest_param(echo: Pytester) -> None: + echo.makeconftest( + ECHO_PLUGIN + + """ +import pytest + +def pytest_generate_tests(metafunc): + metafunc.parametrize( + "value", + [ + pytest.param("good", id="ok"), + pytest.param("bad", marks=pytest.mark.skip(reason="nope")), + ], + ) +""" + ) + result = echo.runpytest("-v") + result.assert_outcomes(passed=2, skipped=2) + result.stdout.fnmatch_lines(["*things.echo::alpha[[]ok[]] PASSED*"]) + + +def test_parametrize_of_undeclared_argname_is_rejected(echo: Pytester) -> None: + echo.makeconftest( + ECHO_PLUGIN + + """ +def pytest_generate_tests(metafunc): + metafunc.parametrize("nope", [1]) +""" + ) + result = echo.runpytest() + result.stdout.fnmatch_lines( + ["*In things.echo::alpha: definition uses no argument 'nope'*"] + ) + assert result.ret != 0 + + +def test_parametrize_marker_on_the_definition(echo: Pytester) -> None: + """Markers on the definition drive parametrization, as they do for Python.""" + echo.makeconftest( + ECHO_PLUGIN + + """ +import pytest + +def pytest_collectstart(collector): + if isinstance(collector, EchoDefinition): + collector.add_marker(pytest.mark.parametrize("value", ["good", "bad"])) +""" + ) + result = echo.runpytest() + result.assert_outcomes(passed=2, failed=2) + + +def test_item_selection_by_nodeid(echo: Pytester) -> None: + echo.makeconftest( + ECHO_PLUGIN + + """ +def pytest_generate_tests(metafunc): + metafunc.parametrize("value", ["good", "bad"]) +""" + ) + result = echo.runpytest("things.echo::alpha[good]") + result.assert_outcomes(passed=1) + + +def test_duplicate_parametrization_is_reported(echo: Pytester) -> None: + echo.makeconftest( + ECHO_PLUGIN + + """ +def pytest_generate_tests(metafunc): + metafunc.parametrize("value", ["a"]) + metafunc.parametrize("value", ["b"]) +""" + ) + result = echo.runpytest() + result.stdout.fnmatch_lines(["*duplicate parametrization of 'value'*"]) + assert result.ret != 0 + + +def test_definition_node_is_a_collector(echo: Pytester) -> None: + """The definition shows up in the tree between the file and the items.""" + result = echo.runpytest("--collect-only") + result.stdout.fnmatch_lines( + [ + "*", + "*", + "*", + ] + ) diff --git a/testing/test_scope.py b/testing/test_scope.py index 3cb811469a9..c5d10a73614 100644 --- a/testing/test_scope.py +++ b/testing/test_scope.py @@ -10,21 +10,24 @@ def test_ordering() -> None: assert Scope.Session > Scope.Package assert Scope.Package > Scope.Module assert Scope.Module > Scope.Class - assert Scope.Class > Scope.Function + assert Scope.Class > Scope.Definition + assert Scope.Definition > Scope.Function def test_next_lower() -> None: assert Scope.Session.next_lower() is Scope.Package assert Scope.Package.next_lower() is Scope.Module assert Scope.Module.next_lower() is Scope.Class - assert Scope.Class.next_lower() is Scope.Function + assert Scope.Class.next_lower() is Scope.Definition + assert Scope.Definition.next_lower() is Scope.Function with pytest.raises(ValueError, match="Function is the lower-most scope"): Scope.Function.next_lower() def test_next_higher() -> None: - assert Scope.Function.next_higher() is Scope.Class + assert Scope.Function.next_higher() is Scope.Definition + assert Scope.Definition.next_higher() is Scope.Class assert Scope.Class.next_higher() is Scope.Module assert Scope.Module.next_higher() is Scope.Package assert Scope.Package.next_higher() is Scope.Session