diff --git a/changelog/14769.bugfix.rst b/changelog/14769.bugfix.rst new file mode 100644 index 00000000000..4478b40285b --- /dev/null +++ b/changelog/14769.bugfix.rst @@ -0,0 +1,11 @@ +Selecting a parametrized test by its node id (``pytest test_x.py::test_a[1]``) +now works when the test definition is a node of the collection tree +(:confval:`collect_function_definition` set to ``pedantic`` or ``messy``). The +argument matcher compares names level by level, and the definition node carries +the bare name while the parametrization lives on the items below it; it now +looks through the definition instead of failing to match. + +Relatedly, skipping a non-Python item whose ``reportinfo()`` reports no line +number no longer crashes pytest with an ``INTERNALERROR``; the skip location is +reported without a line instead. This became easy to hit now that non-Python +definitions can carry ``pytest.param(..., marks=pytest.mark.skip(...))``. diff --git a/changelog/3926.feature.rst b/changelog/3926.feature.rst new file mode 100644 index 00000000000..ca44dc5c59e --- /dev/null +++ b/changelog/3926.feature.rst @@ -0,0 +1,32 @@ +Added the ``"definition"`` fixture scope, which shares one fixture instance +across all invocations generated from a single test definition -- that is, +across the parameter sets of one parametrized test, but not between two +different tests: + +.. code-block:: python + + @pytest.fixture(scope="definition") + def resource(): ... + + + @pytest.mark.parametrize("n", [1, 2, 3]) + def test_it(resource, n): + # one `resource` for all three parameter sets + ... + +The scope needs the test definition to be a node of the collection tree, so it +requires :confval:`collect_function_definition` to be set to ``pedantic`` (or +``messy``). Using it without that fails with an explicit error rather than +silently degrading to function scope. + +``@pytest.mark.parametrize(..., scope="definition")`` is supported as well and +groups the invocations of one definition by parameter value. + +Added :class:`_pytest.nodes.ItemDefinition`, the collector-agnostic base for "a +definition which generates items". Subclassing it lets a non-Python collector +have its test definitions parametrized through :hook:`pytest_generate_tests`, +exactly like Python test functions: declare the accepted names in +``parametrize_argnames``, build one item per parameter set in ``make_item()``, +and the chosen values arrive on ``item.callspec.params``. Parameter ids, +``pytest.param(...)`` marks and node-id selection all work as usual; there is no +fixture resolution on that path. See :ref:`non-python parametrization`. diff --git a/changelog/3926.improvement.rst b/changelog/3926.improvement.rst new file mode 100644 index 00000000000..98f5503663d --- /dev/null +++ b/changelog/3926.improvement.rst @@ -0,0 +1,15 @@ +Added the :confval:`collect_function_definition` option, which can promote the +internal ``FunctionDefinition`` to a real node in the collection tree. + +When set to ``pedantic``, a ``FunctionDefinition`` collector node is inserted +between a :class:`~pytest.Module`/:class:`~pytest.Class` and its test functions, +so that the (possibly parametrized) invocations of each test are collected +underneath a shared definition node instead of as flat siblings. Function-level +markers are scoped to the definition node in this mode. + +The default ``hidden`` keeps the previous flat layout, and the transitional +``messy`` value keeps the definition node out of marker resolution for code that +is not yet prepared for the new scope. + +The node ids of the individual test invocations are unchanged in all modes, so +test selection, caching and reporting behave exactly as before. diff --git a/changelog/3926.misc.rst b/changelog/3926.misc.rst new file mode 100644 index 00000000000..31cce0a7e6a --- /dev/null +++ b/changelog/3926.misc.rst @@ -0,0 +1,7 @@ +Moved the parametrization core out of ``_pytest.python`` into the new +``_pytest.parametrize`` module: ``CallSpec``, ``IdMaker`` and the new +``ParametrizeContext``, of which :class:`pytest.Metafunc` is now the +Python-specific subclass adding the fixture closure, signature validation and +direct-parameter desugaring. + +``_pytest.python.CallSpec2`` keeps warning and resolving as before. diff --git a/doc/en/example/nonpython.rst b/doc/en/example/nonpython.rst index 9ac2b508ceb..06341428e9e 100644 --- a/doc/en/example/nonpython.rst +++ b/doc/en/example/nonpython.rst @@ -105,3 +105,65 @@ interesting to look at the collection tree: ======================== 2 tests collected in 0.12s ======================== + +.. _`non-python parametrization`: + +Letting non-python tests take part in parametrization +----------------------------------------------------- + +.. versionadded:: 9.2 + +A collector which yields items directly, as ``YamlFile`` above does, produces a +fixed set of tests. To let a test *definition* be parametrized -- by +:hook:`pytest_generate_tests`, exactly like a Python test function -- collect +:class:`~_pytest.nodes.ItemDefinition` nodes instead of items. + +Such a node declares the names it accepts in +:attr:`~_pytest.nodes.ItemDefinition.parametrize_argnames` and builds one item +per parameter set in :meth:`~_pytest.nodes.ItemDefinition.make_item`. The values +chosen for a given item arrive on ``item.callspec.params``: + +.. code-block:: python + + # content of conftest.py + from _pytest import nodes + import pytest + + + class YamlItem(pytest.Item): + def __init__(self, *, spec, **kwargs): + super().__init__(**kwargs) + self.spec = spec + + def runtest(self): ... + + def reportinfo(self): + return self.path, 0, self.name + + + class YamlDefinition(nodes.ItemDefinition): + parametrize_argnames = ("value",) + + def make_item(self, parent, *, name, callspec, nodeid, context): + params = dict(callspec.params) if callspec is not None else {} + return YamlItem.from_parent(parent, name=name, nodeid=nodeid, spec=params) + +Any ``pytest_generate_tests`` implementation now applies: + +.. code-block:: python + + def pytest_generate_tests(metafunc): + if "value" in metafunc.fixturenames: + metafunc.parametrize("value", ["good", "bad"]) + +which collects ``test_simple.yaml::hello[good]`` and +``test_simple.yaml::hello[bad]`` under a single ```` node. +Parameter ids, ``pytest.param(...)`` marks and node-id selection all work as +they do for Python tests. + +.. note:: + + This path covers parametrization only: there is no fixture resolution for + non-Python items, so ``indirect=True`` and requesting fixtures from such an + item are not supported. A ``parametrize()`` call naming anything outside + ``parametrize_argnames`` is rejected at collection time. diff --git a/doc/en/how-to/fixtures.rst b/doc/en/how-to/fixtures.rst index 0ffc0778f9f..073c78b405e 100644 --- a/doc/en/how-to/fixtures.rst +++ b/doc/en/how-to/fixtures.rst @@ -389,7 +389,16 @@ to cause a ``smtp_connection`` fixture function, responsible to create a connect once per test *module* (the default is to invoke once per test *function*). Multiple test functions in a test module will thus each receive the same ``smtp_connection`` fixture instance, thus saving time. -Possible values for ``scope`` are: ``function``, ``class``, ``module``, ``package`` or ``session``. +Possible values for ``scope`` are: ``function``, ``definition``, ``class``, +``module``, ``package`` or ``session``. + +.. note:: + + ``definition`` shares one fixture instance across all the invocations + generated from a single test definition -- that is, across the parameter + sets of one parametrized test, but not between two different tests. It + requires the definition to be a node of the collection tree, see + :confval:`collect_function_definition`. The next example puts the fixture function into a separate ``conftest.py`` file so that tests from multiple test modules in the directory can diff --git a/doc/en/proposals/generic_fixture_closures.rst b/doc/en/proposals/generic_fixture_closures.rst new file mode 100644 index 00000000000..e116117334e --- /dev/null +++ b/doc/en/proposals/generic_fixture_closures.rst @@ -0,0 +1,143 @@ +:orphan: + +============================================ +PROPOSAL: Generic fixture closures for items +============================================ + +.. warning:: + + This document is the follow-up half of the ``ItemDefinition`` work: it + outlines what is needed for *non-Python* items to request fixtures. The + parametrization half has shipped, see :ref:`non-python parametrization`. + +What shipped +------------ + +:class:`~_pytest.nodes.ItemDefinition` lets any collector take part in +:hook:`pytest_generate_tests`. A definition declares the names it accepts up +front: + +.. code-block:: python + + class YamlDefinition(nodes.ItemDefinition): + parametrize_argnames = ("value",) + + def make_item(self, parent, *, name, callspec, nodeid, context): + return YamlItem.from_parent( + parent, name=name, nodeid=nodeid, spec=dict(callspec.params) + ) + +and the values chosen for each parameter set arrive on ``item.callspec.params``. + +``ParametrizeContext`` deliberately has no fixture machinery: ``_infer_scope()`` +returns ``Scope.Function``, ``_register_direct_params()`` is a no-op, and +``_validate_argnames()`` checks against ``parametrize_argnames`` rather than a +fixture closure. ``FixtureManager.pytest_generate_tests`` returns early for a +context which is not a ``Metafunc``. + +That covers parametrization. It does **not** cover: + +- ``indirect=True`` -- there is no fixture to hand ``request.param`` to. +- scope inference from fixture scopes -- an unspecified ``scope=`` is always + function scope. +- a non-Python item requesting a fixture at all, including one at the new + ``"definition"`` scope. + +Why the rest is not a small step +-------------------------------- + +Fixture resolution is written against ``Function``, not against ``Item``. The +concrete couplings: + +**TopRequest assumes a Python function item.** +It is constructed as ``TopRequest(pyfuncitem=item)`` and reads +``item._fixtureinfo``, ``item.funcargs``, ``item.fixturenames`` and +``item.obj``. ``_fillfixtures()`` writes into ``item.funcargs``. +``request.instance`` walks to a :class:`~pytest.Class` node and calls +``newinstance()``. + +**getfixtureinfo() is item-scoped and signature-driven.** +Its closure starts from the function's argument names +(``getfuncargnames(func)``) plus autouse names. A non-Python definition has no +signature; its "argnames" are whatever it declares. ``FunctionDefinition`` +already has to ``cast(nodes.Item, self)`` to call it -- the same wart, one level +up. + +**Direct parametrization desugars into DirectParamFixtureDef.** +That is how a direct param reaches the test at setup time. Without a fixture +system there is nothing to desugar into, which is why the generic path delivers +params on the callspec instead. + +**Function.setup() drives everything.** +``nodes.Item.setup()`` is a no-op; the fixture lifecycle only runs because +``Function.setup()`` calls ``self._request._fillfixtures()``. + +Proposed shape +-------------- + +Three steps, each independently useful. + +1. Split a node-level closure entry point out of ``getfixtureinfo()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Today ``getfixtureinfo(node, func, cls, ignore_args)`` derives ``initialnames`` +from ``func``'s signature. Proposed: + +.. code-block:: python + + def getfixtureinfo_for_argnames( + self, node: nodes.Node, argnames: Sequence[str], *, ignore_args=() + ) -> FuncFixtureInfo: ... + +with the current signature-based version becoming a thin wrapper which computes +``argnames`` from the function. This removes the ``cast(nodes.Item, self)`` in +``FunctionDefinition`` as a side effect, and gives ``ItemDefinition`` a closure +without inventing a fake function. + +2. Make the request object item-generic +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Introduce a small protocol for what ``TopRequest`` needs from its item -- +``fixturenames``, somewhere to store the computed values, ``_fixtureinfo``, and +an optional ``instance`` -- and give :class:`~pytest.Item` a default +implementation. :class:`~pytest.Function` keeps its current behaviour +(``funcargs``, bound instance); a generic item gets a plain dict and +``instance is None``. + +The open question here is ``request.instance``, whose meaning is genuinely +Python-specific. Returning ``None`` for non-Python items is the honest answer, +but fixtures in the wild do use ``request.instance`` unconditionally. + +3. Opt in per definition +~~~~~~~~~~~~~~~~~~~~~~~~ + +Fixture support should not be forced on every ``ItemDefinition``; a collector +which only wants parametrization should not pay for closure computation. +Suggested switch: + +.. code-block:: python + + class ItemDefinition(Collector, abc.ABC): + #: Whether items generated from this definition can request fixtures. + supports_fixtures: bool = False + +With ``supports_fixtures = True``, ``make_parametrize_context()`` computes a +closure from ``parametrize_argnames`` via step 1 and returns a fixture-aware +context, so ``indirect=True``, scope inference and ``"definition"`` scoped +fixtures all work uniformly. ``FixtureManager.pytest_generate_tests`` would then +key off the context being fixture-aware rather than off +``isinstance(metafunc, Metafunc)``. + +Open questions +-------------- + +- Does ``Metafunc`` stay a distinct class once the generic context is + fixture-aware, or does it collapse into ``ParametrizeContext`` plus the + ``module``/``cls``/``function`` attributes? +- ``request.instance`` for non-Python items: ``None``, or an error on access? +- Autouse fixtures currently apply to every item in a subtree. Should they apply + to non-Python items which opted into fixtures, or only to those which declare + the corresponding names? +- Should ``parametrize_argnames`` and the fixture closure be the same list, or + should a definition be able to accept a name for parametrization which is not + requestable as a fixture? diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 4f2e0b70850..a3fc26c8ffe 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -937,6 +937,13 @@ Function :members: :show-inheritance: +ItemDefinition +~~~~~~~~~~~~~~ + +.. autoclass:: _pytest.nodes.ItemDefinition() + :members: + :show-inheritance: + FunctionDefinition ~~~~~~~~~~~~~~~~~~ @@ -963,6 +970,12 @@ CallInfo .. autoclass:: pytest.CallInfo() :members: +CallSpec +~~~~~~~~ + +.. autoclass:: _pytest.parametrize.CallSpec() + :members: + CollectReport ~~~~~~~~~~~~~ @@ -1036,6 +1049,13 @@ Metafunc .. autoclass:: pytest.Metafunc() :members: + :inherited-members: + +ParametrizeContext +~~~~~~~~~~~~~~~~~~ + +.. autoclass:: _pytest.parametrize.ParametrizeContext() + :members: Parser ~~~~~~ @@ -1378,6 +1398,79 @@ passed multiple times. The expected format is ``name=value``. For example:: variables, that will be expanded. For more information about cache plugin please refer to :ref:`cache_provider`. +.. confval:: collect_function_definition + :type: ``string`` + :default: ``hidden`` + + .. versionadded:: 9.2 + + Controls whether a ``FunctionDefinition`` collector node is inserted into the + collection tree between a :class:`~pytest.Module`/:class:`~pytest.Class` and + its test functions, grouping the (possibly parametrized) invocations of a test + function *underneath* the definition node instead of as flat siblings. + + It accepts three values: + + ``hidden`` (default) + Keep the flat layout. The definition node is used only internally to drive + parametrization and is kept out of the collection tree. + + ``pedantic`` + Insert the definition node with correct marker scoping: function-level + markers (such as ``@pytest.mark.parametrize`` or custom marks) belong to + the definition node, and each invocation only owns its own parameter-set + markers. This is the intended target layout. + + ``messy`` + A stopgap for migration only. It inserts the definition node but transfers + the function-level markers back down onto each invocation, reproducing the + old (duplicated) marker layout so that code which is not yet prepared for + the definition scope keeps working. It emits a warning in the session + header and exists purely to buy time; fix the offending code and move to + ``pedantic``. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + collect_function_definition = "pedantic" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + collect_function_definition = pedantic + + For example, given a parametrized test: + + .. code-block:: python + + import pytest + + + @pytest.mark.parametrize("x", [1, 2]) + def test_it(x): ... + + the default (``hidden``) layout collects the invocations directly under the + module:: + + + + + + while ``pedantic`` and ``messy`` group them under a definition node:: + + + + + + + This is an opt-in structural change. The node ids of the individual test + invocations are unchanged (for example ``test_it.py::test_it[1]``), so test + selection, caching and reporting behave exactly as before. + .. confval:: collect_imported_tests :type: ``bool`` :default: ``true`` @@ -3677,6 +3770,14 @@ All the command-line flags can also be obtained by running ``pytest --help``:: collect_imported_tests (bool): Whether to collect tests in imported modules outside `testpaths` + collect_function_definition (string): + How the function-definition collector node + participates in the collection tree. - hidden + (default): keep the flat layout, no node in the tree + - pedantic: insert the node and scope function-level + markers to it - messy: insert the node but transfer + markers down to each invocation to preserve the + legacy marker layout (emits a warning) consider_namespace_packages (bool): Consider namespace packages when resolving module names during import diff --git a/src/_pytest/fixtures.py b/src/_pytest/fixtures.py index 1b1591d1f83..ff0aac0e416 100644 --- a/src/_pytest/fixtures.py +++ b/src/_pytest/fixtures.py @@ -25,7 +25,6 @@ from typing import Final from typing import final from typing import Generic -from typing import Literal from typing import NoReturn from typing import overload from typing import TYPE_CHECKING @@ -70,6 +69,7 @@ from _pytest.outcomes import fail from _pytest.outcomes import skip from _pytest.outcomes import TEST_OUTCOME +from _pytest.parametrize import _resolve_args_directness from _pytest.pathlib import absolutepath from _pytest.pathlib import bestrelpath from _pytest.scope import HIGH_SCOPES @@ -84,9 +84,9 @@ if TYPE_CHECKING: - from _pytest.python import CallSpec + from _pytest.parametrize import CallSpec + from _pytest.parametrize import ParametrizeContext from _pytest.python import Function - from _pytest.python import Metafunc from _pytest.reports import CollectReport @@ -136,6 +136,23 @@ def get_scope_package( return node.session +def fail_definition_scope_unavailable(nodeid: str, what: str) -> NoReturn: + """Fail because the ``"definition"`` scope was used where no + :class:`~_pytest.nodes.ItemDefinition` node exists to anchor it on. + + Unlike the other scopes, ``"definition"`` needs the test definition to be a + node of the collection tree; there is nothing to fall back to that would + preserve its meaning of "shared by all invocations of one definition". + """ + fail( + f"ScopeUnavailable: {what} needs the 'definition' scope, but {nodeid} " + f"has no definition node in the collection tree.\n" + f"For Python tests, set the collect_function_definition ini option to " + f"'pedantic' to make function definitions part of the tree.", + pytrace=False, + ) + + def is_visibility_more_specific( candidate: FixtureDef[Any], other: FixtureDef[Any] ) -> bool: @@ -174,6 +191,9 @@ def get_scope_node(node: nodes.Node, scope: Scope) -> nodes.Node | None: # Type ignored because this is actually safe, see: # https://github.com/python/mypy/issues/4717 return node.getparent(nodes.Item) # type: ignore[type-abstract] + elif scope is Scope.Definition: + # Type ignored for the same reason as Scope.Function above. + return node.getparent(nodes.ItemDefinition) # type: ignore[type-abstract] elif scope is Scope.Class: return node.getparent(_pytest.python.Class) elif scope is Scope.Module: @@ -277,6 +297,9 @@ class ParamArgKey: scoped_item_path: Path | None #: For Class scope, the class where the item is defined. item_cls: type | None + #: For Definition scope, the nodeid of the definition the item was + #: generated from. + scoped_definition_id: str | None = None _V = TypeVar("_V") @@ -293,6 +316,7 @@ def get_param_argkeys(item: nodes.Item, scope: Scope) -> Iterator[ParamArgKey]: return item_cls = None + scoped_definition_id = None if scope is Scope.Session: scoped_item_path = None elif scope is Scope.Package: @@ -302,15 +326,28 @@ def get_param_argkeys(item: nodes.Item, scope: Scope) -> Iterator[ParamArgKey]: scoped_item_path = item.path elif scope is Scope.Class: scoped_item_path = item.path - item_cls = item.cls # type: ignore[attr-defined] + # Items which are not Python test functions have no class. + item_cls = getattr(item, "cls", None) + elif scope is Scope.Definition: + scoped_item_path = item.path else: assert_never(scope) for argname, param in callspec.params.items(): if callspec._arg2scope[argname] != scope: continue + if scope is Scope.Definition and scoped_definition_id is None: + # Resolved only once a definition-scoped param is actually found: + # items without one need not have a definition node at all. A param + # at this scope can only have been created while the definition was + # a node, so the lookup cannot come up empty here. + definition = get_scope_node(item, Scope.Definition) + assert definition is not None + scoped_definition_id = definition.nodeid param_key = ParamValueKey(param, callspec.indices[argname]) - yield ParamArgKey(argname, param_key, scoped_item_path, item_cls) + yield ParamArgKey( + argname, param_key, scoped_item_path, item_cls, scoped_definition_id + ) def reorder_items(items: Sequence[nodes.Item]) -> list[nodes.Item]: @@ -546,7 +583,8 @@ def _scope(self) -> Scope: @property def scope(self) -> ScopeName: - """Scope string, one of "function", "class", "module", "package", "session".""" + """Scope string, one of "function", "definition", "class", "module", + "package", "session".""" return self._scope.value @abc.abstractmethod @@ -908,6 +946,10 @@ def __init__( if node is None and scope is Scope.Class: # Fallback to function item itself. node = self._pyfuncitem + if node is None and scope is Scope.Definition: + fail_definition_scope_unavailable( + self._pyfuncitem.nodeid, f"fixture {fixturedef.argname!r}" + ) assert node, ( f'Could not obtain a node for scope "{scope}" for function {self._pyfuncitem!r}' ) @@ -1193,7 +1235,8 @@ def __init__( @property def scope(self) -> ScopeName: - """Scope string, one of "function", "class", "module", "package", "session".""" + """Scope string, one of "function", "definition", "class", "module", + "package", "session".""" return self._scope.value @property @@ -1556,7 +1599,13 @@ def fixture( :param scope: The scope for which this fixture is shared; one of ``"function"`` - (default), ``"class"``, ``"module"``, ``"package"`` or ``"session"``. + (default), ``"definition"``, ``"class"``, ``"module"``, ``"package"`` + or ``"session"``. + + ``"definition"`` shares the fixture across all invocations generated + from one test definition, i.e. across the parameter sets of a + parametrized test. It requires the definition to be a node of the + collection tree, see :confval:`collect_function_definition`. This parameter may also be a callable which receives ``(fixture_name, config)`` as parameters, and must return a ``str`` with one of the values mentioned above. @@ -1683,45 +1732,6 @@ def pytest_cmdline_main(config: Config) -> int | ExitCode | None: return None -def _resolve_args_directness( - argnames: Sequence[str], - indirect: bool | Sequence[str], - nodeid: str, -) -> dict[str, Literal["indirect", "direct"]]: - """Resolve if each parametrized argument must be considered an indirect - parameter to a fixture of the same name, or a direct parameter to the - parametrized function, based on the ``indirect`` parameter of the - parametrize() call. - - :param argnames: - List of argument names passed to ``parametrize()``. - :param indirect: - Same as the ``indirect`` parameter of ``parametrize()``. - :param nodeid: - Node ID to which the parametrization is applied. - :returns: - A dict mapping each arg name to either "indirect" or "direct". - """ - arg_directness: dict[str, Literal["indirect", "direct"]] - if isinstance(indirect, bool): - arg_directness = dict.fromkeys(argnames, "indirect" if indirect else "direct") - elif isinstance(indirect, Sequence): - arg_directness = dict.fromkeys(argnames, "direct") - for arg in indirect: - if arg not in argnames: - fail( - f"In {nodeid}: indirect fixture '{arg}' doesn't exist", - pytrace=False, - ) - arg_directness[arg] = "indirect" - else: - fail( - f"In {nodeid}: expected Sequence or boolean for indirect, got {type(indirect).__name__}", - pytrace=False, - ) - return arg_directness - - def _get_direct_parametrize_args(node: nodes.Node) -> set[str]: """Return all direct parametrization arguments of a node, so we don't mistake them for fixtures. @@ -1995,8 +2005,14 @@ def sort_by_scope(arg_name: str) -> Scope: return fixturenames_closure, arg2fixturedefs - def pytest_generate_tests(self, metafunc: Metafunc) -> None: + def pytest_generate_tests(self, metafunc: ParametrizeContext) -> None: """Generate new tests based on parametrized fixtures used by the given metafunc""" + from _pytest.python import Metafunc + + if not isinstance(metafunc, Metafunc): + # The definition's parametrization is not backed by fixtures, so + # there are no parametrized fixtures to expand. + return def get_parametrize_mark_argnames(mark: Mark) -> Sequence[str]: args, _ = ParameterSet._parse_parametrize_args(*mark.args, **mark.kwargs) diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 1b337e20c7e..08218f07ecc 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -251,6 +251,17 @@ def pytest_addoption(parser: Parser) -> None: type="bool", default=True, ) + parser.addini( + "collect_function_definition", + "How the function-definition collector node participates in the " + "collection tree.\n" + "- hidden (default): keep the flat layout, no node in the tree\n" + "- pedantic: insert the node and scope function-level markers to it\n" + "- messy: insert the node but transfer markers down to each invocation " + "to preserve the legacy marker layout (emits a warning)", + type="string", + default="hidden", + ) parser.addini( "consider_namespace_packages", type="bool", @@ -1007,6 +1018,7 @@ def collect(self) -> Iterator[nodes.Item | nodes.Collector]: # Prune this level. any_matched_in_collector = False for node in reversed(subnodes): + remaining = matchparts[1:] # Path part e.g. `/a/b/` in `/a/b/test_file.py::TestIt::test_it`. if isinstance(matchparts[0], Path): is_match = node.path == matchparts[0] @@ -1021,7 +1033,13 @@ def collect(self) -> Iterator[nodes.Item | nodes.Collector]: else: if len(matchparts) == 1: # This the last part, one parametrization goes. - if parametrization is not None: + if isinstance(node, nodes.ItemDefinition): + # A definition node carries the bare name, while + # the parametrization is on the items below it. + # Descend and match the same part again there. + is_match = node.name == matchparts[0] + remaining = matchparts + elif parametrization is not None: # A parametrized arg must match exactly. is_match = node.name == matchparts[0] + parametrization else: @@ -1032,7 +1050,7 @@ def collect(self) -> Iterator[nodes.Item | nodes.Collector]: else: is_match = node.name == matchparts[0] if is_match: - work.append((node, matchparts[1:])) + work.append((node, remaining)) any_matched_in_collector = True if not any_matched_in_collector: diff --git a/src/_pytest/nodes.py b/src/_pytest/nodes.py index f0629c2daf7..49e1df379e4 100644 --- a/src/_pytest/nodes.py +++ b/src/_pytest/nodes.py @@ -6,6 +6,7 @@ from collections.abc import Iterable from collections.abc import Iterator from collections.abc import MutableMapping +from collections.abc import Sequence from functools import cached_property from functools import lru_cache import os @@ -35,6 +36,8 @@ from _pytest.mark.structures import MarkDecorator from _pytest.mark.structures import NodeKeywords from _pytest.outcomes import fail +from _pytest.parametrize import CallSpec +from _pytest.parametrize import ParametrizeContext from _pytest.pathlib import absolutepath from _pytest.stash import Stash from _pytest.warning_types import PytestWarning @@ -762,3 +765,138 @@ def location(self) -> tuple[str, int | None, str]: relfspath = self.session._node_location_to_relpath(path) assert type(location[2]) is str return (relfspath, location[1], location[2]) + + +class ItemDefinition(Collector, abc.ABC): + """Base class for collectors standing for a single test *definition*. + + Its children are the (possibly parametrized) items generated from that one + definition. Giving the definition a node of its own makes it addressable: + it owns the definition's markers, it is what :hook:`pytest_generate_tests` + parametrizes, and it is the node ``"definition"`` scoped fixtures are + cached on, shared by every invocation of the definition. + + :class:`~_pytest.python.FunctionDefinition` is the implementation for + Python test functions. Other collectors can subclass this to let their test + definitions be parametrized through the same hook: declare the names that + may be parametrized in :attr:`parametrize_argnames`, and turn one callspec + into an item in :meth:`make_item`. There is no fixture resolution on that + path -- the chosen values arrive on ``item.callspec.params``. + + .. versionadded:: 9.2 + + :ref:`non-python tests`. + """ + + #: The argument names this definition may be parametrized on. + #: + #: A ``parametrize()`` call naming anything else is rejected, mirroring the + #: "uses no argument" error raised for Python test functions. + parametrize_argnames: Sequence[str] = () + + @abc.abstractmethod + def make_item( + self, + parent: Collector, + *, + name: str, + callspec: CallSpec | None, + nodeid: str, + context: ParametrizeContext, + ) -> Item: + """Build the item for a single invocation of this definition. + + :param parent: + The collector to attach the item to. Usually the definition itself, + but not when the definition is kept out of the collection tree (see + :confval:`collect_function_definition`). + :param name: + Name for the item: the definition's name, with the callspec id + appended when parametrized. + :param callspec: + The parametrization of this invocation, or ``None`` when the + definition is not parametrized. + :param nodeid: + The nodeid to give the item. Passed explicitly so that item nodeids + stay the same whether or not the definition is part of the tree. + :param context: + The context the parametrization was collected in. + """ + raise NotImplementedError + + def make_parametrize_context(self) -> ParametrizeContext: + """Create the object to hand to :hook:`pytest_generate_tests`.""" + return ParametrizeContext( + self, + self.config, + argnames=self.parametrize_argnames, + _ispytest=True, + ) + + def parametrize_hook_extras(self) -> Sequence[Callable[..., object]]: + """Extra :hook:`pytest_generate_tests` implementations to call. + + Used by the Python collector to pick up ``pytest_generate_tests`` + functions defined in the test's own module or class. + """ + return () + + def finalize_parametrization(self, context: ParametrizeContext) -> None: + """Called once the hook has run, before any item is built. + + Only called when the definition ended up parametrized. + """ + context._recompute_direct_params_indices() + + def collect(self) -> Iterable[Item | Collector]: + return list(self.generate_items(self)) + + def generate_items(self, parent: Collector) -> Iterator[Item]: + """Run the generate-tests protocol, yielding the items under ``parent``. + + ``parent`` is the definition itself when it is part of the collection + tree, and the collector containing it otherwise. + """ + context = self.make_parametrize_context() + self.ihook.pytest_generate_tests.call_extra( + list(self.parametrize_hook_extras()), dict(metafunc=context) + ) + + # The items keep a flat nodeid anchored at the collector containing the + # definition, whether or not the definition itself is part of the tree, + # so that nodeids -- and with them selection, caching and reporting -- + # do not depend on the shape of the tree. + assert self.parent is not None + base_nodeid = self.parent.nodeid + name = self.name + + if not context._calls: + yield self.make_item( + parent, + name=name, + callspec=None, + nodeid=f"{base_nodeid}::{name}", + context=context, + ) + else: + self.finalize_parametrization(context) + for callspec in context._calls: + subname = f"{name}[{callspec.id}]" if callspec._idlist else name + item = self.make_item( + parent, + name=subname, + callspec=callspec, + nodeid=f"{base_nodeid}::{subname}", + context=context, + ) + if getattr(item, "callspec", None) is None: + # Attaching the parametrization is part of the protocol, not + # of each make_item(): the callspec drives reordering and + # high-scoped param caching, and its marks and id have to + # reach the item for markers and -k to work. Skipped when + # make_item() already did it -- Function does, because it + # can also be constructed directly. + item.callspec = callspec # type: ignore[attr-defined] + item.own_markers.extend(callspec.marks) + item.keywords[callspec.id] = True + yield item diff --git a/src/_pytest/parametrize.py b/src/_pytest/parametrize.py new file mode 100644 index 00000000000..09cbbe58b6e --- /dev/null +++ b/src/_pytest/parametrize.py @@ -0,0 +1,809 @@ +"""The parametrization core: turning ``parametrize()`` calls into callspecs. + +This machinery is deliberately independent of the Python test collector, so +that any :class:`~_pytest.nodes.ItemDefinition` -- not only +:class:`~_pytest.python.FunctionDefinition` -- can be parametrized through +:hook:`pytest_generate_tests`. + +:class:`~_pytest.python.Metafunc` is the Python-specific subclass of +:class:`ParametrizeContext` and remains the object plugins receive for Python +tests. ``CallSpec``, ``IdMaker`` and friends live here rather than in +``python.py`` for the same reason, and are re-exported from there. +""" + +from __future__ import annotations + +from collections import Counter +from collections import defaultdict +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Mapping +from collections.abc import Sequence +import dataclasses +import enum +import hashlib +import itertools +import re +import textwrap +from typing import Any +from typing import cast +from typing import final +from typing import get_args +from typing import Literal +from typing import NoReturn +from typing import TYPE_CHECKING + +from _pytest._io.saferepr import saferepr +from _pytest.compat import ascii_escaped +from _pytest.compat import NOTSET +from _pytest.config import Config +from _pytest.config import UsageError +from _pytest.deprecated import check_ispytest +from _pytest.mark.structures import _HiddenParam +from _pytest.mark.structures import HIDDEN_PARAM +from _pytest.mark.structures import Mark +from _pytest.mark.structures import MarkDecorator +from _pytest.mark.structures import normalize_mark_list +from _pytest.mark.structures import ParameterSet +from _pytest.outcomes import fail +from _pytest.scope import Scope +from _pytest.scope import ScopeName + + +if TYPE_CHECKING: + from _pytest import nodes + from _pytest.fixtures import FixtureDef + + +LongStrIdStrategy = Literal["short", "sha256", "legacy", "disallow"] +_LONG_STR_STRATEGIES: frozenset[LongStrIdStrategy] = frozenset( + get_args(LongStrIdStrategy) +) + + +def _collect_error(msg: str) -> Exception: + """Build a ``Collector.CollectError``. + + ``nodes`` imports this module (for :class:`~_pytest.nodes.ItemDefinition`), + so it can only be imported here lazily. + """ + from _pytest.nodes import Collector + + return Collector.CollectError(msg) + + +@final +@dataclasses.dataclass(frozen=True) +class IdMaker: + """Make IDs for a parametrization.""" + + __slots__ = ( + "argnames", + "config", + "idfn", + "ids", + "nodeid", + "parametersets", + ) + + # The argnames of the parametrization. + argnames: Sequence[str] + # The ParameterSets of the parametrization. + parametersets: Sequence[ParameterSet] + # Optionally, a user-provided callable to make IDs for parameters in a + # ParameterSet. + idfn: Callable[[Any], object | None] | None + # Optionally, explicit IDs for ParameterSets by index. + ids: Sequence[object | None] | None + # Optionally, the pytest config. + # Used for controlling ASCII escaping, determining parametrization ID + # strictness, and for calling the :hook:`pytest_make_parametrize_id` hook. + config: Config | None + # Optionally, the ID of the node being parametrized. + # Used only for clearer error messages. + nodeid: str | None + + def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]: + """Make a unique identifier for each ParameterSet, that may be used to + identify the parametrization in a node ID. + + If strict_parametrization_ids is enabled, and duplicates are detected, + raises CollectError. Otherwise makes the IDs unique as follows: + + Format is -...-[counter], where prm_x_token is + - user-provided id, if given + - else an id derived from the value, applicable for certain types + - else + The counter suffix is appended only in case a string wouldn't be unique + otherwise. + """ + resolved_ids = list(self._resolve_ids()) + # All IDs must be unique! + if len(resolved_ids) != len(set(resolved_ids)): + # Record the number of occurrences of each ID. + id_counts = Counter(resolved_ids) + + if self._strict_parametrization_ids_enabled(): + parameters = ", ".join(self.argnames) + parametersets = ", ".join( + [saferepr(list(param.values)) for param in self.parametersets] + ) + ids = ", ".join( + id if id is not HIDDEN_PARAM else "" for id in resolved_ids + ) + duplicates = ", ".join( + id if id is not HIDDEN_PARAM else "" + for id, count in id_counts.items() + if count > 1 + ) + msg = textwrap.dedent(f""" + Duplicate parametrization IDs detected, but strict_parametrization_ids is set. + + Test name: {self.nodeid} + Parameters: {parameters} + Parameter sets: {parametersets} + IDs: {ids} + Duplicates: {duplicates} + + You can fix this problem using `@pytest.mark.parametrize(..., ids=...)` or `pytest.param(..., id=...)`. + """).strip() # noqa: E501 + raise _collect_error(msg) + + # Map the ID to its next suffix. + id_suffixes: dict[str, int] = defaultdict(int) + # Suffix non-unique IDs to make them unique. + for index, id in enumerate(resolved_ids): + if id_counts[id] > 1: + if id is HIDDEN_PARAM: + self._complain_multiple_hidden_parameter_sets() + suffix = "" + if id and id[-1].isdigit(): + suffix = "_" + new_id = f"{id}{suffix}{id_suffixes[id]}" + while new_id in set(resolved_ids): + id_suffixes[id] += 1 + new_id = f"{id}{suffix}{id_suffixes[id]}" + resolved_ids[index] = new_id + id_suffixes[id] += 1 + assert len(resolved_ids) == len(set(resolved_ids)), ( + f"Internal error: {resolved_ids=}" + ) + return resolved_ids + + def _strict_parametrization_ids_enabled(self) -> bool: + if self.config is None: + return False + strict_parametrization_ids = self.config.getini("strict_parametrization_ids") + if strict_parametrization_ids is None: + strict_parametrization_ids = self.config.getini("strict") + return cast(bool, strict_parametrization_ids) + + def _resolve_ids(self) -> Iterable[str | _HiddenParam]: + """Resolve IDs for all ParameterSets (may contain duplicates).""" + for idx, parameterset in enumerate(self.parametersets): + if parameterset.id is not None: + # ID provided directly - pytest.param(..., id="...") + if parameterset.id is HIDDEN_PARAM: + yield HIDDEN_PARAM + else: + yield _ascii_escaped_by_config(parameterset.id, self.config) + elif self.ids and idx < len(self.ids) and self.ids[idx] is not None: + # ID provided in the IDs list - parametrize(..., ids=[...]). + if self.ids[idx] is HIDDEN_PARAM: + yield HIDDEN_PARAM + else: + yield self._idval_from_value_required(self.ids[idx], idx) + else: + # ID not provided - generate it. + yield "-".join( + self._idval(val, argname, idx) + for val, argname in zip( + parameterset.values, self.argnames, strict=True + ) + ) + + def _idval(self, val: object, argname: str, idx: int) -> str: + """Make an ID for a parameter in a ParameterSet.""" + idval = self._idval_from_function(val, argname, idx) + if idval is not None: + return idval + idval = self._idval_from_hook(val, argname) + if idval is not None: + return idval + if isinstance(val, str | bytes): + idval = self._apply_long_str_strategy(val, argname, idx) + if idval is not None: + return idval + else: + idval = self._idval_from_value(val) + if idval is not None: + return idval + return self._idval_from_argname(argname, idx) + + def _get_long_str_strategy(self) -> LongStrIdStrategy: + if not self.config: + return "short" + value = self.config.getini("parametrize_long_str_id_strategy") + if value not in _LONG_STR_STRATEGIES: + raise UsageError( + f"Unknown parametrize_long_str_id_strategy: {value!r}. " + f"Valid values: {', '.join(sorted(_LONG_STR_STRATEGIES))}" + ) + return cast(LongStrIdStrategy, value) + + def _apply_long_str_strategy( + self, val: str | bytes, argname: str, idx: int + ) -> str | None: + """Apply the configured strategy for long str/bytes parameter values. + + Only used for auto-generated IDs (not explicit ids=[...] or + pytest.param(id=...)). + """ + if len(val) <= 100: + return _ascii_escaped_by_config(val, self.config) + match self._get_long_str_strategy(): + case "legacy": + return _ascii_escaped_by_config(val, self.config) + case "short": + return None + case "sha256": + encoded = val.encode("utf-8") if isinstance(val, str) else val + return hashlib.sha256(encoded).hexdigest() + case "disallow": # pragma: no branch -- fail() raises, confuses coverage + prefix = self._make_error_prefix() + fail( + f"{prefix}parametrize value for '{argname}' at index {idx} " + f"is too long for an auto-generated ID ({len(val)} characters). " + f"Use pytest.param(..., id=...) or parametrize(..., ids=...) " + f"to set an explicit ID, or change parametrize_long_str_id_strategy.", + pytrace=False, + ) + + def _idval_from_function(self, val: object, argname: str, idx: int) -> str | None: + """Try to make an ID for a parameter in a ParameterSet using the + user-provided id callable, if given.""" + if self.idfn is None: + return None + try: + id = self.idfn(val) + except Exception as e: + prefix = f"{self.nodeid}: " if self.nodeid is not None else "" + msg = "error raised while trying to determine id of parameter '{}' at position {}" + msg = prefix + msg.format(argname, idx) + raise ValueError(msg) from e + if id is None: + return None + return self._idval_from_value(id) + + def _idval_from_hook(self, val: object, argname: str) -> str | None: + """Try to make an ID for a parameter in a ParameterSet by calling the + :hook:`pytest_make_parametrize_id` hook.""" + if self.config: + id: str | None = self.config.hook.pytest_make_parametrize_id( + config=self.config, val=val, argname=argname + ) + return id + return None + + def _idval_from_value(self, val: object) -> str | None: + """Try to make an ID for a parameter in a ParameterSet from its value, + if the value type is supported.""" + if isinstance(val, str | bytes): + return _ascii_escaped_by_config(val, self.config) + elif val is None or isinstance(val, float | int | bool | complex): + return str(val) + elif isinstance(val, re.Pattern): + return ascii_escaped(val.pattern) + elif val is NOTSET: + # Fallback to default. Note that NOTSET is an enum.Enum. + pass + elif isinstance(val, enum.Enum): + return str(val) + elif isinstance(getattr(val, "__name__", None), str): + # Name of a class, function, module, etc. + name: str = getattr(val, "__name__") + return name + return None + + def _idval_from_value_required(self, val: object, idx: int) -> str: + """Like _idval_from_value(), but fails if the type is not supported.""" + id = self._idval_from_value(val) + if id is not None: + return id + + # Fail. + prefix = self._make_error_prefix() + msg = ( + f"{prefix}ids contains unsupported value {saferepr(val)} (type: {type(val)!r}) at index {idx}. " + "Supported types are: str, bytes, int, float, complex, bool, enum, regex or anything with a __name__." + ) + fail(msg, pytrace=False) + + @staticmethod + def _idval_from_argname(argname: str, idx: int) -> str: + """Make an ID for a parameter in a ParameterSet from the argument name + and the index of the ParameterSet.""" + return str(argname) + str(idx) + + def _complain_multiple_hidden_parameter_sets(self) -> NoReturn: + fail( + f"{self._make_error_prefix()}multiple instances of HIDDEN_PARAM " + "cannot be used in the same parametrize call, " + "because the tests names need to be unique." + ) + + def _make_error_prefix(self) -> str: + if self.nodeid is not None: + return f"In {self.nodeid}: " + else: + return "" + + +@final +@dataclasses.dataclass(frozen=True) +class CallSpec: + """A planned parameterized invocation of a test function. + + Calculated during collection for a given test function's Metafunc. + Once collection is over, each callspec is turned into a single Item + and stored in item.callspec. + """ + + # arg name -> arg value which will be passed to a fixture of the same name. + params: dict[str, object] = dataclasses.field(default_factory=dict) + # arg name -> arg index. + indices: dict[str, int] = dataclasses.field(default_factory=dict) + # arg name -> parameter scope. + # Used for sorting parametrized resources. + _arg2scope: Mapping[str, Scope] = dataclasses.field(default_factory=dict) + # Parts which will be added to the item's name in `[..]` separated by "-". + _idlist: Sequence[str] = dataclasses.field(default_factory=tuple) + # Marks which will be applied to the item. + marks: list[Mark] = dataclasses.field(default_factory=list) + + def setmulti( + self, + *, + argnames: Iterable[str], + valset: Iterable[object], + id: str | _HiddenParam, + marks: Iterable[Mark | MarkDecorator], + scope: Scope, + param_index: int, + nodeid: str, + ) -> CallSpec: + params = self.params.copy() + indices = self.indices.copy() + arg2scope = dict(self._arg2scope) + for arg, val in zip(argnames, valset, strict=True): + if arg in params: + raise _collect_error(f"{nodeid}: duplicate parametrization of {arg!r}") + params[arg] = val + indices[arg] = param_index + arg2scope[arg] = scope + return CallSpec( + params=params, + indices=indices, + _arg2scope=arg2scope, + _idlist=self._idlist if id is HIDDEN_PARAM else [*self._idlist, id], + marks=[*self.marks, *normalize_mark_list(marks)], + ) + + def getparam(self, name: str) -> object: + try: + return self.params[name] + except KeyError as e: + raise ValueError(name) from e + + @property + def id(self) -> str: + return "-".join(self._idlist) + + +def _ascii_escaped_by_config(val: str | bytes, config: Config | None) -> str: + if config is None: + escape_option = False + else: + escape_option = config.getini( + "disable_test_id_escaping_and_forfeit_all_rights_to_community_support" + ) + # TODO: If escaping is turned off and the user passes bytes, + # will return a bytes. For now we ignore this but the + # code *probably* doesn't handle this case. + return val if escape_option else ascii_escaped(val) # type: ignore + + +def _infer_parametrize_scope( + argnames: Sequence[str], + arg2fixturedefs: Mapping[str, Sequence[FixtureDef[object]]], + indirect: bool | Sequence[str], +) -> Scope: + """Infer the most appropriate scope for a parametrize() call based on its + arguments, for when the scope is not explicitly specified. + + When there's at least one direct argument, always use "function" scope. + + When a test function is parametrized and all its arguments are indirect + (e.g. fixtures), return the most narrow scope based on the fixtures used. + + Related to issue #1832, based on code posted by @Kingdread. + """ + if isinstance(indirect, Sequence): + all_arguments_are_fixtures = len(indirect) == len(argnames) + else: + all_arguments_are_fixtures = bool(indirect) + + if all_arguments_are_fixtures: + # Takes the most narrow scope from used fixtures. + used_scopes = ( + # Higher scope can't request lower scope, so it's OK to only + # look at the first fixturedef in the override chain. + arg2fixturedefs[argname][-1]._scope + for argname in argnames + if argname in arg2fixturedefs + ) + return min(used_scopes, default=Scope.Function) + + return Scope.Function + + +def _resolve_args_directness( + argnames: Sequence[str], + indirect: bool | Sequence[str], + nodeid: str, +) -> dict[str, Literal["indirect", "direct"]]: + """Resolve if each parametrized argument must be considered an indirect + parameter to a fixture of the same name, or a direct parameter to the + parametrized function, based on the ``indirect`` parameter of the + parametrize() call. + + :param argnames: + List of argument names passed to ``parametrize()``. + :param indirect: + Same as the ``indirect`` parameter of ``parametrize()``. + :param nodeid: + Node ID to which the parametrization is applied. + :returns: + A dict mapping each arg name to either "indirect" or "direct". + """ + arg_directness: dict[str, Literal["indirect", "direct"]] + if isinstance(indirect, bool): + arg_directness = dict.fromkeys(argnames, "indirect" if indirect else "direct") + elif isinstance(indirect, Sequence): + arg_directness = dict.fromkeys(argnames, "direct") + for arg in indirect: + if arg not in argnames: + fail( + f"In {nodeid}: indirect fixture '{arg}' doesn't exist", + pytrace=False, + ) + arg_directness[arg] = "indirect" + else: + fail( + f"In {nodeid}: expected Sequence or boolean for indirect, got {type(indirect).__name__}", + pytrace=False, + ) + return arg_directness + + +class ParametrizeContext: + """The object passed to :hook:`pytest_generate_tests`. + + Accumulates :meth:`parametrize` calls into a list of :class:`CallSpec` + objects, which the owning :class:`~_pytest.nodes.ItemDefinition` then turns + into items. + + For Python tests this is subclassed by :class:`pytest.Metafunc`, which adds + the fixture-aware behaviour: scope inference from fixture scopes, indirect + parametrization and validation against the test function's signature. + + A non-Python definition can use this class as-is. It declares the names it + accepts up front (``argnames``), and the values chosen for them are handed + to the generated items on ``item.callspec.params``; there is no fixture + resolution involved. + """ + + def __init__( + self, + definition: nodes.ItemDefinition, + config: Config, + *, + argnames: Sequence[str] = (), + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + + #: The definition being parametrized. + self.definition = definition + + #: Access to the :class:`pytest.Config` object for the test session. + self.config = config + + #: The names this definition can be parametrized on. + #: + #: Named ``fixturenames`` rather than ``argnames`` because that is what + #: :hook:`pytest_generate_tests` implementations in the wild inspect -- + #: keeping the name is what lets them work with non-Python definitions. + #: + #: Stored by reference, so that a caller which later narrows the + #: sequence in place (as the Python fixture closure does) is reflected + #: here. + self.fixturenames: Sequence[str] = argnames + + # Result of parametrize(). + self._calls: list[CallSpec] = [] + + self._params_directness: dict[str, Literal["indirect", "direct"]] = {} + + # Seams for subclasses. The Python implementation (Metafunc) overrides + # these to bring fixtures and the test function's signature into play. + + @property + def _introspection_target(self) -> object: + """The object to introspect for a signature and a source location. + + Used when reporting an empty parameter set. Defaults to the definition + node, for which the source location degrades to "unknown"; Metafunc + returns the test function. + """ + return self.definition + + @property + def _target_name(self) -> str: + """Human readable name of what is being parametrized.""" + return self.definition.name + + def _infer_scope( + self, argnames: Sequence[str], indirect: bool | Sequence[str] + ) -> Scope: + """Determine the scope of a parametrize() call that did not specify one.""" + return Scope.Function + + def _validate_argnames( + self, argnames: Sequence[str], indirect: bool | Sequence[str] + ) -> None: + """Check that the definition actually accepts all of ``argnames``.""" + for argname in argnames: + if argname not in self.fixturenames: + kind = "fixture" if indirect else "argument" + fail( + f"In {self.definition.nodeid}: definition uses no {kind} '{argname}'", + pytrace=False, + ) + + def _register_direct_params( + self, + argnames: Sequence[str], + arg_directness: Mapping[str, Literal["indirect", "direct"]], + scope: Scope, + ) -> None: + """Hook for making the chosen params reachable at setup time. + + Only meaningful where parametrization feeds a fixture system; the + generic path delivers params on ``item.callspec.params`` instead. + """ + + def parametrize( + self, + argnames: str | Sequence[str], + argvalues: Iterable[ParameterSet | Sequence[object] | object], + *, + indirect: bool | Sequence[str] = False, + ids: Iterable[object | None] | Callable[[Any], object | None] | None = None, + scope: ScopeName | None = None, + _param_mark: Mark | None = None, + ) -> None: + """Add new invocations to the underlying test function using the list + of argvalues for the given argnames. Parametrization is performed + during the collection phase. If you need to setup expensive resources + see about setting ``indirect`` to do it at test setup time instead. + + Can be called multiple times per test function (but only on different + argument names), in which case each call parametrizes all previous + parametrizations, e.g. + + :: + + unparametrized: t + parametrize ["x", "y"]: t[x], t[y] + parametrize [1, 2]: t[x-1], t[x-2], t[y-1], t[y-2] + + :param argnames: + A comma-separated string denoting one or more argument names, or + a list/tuple of argument strings. + + :param argvalues: + The list of argvalues determines how often a test is invoked with + different argument values. + + If only one argname was specified argvalues is a list of values. + If N argnames were specified, argvalues must be a list of + N-tuples, where each tuple-element specifies a value for its + respective argname. + + .. versionchanged:: 9.1 + + Passing a non-:class:`~collections.abc.Collection` iterable + (such as a generator or iterator) is deprecated. See + :ref:`parametrize-iterators` for details. + + :param indirect: + A list of arguments' names (subset of argnames) or a boolean. + If True the list contains all names from the argnames. Each + argvalue corresponding to an argname in this list will + be passed as request.param to its respective argname fixture + function so that it can perform more expensive setups during the + setup phase of a test rather than at collection time. + + :param ids: + Sequence of (or generator for) ids for ``argvalues``, + or a callable to return part of the id for each argvalue. + + With sequences (and generators like ``itertools.count()``) the + returned ids should be of type ``string``, ``int``, ``float``, + ``bool``, or ``None``. + They are mapped to the corresponding index in ``argvalues``. + ``None`` means to use the auto-generated id. + + .. versionadded:: 8.4 + :ref:`hidden-param` means to hide the parameter set + from the test name. Can only be used at most 1 time, as + test names need to be unique. + + If it is a callable it will be called for each entry in + ``argvalues``, and the return value is used as part of the + auto-generated id for the whole set (where parts are joined with + dashes ("-")). + This is useful to provide more specific ids for certain items, e.g. + dates. Returning ``None`` will use an auto-generated id. + + If no ids are provided they will be generated automatically from + the argvalues. + + :param scope: + If specified it denotes the scope of the parameters. + The scope is used for grouping tests by parameter instances. + It will also override any fixture-function defined scope, allowing + to set a dynamic scope using test context or configuration. + + .. versionchanged:: 9.1 + + ``indirect``, ``ids`` and ``scope`` are now keyword-only. + """ + nodeid = self.definition.nodeid + + argnames, parametersets = ParameterSet._for_parametrize( + argnames, + argvalues, + self._introspection_target, + self.config, + nodeid=nodeid, + ) + del argvalues + + if "request" in argnames: + fail( + f"{nodeid}: 'request' is a reserved name and cannot be used in @pytest.mark.parametrize", + pytrace=False, + ) + + if scope is not None: + scope_ = Scope.from_user( + scope, descr=f"parametrize() call in {self._target_name}" + ) + else: + scope_ = self._infer_scope(argnames, indirect) + + self._validate_argnames(argnames, indirect) + + # Use any already (possibly) generated ids with parametrize Marks. + generated_ids = None + if _param_mark and _param_mark._param_ids_from: + generated_ids = _param_mark._param_ids_from._param_ids_generated + if generated_ids is not None: + ids = generated_ids + + ids = self._resolve_parameter_set_ids( + argnames, ids, parametersets, nodeid=nodeid + ) + + # Store used (possibly generated) ids with parametrize Marks. + if _param_mark and _param_mark._param_ids_from and generated_ids is None: + object.__setattr__(_param_mark._param_ids_from, "_param_ids_generated", ids) + + # Calculate directness. + arg_directness = _resolve_args_directness(argnames, indirect, nodeid) + self._params_directness.update(arg_directness) + + self._register_direct_params(argnames, arg_directness, scope_) + + # Create the new calls: if we are parametrize() multiple times (by applying the decorator + # more than once) then we accumulate those calls generating the cartesian product + # of all calls. + newcalls = [] + for callspec in self._calls or [CallSpec()]: + for param_index, (param_id, param_set) in enumerate( + zip(ids, parametersets, strict=True) + ): + newcallspec = callspec.setmulti( + argnames=argnames, + valset=param_set.values, + id=param_id, + marks=param_set.marks, + scope=scope_, + param_index=param_index, + nodeid=nodeid, + ) + newcalls.append(newcallspec) + self._calls = newcalls + + def _resolve_parameter_set_ids( + self, + argnames: Sequence[str], + ids: Iterable[object | None] | Callable[[Any], object | None] | None, + parametersets: Sequence[ParameterSet], + nodeid: str, + ) -> list[str | _HiddenParam]: + """Resolve the actual ids for the given parameter sets. + + :param argnames: + Argument names passed to ``parametrize()``. + :param ids: + The `ids` parameter of the ``parametrize()`` call (see docs). + :param parametersets: + The parameter sets, each containing a set of values corresponding + to ``argnames``. + :param nodeid str: + The nodeid of the definition item that generated this + parametrization. + :returns: + List with ids for each parameter set given. + """ + if ids is None: + idfn = None + ids_ = None + elif callable(ids): + idfn = ids + ids_ = None + else: + idfn = None + ids_ = self._validate_ids(ids, parametersets) + id_maker = IdMaker( + argnames, + parametersets, + idfn, + ids_, + self.config, + nodeid=nodeid, + ) + return id_maker.make_unique_parameterset_ids() + + def _validate_ids( + self, + ids: Iterable[object | None], + parametersets: Sequence[ParameterSet], + ) -> list[object | None]: + try: + num_ids = len(ids) # type: ignore[arg-type] + except TypeError: + try: + iter(ids) + except TypeError as e: + raise TypeError("ids must be a callable or an iterable") from e + num_ids = len(parametersets) + + # num_ids == 0 is a special case: https://github.com/pytest-dev/pytest/issues/1849 + if num_ids != len(parametersets) and num_ids != 0: + nodeid = self.definition.nodeid + fail( + f"In {nodeid}: {len(parametersets)} parameter sets specified, with different number of ids: {num_ids}", + pytrace=False, + ) + + return list(itertools.islice(ids, num_ids)) + + def _recompute_direct_params_indices(self) -> None: + for argname, param_type in self._params_directness.items(): + if param_type == "direct": + for i, callspec in enumerate(self._calls): + callspec.indices[argname] = i diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 4f168012c36..f71fa55bc97 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -4,32 +4,23 @@ from __future__ import annotations import abc -from collections import Counter -from collections import defaultdict from collections.abc import Callable from collections.abc import Generator from collections.abc import Iterable from collections.abc import Iterator from collections.abc import Mapping from collections.abc import Sequence -import dataclasses -import enum import fnmatch from functools import partial -import hashlib import inspect -import itertools import os from pathlib import Path -import re -import textwrap import types from typing import Any from typing import cast from typing import final from typing import get_args from typing import Literal -from typing import NoReturn from typing import TYPE_CHECKING import warnings @@ -41,8 +32,6 @@ from _pytest._code.code import ExceptionInfo from _pytest._code.code import TerminalRepr from _pytest._code.code import Traceback -from _pytest._io.saferepr import saferepr -from _pytest.compat import ascii_escaped from _pytest.compat import get_default_arg_names from _pytest.compat import get_real_func from _pytest.compat import getimfunc @@ -56,28 +45,23 @@ from _pytest.config.argparsing import Parser from _pytest.deprecated import CALLSPEC2_RENAMED from _pytest.deprecated import check_ispytest -from _pytest.fixtures import _resolve_args_directness from _pytest.fixtures import FixtureDef from _pytest.fixtures import FixtureRequest from _pytest.fixtures import FixtureValue from _pytest.fixtures import FuncFixtureInfo from _pytest.fixtures import get_scope_node from _pytest.main import Session -from _pytest.mark import ParameterSet -from _pytest.mark.structures import _HiddenParam from _pytest.mark.structures import get_unpacked_marks -from _pytest.mark.structures import HIDDEN_PARAM -from _pytest.mark.structures import Mark -from _pytest.mark.structures import MarkDecorator -from _pytest.mark.structures import normalize_mark_list from _pytest.outcomes import fail from _pytest.outcomes import skip +from _pytest.parametrize import _infer_parametrize_scope +from _pytest.parametrize import CallSpec +from _pytest.parametrize import ParametrizeContext from _pytest.pathlib import fnmatch_ex from _pytest.pathlib import import_path from _pytest.pathlib import ImportPathMismatchError from _pytest.pathlib import scandir from _pytest.scope import Scope -from _pytest.scope import ScopeName from _pytest.stash import StashKey from _pytest.warning_types import PytestCollectionWarning from _pytest.warning_types import PytestReturnNotNoneWarning @@ -86,12 +70,30 @@ if TYPE_CHECKING: from typing_extensions import Self -LongStrIdStrategy = Literal["short", "sha256", "legacy", "disallow"] -_LONG_STR_STRATEGIES: frozenset[LongStrIdStrategy] = frozenset( - get_args(LongStrIdStrategy) +# Modes for the ``collect_function_definition`` option. +# - "hidden": legacy flat layout; the FunctionDefinition is used transiently to +# drive parametrization and kept out of ("hidden" from) the tree. +# - "pedantic": insert the FunctionDefinition node; function-level markers are +# scoped to it and each invocation only owns its callspec markers. +# - "messy": insert the node, but transfer the function-level markers down onto +# each invocation to preserve the legacy marker layout. Emits a +# warning, as this defeats the purpose of the definition scope. +FunctionDefinitionMode = Literal["hidden", "pedantic", "messy"] +_FUNCTION_DEFINITION_MODES: frozenset[FunctionDefinitionMode] = frozenset( + get_args(FunctionDefinitionMode) ) +def _collect_function_definition_mode(config: Config) -> FunctionDefinitionMode: + value = config.getini("collect_function_definition") + if value not in _FUNCTION_DEFINITION_MODES: + raise UsageError( + f"Unknown collect_function_definition: {value!r}. " + f"Valid values: {', '.join(sorted(_FUNCTION_DEFINITION_MODES))}" + ) + return cast(FunctionDefinitionMode, value) + + def pytest_addoption(parser: Parser) -> None: parser.addini( "python_files", @@ -163,6 +165,16 @@ def pytest_configure(config: Config) -> None: ) +def pytest_report_header(config: Config) -> list[str] | None: + if _collect_function_definition_mode(config) == "messy": + return [ + "warning: collect_function_definition=messy transfers markers to the " + "individual invocations to preserve the legacy layout; prefer " + "'pedantic' once your plugins handle the definition scope", + ] + return None + + def async_fail(nodeid: str) -> None: msg = ( "async def functions are not natively supported.\n" @@ -331,6 +343,11 @@ def getmodpath(self, stopatmodule: bool = True, includemodule: bool = False) -> """Return Python path relative to the containing module.""" parts = [] for node in self.iter_parents(): + # A FunctionDefinition parent contributes the same name as the + # Function collected under it, so skip it to avoid duplication + # (but keep it when it is the node itself). + if node is not self and isinstance(node, FunctionDefinition): + continue name = node.name if isinstance(node, Module): name = os.path.splitext(name)[0] @@ -467,54 +484,19 @@ def collect(self) -> Iterable[nodes.Item | nodes.Collector]: result.extend(values) return result - def _genfunctions(self, name: str, funcobj) -> Iterator[Function]: - modulecol = self.getparent(Module) - assert modulecol is not None - module = modulecol.obj - clscol = self.getparent(Class) - cls = (clscol and clscol.obj) or None - + def _genfunctions( + self, name: str, funcobj + ) -> Iterator[nodes.Item | FunctionDefinition]: definition = FunctionDefinition.from_parent(self, name=name, callobj=funcobj) - fixtureinfo = definition._fixtureinfo - - # pytest_generate_tests impls call metafunc.parametrize() which fills - # metafunc._calls, the outcome of the hook. - metafunc = Metafunc( - definition=definition, - fixtureinfo=fixtureinfo, - config=self.config, - cls=cls, - module=module, - _ispytest=True, - ) - methods = [] - if hasattr(module, "pytest_generate_tests"): - methods.append(module.pytest_generate_tests) - if cls is not None and hasattr(cls, "pytest_generate_tests"): - methods.append(cls().pytest_generate_tests) - self.ihook.pytest_generate_tests.call_extra(methods, dict(metafunc=metafunc)) - - if not metafunc._calls: - yield Function.from_parent(self, name=name, fixtureinfo=fixtureinfo) + if not definition.in_collection_tree: + # Legacy flat layout: the definition is used only to drive + # parametrization and is discarded ("hidden"), the invocations are + # collected directly under this collector. + yield from definition.generate_items(self) else: - metafunc._recompute_direct_params_indices() - # Direct parametrizations taking place in module/class-specific - # `metafunc.parametrize` calls may have shadowed some fixtures, so make sure - # we update what the function really needs a.k.a its fixture closure. Note that - # direct parametrizations using `@pytest.mark.parametrize` have already been considered - # into making the closure using `ignore_args` arg to `getfixtureclosure`. - fixtureinfo.prune_dependency_tree() - - for callspec in metafunc._calls: - subname = f"{name}[{callspec.id}]" if callspec._idlist else name - yield Function.from_parent( - self, - name=subname, - callspec=callspec, - fixtureinfo=fixtureinfo, - keywords={callspec.id: True}, - originalname=name, - ) + # Insert the function definition as a collector node into the tree; + # its ``collect()`` yields the (possibly parametrized) invocations. + yield definition def importtestmodule( @@ -884,336 +866,6 @@ def hasnew(obj: object) -> bool: return False -@final -@dataclasses.dataclass(frozen=True) -class IdMaker: - """Make IDs for a parametrization.""" - - __slots__ = ( - "argnames", - "config", - "idfn", - "ids", - "nodeid", - "parametersets", - ) - - # The argnames of the parametrization. - argnames: Sequence[str] - # The ParameterSets of the parametrization. - parametersets: Sequence[ParameterSet] - # Optionally, a user-provided callable to make IDs for parameters in a - # ParameterSet. - idfn: Callable[[Any], object | None] | None - # Optionally, explicit IDs for ParameterSets by index. - ids: Sequence[object | None] | None - # Optionally, the pytest config. - # Used for controlling ASCII escaping, determining parametrization ID - # strictness, and for calling the :hook:`pytest_make_parametrize_id` hook. - config: Config | None - # Optionally, the ID of the node being parametrized. - # Used only for clearer error messages. - nodeid: str | None - - def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]: - """Make a unique identifier for each ParameterSet, that may be used to - identify the parametrization in a node ID. - - If strict_parametrization_ids is enabled, and duplicates are detected, - raises CollectError. Otherwise makes the IDs unique as follows: - - Format is -...-[counter], where prm_x_token is - - user-provided id, if given - - else an id derived from the value, applicable for certain types - - else - The counter suffix is appended only in case a string wouldn't be unique - otherwise. - """ - resolved_ids = list(self._resolve_ids()) - # All IDs must be unique! - if len(resolved_ids) != len(set(resolved_ids)): - # Record the number of occurrences of each ID. - id_counts = Counter(resolved_ids) - - if self._strict_parametrization_ids_enabled(): - parameters = ", ".join(self.argnames) - parametersets = ", ".join( - [saferepr(list(param.values)) for param in self.parametersets] - ) - ids = ", ".join( - id if id is not HIDDEN_PARAM else "" for id in resolved_ids - ) - duplicates = ", ".join( - id if id is not HIDDEN_PARAM else "" - for id, count in id_counts.items() - if count > 1 - ) - msg = textwrap.dedent(f""" - Duplicate parametrization IDs detected, but strict_parametrization_ids is set. - - Test name: {self.nodeid} - Parameters: {parameters} - Parameter sets: {parametersets} - IDs: {ids} - Duplicates: {duplicates} - - You can fix this problem using `@pytest.mark.parametrize(..., ids=...)` or `pytest.param(..., id=...)`. - """).strip() # noqa: E501 - raise nodes.Collector.CollectError(msg) - - # Map the ID to its next suffix. - id_suffixes: dict[str, int] = defaultdict(int) - # Suffix non-unique IDs to make them unique. - for index, id in enumerate(resolved_ids): - if id_counts[id] > 1: - if id is HIDDEN_PARAM: - self._complain_multiple_hidden_parameter_sets() - suffix = "" - if id and id[-1].isdigit(): - suffix = "_" - new_id = f"{id}{suffix}{id_suffixes[id]}" - while new_id in set(resolved_ids): - id_suffixes[id] += 1 - new_id = f"{id}{suffix}{id_suffixes[id]}" - resolved_ids[index] = new_id - id_suffixes[id] += 1 - assert len(resolved_ids) == len(set(resolved_ids)), ( - f"Internal error: {resolved_ids=}" - ) - return resolved_ids - - def _strict_parametrization_ids_enabled(self) -> bool: - if self.config is None: - return False - strict_parametrization_ids = self.config.getini("strict_parametrization_ids") - if strict_parametrization_ids is None: - strict_parametrization_ids = self.config.getini("strict") - return cast(bool, strict_parametrization_ids) - - def _resolve_ids(self) -> Iterable[str | _HiddenParam]: - """Resolve IDs for all ParameterSets (may contain duplicates).""" - for idx, parameterset in enumerate(self.parametersets): - if parameterset.id is not None: - # ID provided directly - pytest.param(..., id="...") - if parameterset.id is HIDDEN_PARAM: - yield HIDDEN_PARAM - else: - yield _ascii_escaped_by_config(parameterset.id, self.config) - elif self.ids and idx < len(self.ids) and self.ids[idx] is not None: - # ID provided in the IDs list - parametrize(..., ids=[...]). - if self.ids[idx] is HIDDEN_PARAM: - yield HIDDEN_PARAM - else: - yield self._idval_from_value_required(self.ids[idx], idx) - else: - # ID not provided - generate it. - yield "-".join( - self._idval(val, argname, idx) - for val, argname in zip( - parameterset.values, self.argnames, strict=True - ) - ) - - def _idval(self, val: object, argname: str, idx: int) -> str: - """Make an ID for a parameter in a ParameterSet.""" - idval = self._idval_from_function(val, argname, idx) - if idval is not None: - return idval - idval = self._idval_from_hook(val, argname) - if idval is not None: - return idval - if isinstance(val, str | bytes): - idval = self._apply_long_str_strategy(val, argname, idx) - if idval is not None: - return idval - else: - idval = self._idval_from_value(val) - if idval is not None: - return idval - return self._idval_from_argname(argname, idx) - - def _get_long_str_strategy(self) -> LongStrIdStrategy: - if not self.config: - return "short" - value = self.config.getini("parametrize_long_str_id_strategy") - if value not in _LONG_STR_STRATEGIES: - raise UsageError( - f"Unknown parametrize_long_str_id_strategy: {value!r}. " - f"Valid values: {', '.join(sorted(_LONG_STR_STRATEGIES))}" - ) - return cast(LongStrIdStrategy, value) - - def _apply_long_str_strategy( - self, val: str | bytes, argname: str, idx: int - ) -> str | None: - """Apply the configured strategy for long str/bytes parameter values. - - Only used for auto-generated IDs (not explicit ids=[...] or - pytest.param(id=...)). - """ - if len(val) <= 100: - return _ascii_escaped_by_config(val, self.config) - match self._get_long_str_strategy(): - case "legacy": - return _ascii_escaped_by_config(val, self.config) - case "short": - return None - case "sha256": - encoded = val.encode("utf-8") if isinstance(val, str) else val - return hashlib.sha256(encoded).hexdigest() - case "disallow": # pragma: no branch -- fail() raises, confuses coverage - prefix = self._make_error_prefix() - fail( - f"{prefix}parametrize value for '{argname}' at index {idx} " - f"is too long for an auto-generated ID ({len(val)} characters). " - f"Use pytest.param(..., id=...) or parametrize(..., ids=...) " - f"to set an explicit ID, or change parametrize_long_str_id_strategy.", - pytrace=False, - ) - - def _idval_from_function(self, val: object, argname: str, idx: int) -> str | None: - """Try to make an ID for a parameter in a ParameterSet using the - user-provided id callable, if given.""" - if self.idfn is None: - return None - try: - id = self.idfn(val) - except Exception as e: - prefix = f"{self.nodeid}: " if self.nodeid is not None else "" - msg = "error raised while trying to determine id of parameter '{}' at position {}" - msg = prefix + msg.format(argname, idx) - raise ValueError(msg) from e - if id is None: - return None - return self._idval_from_value(id) - - def _idval_from_hook(self, val: object, argname: str) -> str | None: - """Try to make an ID for a parameter in a ParameterSet by calling the - :hook:`pytest_make_parametrize_id` hook.""" - if self.config: - id: str | None = self.config.hook.pytest_make_parametrize_id( - config=self.config, val=val, argname=argname - ) - return id - return None - - def _idval_from_value(self, val: object) -> str | None: - """Try to make an ID for a parameter in a ParameterSet from its value, - if the value type is supported.""" - if isinstance(val, str | bytes): - return _ascii_escaped_by_config(val, self.config) - elif val is None or isinstance(val, float | int | bool | complex): - return str(val) - elif isinstance(val, re.Pattern): - return ascii_escaped(val.pattern) - elif val is NOTSET: - # Fallback to default. Note that NOTSET is an enum.Enum. - pass - elif isinstance(val, enum.Enum): - return str(val) - elif isinstance(getattr(val, "__name__", None), str): - # Name of a class, function, module, etc. - name: str = getattr(val, "__name__") - return name - return None - - def _idval_from_value_required(self, val: object, idx: int) -> str: - """Like _idval_from_value(), but fails if the type is not supported.""" - id = self._idval_from_value(val) - if id is not None: - return id - - # Fail. - prefix = self._make_error_prefix() - msg = ( - f"{prefix}ids contains unsupported value {saferepr(val)} (type: {type(val)!r}) at index {idx}. " - "Supported types are: str, bytes, int, float, complex, bool, enum, regex or anything with a __name__." - ) - fail(msg, pytrace=False) - - @staticmethod - def _idval_from_argname(argname: str, idx: int) -> str: - """Make an ID for a parameter in a ParameterSet from the argument name - and the index of the ParameterSet.""" - return str(argname) + str(idx) - - def _complain_multiple_hidden_parameter_sets(self) -> NoReturn: - fail( - f"{self._make_error_prefix()}multiple instances of HIDDEN_PARAM " - "cannot be used in the same parametrize call, " - "because the tests names need to be unique." - ) - - def _make_error_prefix(self) -> str: - if self.nodeid is not None: - return f"In {self.nodeid}: " - else: - return "" - - -@final -@dataclasses.dataclass(frozen=True) -class CallSpec: - """A planned parameterized invocation of a test function. - - Calculated during collection for a given test function's Metafunc. - Once collection is over, each callspec is turned into a single Item - and stored in item.callspec. - """ - - # arg name -> arg value which will be passed to a fixture of the same name. - params: dict[str, object] = dataclasses.field(default_factory=dict) - # arg name -> arg index. - indices: dict[str, int] = dataclasses.field(default_factory=dict) - # arg name -> parameter scope. - # Used for sorting parametrized resources. - _arg2scope: Mapping[str, Scope] = dataclasses.field(default_factory=dict) - # Parts which will be added to the item's name in `[..]` separated by "-". - _idlist: Sequence[str] = dataclasses.field(default_factory=tuple) - # Marks which will be applied to the item. - marks: list[Mark] = dataclasses.field(default_factory=list) - - def setmulti( - self, - *, - argnames: Iterable[str], - valset: Iterable[object], - id: str | _HiddenParam, - marks: Iterable[Mark | MarkDecorator], - scope: Scope, - param_index: int, - nodeid: str, - ) -> CallSpec: - params = self.params.copy() - indices = self.indices.copy() - arg2scope = dict(self._arg2scope) - for arg, val in zip(argnames, valset, strict=True): - if arg in params: - raise nodes.Collector.CollectError( - f"{nodeid}: duplicate parametrization of {arg!r}" - ) - params[arg] = val - indices[arg] = param_index - arg2scope[arg] = scope - return CallSpec( - params=params, - indices=indices, - _arg2scope=arg2scope, - _idlist=self._idlist if id is HIDDEN_PARAM else [*self._idlist, id], - marks=[*self.marks, *normalize_mark_list(marks)], - ) - - def getparam(self, name: str) -> object: - try: - return self.params[name] - except KeyError as e: - raise ValueError(name) from e - - @property - def id(self) -> str: - return "-".join(self._idlist) - - if TYPE_CHECKING: # Deprecated alias kept for type checkers; runtime access goes through __getattr__. CallSpec2 = CallSpec @@ -1251,14 +903,20 @@ def __init__(self, *, node: nodes.Node, argname: str, scope: Scope) -> None: @final -class Metafunc: +class Metafunc(ParametrizeContext): """Objects passed to the :hook:`pytest_generate_tests` hook. They help to inspect a test function and to generate tests according to test configuration or values specified in the class or module where a test function is defined. + + The Python-specific :class:`~_pytest.parametrize.ParametrizeContext`: it + resolves ``argnames`` against the test function's fixture closure and + signature, and desugars direct parametrization into fixtures. """ + definition: FunctionDefinition + def __init__( self, definition: FunctionDefinition, @@ -1270,12 +928,19 @@ def __init__( _ispytest: bool = False, ) -> None: check_ispytest(_ispytest) - #: Access to the underlying :class:`_pytest.python.FunctionDefinition`. - self.definition = definition - - #: Access to the :class:`pytest.Config` object for the test session. - self.config = config + #: + #: Access to the :class:`pytest.Config` object for the test session is + #: available as ``config``, and the set of fixture names required by the + #: test function as ``fixturenames``. + super().__init__( + definition, + config, + # Note: passed through by reference, so that a later + # prune_dependency_tree() is reflected in ``fixturenames``. + argnames=fixtureinfo.names_closure, + _ispytest=True, + ) #: The module object where the test function is defined in. self.module = module @@ -1283,177 +948,53 @@ def __init__( #: Underlying Python test function. self.function = definition.obj - #: Set of fixture names required by the test function. - self.fixturenames = fixtureinfo.names_closure - #: Class object where the test function is defined in or ``None``. self.cls = cls + # The fixture closure driving this parametrization, handed on to the + # generated Functions by FunctionDefinition.make_item(). + self._fixtureinfo = fixtureinfo self._arg2fixturedefs = fixtureinfo.name2fixturedefs - # Result of parametrize(). - self._calls: list[CallSpec] = [] + @property + def _introspection_target(self) -> object: + return self.function + + @property + def _target_name(self) -> str: + return cast(str, self.function.__name__) - self._params_directness: dict[str, Literal["indirect", "direct"]] = {} + def _infer_scope( + self, argnames: Sequence[str], indirect: bool | Sequence[str] + ) -> Scope: + return _infer_parametrize_scope(argnames, self._arg2fixturedefs, indirect) - def parametrize( + def _validate_argnames( self, - argnames: str | Sequence[str], - argvalues: Iterable[ParameterSet | Sequence[object] | object], - *, - indirect: bool | Sequence[str] = False, - ids: Iterable[object | None] | Callable[[Any], object | None] | None = None, - scope: ScopeName | None = None, - _param_mark: Mark | None = None, + argnames: Sequence[str], + indirect: bool | Sequence[str], ) -> None: - """Add new invocations to the underlying test function using the list - of argvalues for the given argnames. Parametrization is performed - during the collection phase. If you need to setup expensive resources - see about setting ``indirect`` to do it at test setup time instead. - - Can be called multiple times per test function (but only on different - argument names), in which case each call parametrizes all previous - parametrizations, e.g. - - :: - - unparametrized: t - parametrize ["x", "y"]: t[x], t[y] - parametrize [1, 2]: t[x-1], t[x-2], t[y-1], t[y-2] - - :param argnames: - A comma-separated string denoting one or more argument names, or - a list/tuple of argument strings. - - :param argvalues: - The list of argvalues determines how often a test is invoked with - different argument values. - - If only one argname was specified argvalues is a list of values. - If N argnames were specified, argvalues must be a list of - N-tuples, where each tuple-element specifies a value for its - respective argname. - - .. versionchanged:: 9.1 - - Passing a non-:class:`~collections.abc.Collection` iterable - (such as a generator or iterator) is deprecated. See - :ref:`parametrize-iterators` for details. - - :param indirect: - A list of arguments' names (subset of argnames) or a boolean. - If True the list contains all names from the argnames. Each - argvalue corresponding to an argname in this list will - be passed as request.param to its respective argname fixture - function so that it can perform more expensive setups during the - setup phase of a test rather than at collection time. - - :param ids: - Sequence of (or generator for) ids for ``argvalues``, - or a callable to return part of the id for each argvalue. - - With sequences (and generators like ``itertools.count()``) the - returned ids should be of type ``string``, ``int``, ``float``, - ``bool``, or ``None``. - They are mapped to the corresponding index in ``argvalues``. - ``None`` means to use the auto-generated id. - - .. versionadded:: 8.4 - :ref:`hidden-param` means to hide the parameter set - from the test name. Can only be used at most 1 time, as - test names need to be unique. - - If it is a callable it will be called for each entry in - ``argvalues``, and the return value is used as part of the - auto-generated id for the whole set (where parts are joined with - dashes ("-")). - This is useful to provide more specific ids for certain items, e.g. - dates. Returning ``None`` will use an auto-generated id. - - If no ids are provided they will be generated automatically from - the argvalues. - - :param scope: - If specified it denotes the scope of the parameters. - The scope is used for grouping tests by parameter instances. - It will also override any fixture-function defined scope, allowing - to set a dynamic scope using test context or configuration. - - .. versionchanged:: 9.1 - - ``indirect``, ``ids`` and ``scope`` are now keyword-only. - """ - nodeid = self.definition.nodeid - - argnames, parametersets = ParameterSet._for_parametrize( - argnames, - argvalues, - self.function, - self.config, - nodeid=self.definition.nodeid, - ) - del argvalues - - if "request" in argnames: - fail( - f"{nodeid}: 'request' is a reserved name and cannot be used in @pytest.mark.parametrize", - pytrace=False, - ) - - if scope is not None: - scope_ = Scope.from_user( - scope, descr=f"parametrize() call in {self.function.__name__}" - ) - else: - scope_ = _infer_parametrize_scope(argnames, self._arg2fixturedefs, indirect) - self._validate_if_using_arg_names(argnames, indirect) - # Use any already (possibly) generated ids with parametrize Marks. - if _param_mark and _param_mark._param_ids_from: - generated_ids = _param_mark._param_ids_from._param_ids_generated - if generated_ids is not None: - ids = generated_ids - - ids = self._resolve_parameter_set_ids( - argnames, ids, parametersets, nodeid=self.definition.nodeid - ) - - # Store used (possibly generated) ids with parametrize Marks. - if _param_mark and _param_mark._param_ids_from and generated_ids is None: - object.__setattr__(_param_mark._param_ids_from, "_param_ids_generated", ids) - - # Calculate directness. - arg_directness = _resolve_args_directness( - argnames, indirect, self.definition.nodeid - ) - self._params_directness.update(arg_directness) + def _register_direct_params( + self, + argnames: Sequence[str], + arg_directness: Mapping[str, Literal["indirect", "direct"]], + scope: Scope, + ) -> None: + """Desugar direct parametrizations into artificial fixturedefs. - # Add direct parametrizations as fixturedefs to arg2fixturedefs by - # registering artificial DirectParamFixtureDef's such that later at test - # setup time we can rely on FixtureDefs to exist for all argnames. - node = None + Registering a :class:`DirectParamFixtureDef` for every direct argname + means that at test setup time we can rely on a FixtureDef existing for + all argnames. + """ # For scopes higher than function, a DirectParamFixtureDef might have # already been created for the scope. We thus store and cache the # DirectParamFixtureDef on the node related to the scope. - if scope_ is Scope.Function: + if scope is Scope.Function: name2directparamfixturedef = None else: - collector = self.definition.parent - assert collector is not None - node = get_scope_node(collector, scope_) - if node is None: - # If used class scope and there is no class, use module-level - # collector (for now). - if scope_ is Scope.Class: - assert isinstance(collector, Module) - node = collector - # If used package scope and there is no package, use session - # (for now). - elif scope_ is Scope.Package: - node = collector.session - else: - assert False, f"Unhandled missing scope: {scope}" + node = self._scope_node_for_direct_params(scope) default: dict[str, DirectParamFixtureDef[object]] = {} name2directparamfixturedef = node.stash.setdefault( name2directparamfixturedef_key, default @@ -1470,96 +1011,39 @@ def parametrize( fixturedef = DirectParamFixtureDef( node=self.definition.session, argname=argname, - scope=scope_, + scope=scope, ) if name2directparamfixturedef is not None: name2directparamfixturedef[argname] = fixturedef self._arg2fixturedefs[argname] = [fixturedef] - # Create the new calls: if we are parametrize() multiple times (by applying the decorator - # more than once) then we accumulate those calls generating the cartesian product - # of all calls. - newcalls = [] - for callspec in self._calls or [CallSpec()]: - for param_index, (param_id, param_set) in enumerate( - zip(ids, parametersets, strict=True) - ): - newcallspec = callspec.setmulti( - argnames=argnames, - valset=param_set.values, - id=param_id, - marks=param_set.marks, - scope=scope_, - param_index=param_index, - nodeid=nodeid, + def _scope_node_for_direct_params(self, scope: Scope) -> nodes.Node: + """The node the DirectParamFixtureDefs for ``scope`` are cached on.""" + if scope is Scope.Definition: + if not self.definition.in_collection_tree: + fixtures.fail_definition_scope_unavailable( + self.definition.nodeid, + f"parametrize(scope='definition') in {self._target_name}", ) - newcalls.append(newcallspec) - self._calls = newcalls - - def _resolve_parameter_set_ids( - self, - argnames: Sequence[str], - ids: Iterable[object | None] | Callable[[Any], object | None] | None, - parametersets: Sequence[ParameterSet], - nodeid: str, - ) -> list[str | _HiddenParam]: - """Resolve the actual ids for the given parameter sets. - - :param argnames: - Argument names passed to ``parametrize()``. - :param ids: - The `ids` parameter of the ``parametrize()`` call (see docs). - :param parametersets: - The parameter sets, each containing a set of values corresponding - to ``argnames``. - :param nodeid str: - The nodeid of the definition item that generated this - parametrization. - :returns: - List with ids for each parameter set given. - """ - if ids is None: - idfn = None - ids_ = None - elif callable(ids): - idfn = ids - ids_ = None - else: - idfn = None - ids_ = self._validate_ids(ids, parametersets) - id_maker = IdMaker( - argnames, - parametersets, - idfn, - ids_, - self.config, - nodeid=nodeid, - ) - return id_maker.make_unique_parameterset_ids() - - def _validate_ids( - self, - ids: Iterable[object | None], - parametersets: Sequence[ParameterSet], - ) -> list[object | None]: - try: - num_ids = len(ids) # type: ignore[arg-type] - except TypeError: - try: - iter(ids) - except TypeError as e: - raise TypeError("ids must be a callable or an iterable") from e - num_ids = len(parametersets) - - # num_ids == 0 is a special case: https://github.com/pytest-dev/pytest/issues/1849 - if num_ids != len(parametersets) and num_ids != 0: - nodeid = self.definition.nodeid - fail( - f"In {nodeid}: {len(parametersets)} parameter sets specified, with different number of ids: {num_ids}", - pytrace=False, - ) - - return list(itertools.islice(ids, num_ids)) + # The definition is the scope node for its own invocations; unlike + # the other scopes it is not found by looking at ancestors. + return self.definition + collector = self.definition.parent + assert collector is not None + node = get_scope_node(collector, scope) + if node is None: + # If used class scope and there is no class, use module-level + # collector (for now). + if scope is Scope.Class: + assert isinstance(collector, Module) + node = collector + # If used package scope and there is no package, use session + # (for now). + elif scope is Scope.Package: + node = collector.session + else: + assert False, f"Unhandled missing scope: {scope}" + return node def _validate_if_using_arg_names( self, @@ -1591,59 +1075,6 @@ def _validate_if_using_arg_names( pytrace=False, ) - def _recompute_direct_params_indices(self) -> None: - for argname, param_type in self._params_directness.items(): - if param_type == "direct": - for i, callspec in enumerate(self._calls): - callspec.indices[argname] = i - - -def _infer_parametrize_scope( - argnames: Sequence[str], - arg2fixturedefs: Mapping[str, Sequence[fixtures.FixtureDef[object]]], - indirect: bool | Sequence[str], -) -> Scope: - """Infer the most appropriate scope for a parametrize() call based on its - arguments, for when the scope is not explicitly specified. - - When there's at least one direct argument, always use "function" scope. - - When a test function is parametrized and all its arguments are indirect - (e.g. fixtures), return the most narrow scope based on the fixtures used. - - Related to issue #1832, based on code posted by @Kingdread. - """ - if isinstance(indirect, Sequence): - all_arguments_are_fixtures = len(indirect) == len(argnames) - else: - all_arguments_are_fixtures = bool(indirect) - - if all_arguments_are_fixtures: - # Takes the most narrow scope from used fixtures. - used_scopes = ( - # Higher scope can't request lower scope, so it's OK to only - # look at the first fixturedef in the override chain. - arg2fixturedefs[argname][-1]._scope - for argname in argnames - if argname in arg2fixturedefs - ) - return min(used_scopes, default=Scope.Function) - - return Scope.Function - - -def _ascii_escaped_by_config(val: str | bytes, config: Config | None) -> str: - if config is None: - escape_option = False - else: - escape_option = config.getini( - "disable_test_id_escaping_and_forfeit_all_rights_to_community_support" - ) - # TODO: If escaping is turned off and the user passes bytes, - # will return a bytes. For now we ignore this but the - # code *probably* doesn't handle this case. - return val if escape_option else ascii_escaped(val) # type: ignore - class Function(PyobjMixin, nodes.Item): """Item responsible for setting up and executing a Python test function. @@ -1688,8 +1119,9 @@ def __init__( session: Session | None = None, fixtureinfo: FuncFixtureInfo | None = None, originalname: str | None = None, + nodeid: str | None = None, ) -> None: - super().__init__(name, parent, config=config, session=session) + super().__init__(name, parent, config=config, session=session, nodeid=nodeid) if callobj is not NOTSET: self._obj = callobj @@ -1706,7 +1138,12 @@ def __init__( # Note: when FunctionDefinition is introduced, we should change ``originalname`` # to a readonly property that returns FunctionDefinition.name. - self.own_markers.extend(get_unpacked_marks(self.obj)) + # Function-level markers are owned by the FunctionDefinition scope when + # that node is part of the collection tree; otherwise (flat layout) the + # Function owns them itself. In "messy" mode FunctionDefinition.collect() + # transfers them back onto each invocation for legacy compatibility. + if not isinstance(self.parent, FunctionDefinition): + self.own_markers.extend(get_unpacked_marks(self.obj)) if callspec: self.callspec = callspec self.own_markers.extend(callspec.marks) @@ -1748,17 +1185,16 @@ def instance(self): try: return self._instance except AttributeError: - if isinstance(self.parent, Class): - # Each Function gets a fresh class instance. - self._instance = self._getinstance() - else: - self._instance = None + self._instance = self._getinstance() return self._instance def _getinstance(self): - if isinstance(self.parent, Class): + # The containing class, if any -- skipping over an interposed + # FunctionDefinition node (see the ``collect_function_definition`` option). + cls = self.getparent(Class) + if cls is not None: # Each Function gets a fresh class instance. - return self.parent.newinstance() + return cls.newinstance() else: return None @@ -1767,8 +1203,13 @@ def _getobj(self): if instance is not None: parent_obj = instance else: - assert self.parent is not None - parent_obj = self.parent.obj # type: ignore[attr-defined] + # The namespace this function was collected from -- a Module (or a + # Class handled above), skipping over an interposed FunctionDefinition. + parent = self.parent + while isinstance(parent, FunctionDefinition): + parent = parent.parent + assert parent is not None + parent_obj = parent.obj # type: ignore[attr-defined] return getattr(parent_obj, self.originalname) @property @@ -1823,14 +1264,146 @@ def repr_failure( # type: ignore[override] return self._repr_failure_py(excinfo, style=style) -class FunctionDefinition(Function): - """This class is a stop gap solution until we evolve to have actual function - definition nodes and manage to get rid of ``metafunc``.""" +class FunctionDefinition(nodes.ItemDefinition, PyCollector): + """Collector node for a single test function definition. - def runtest(self) -> None: - raise RuntimeError("function definitions are not supposed to be run as tests") + Its children are the (possibly parametrized) :class:`Function` invocations + generated from the definition via :hook:`pytest_generate_tests`. - setup = runtest + This node is only inserted into the collection tree when the + :confval:`collect_function_definition` option is enabled. Otherwise it is + created transiently to drive parametrization (backing :class:`Metafunc`) + and then discarded, with the resulting :class:`Function` items collected + directly under the containing :class:`Class`/:class:`Module`. + """ + + # Markers are handled explicitly below, mirroring Function. + _ALLOW_MARKERS = False + + def __init__( + self, + name: str, + parent, + callobj, + config: Config | None = None, + session: Session | None = None, + ) -> None: + super().__init__(name, parent, config=config, session=session) + + # The definition always stands for a concrete function object; unlike + # Function it never looks the object up from its parent lazily. + self._obj = callobj + + self.own_markers.extend(get_unpacked_marks(self.obj)) + self.keywords.update((mark.name, mark) for mark in self.own_markers) + self.keywords.update(self.obj.__dict__) + + @property + def in_collection_tree(self) -> bool: + """Whether this definition is a node of the collection tree. + + False in the default ``hidden`` mode of + :confval:`collect_function_definition`, where the definition only exists + transiently during collection -- and thus cannot anchor anything at run + time, such as a ``"definition"`` scoped fixture. + """ + return _collect_function_definition_mode(self.config) != "hidden" + + def collect(self) -> Iterable[nodes.Item | nodes.Collector]: + children = list(super().collect()) + if _collect_function_definition_mode(self.config) == "messy": + # Legacy marker layout: transfer this scope's markers back onto each + # invocation and drop them here, so marker resolution matches the flat + # layout exactly (no duplication when walking parents in iter_markers). + marks = self.own_markers + for child in children: + child.own_markers[:0] = marks + child.keywords.update((mark.name, mark) for mark in marks) + self.own_markers = [] + return children + + # The generate-tests protocol, see nodes.ItemDefinition. + + @property + def _module_obj(self) -> types.ModuleType: + modulecol = self.getparent(Module) + assert modulecol is not None + return cast(types.ModuleType, modulecol.obj) + + @property + def _cls_obj(self) -> type | None: + clscol = self.getparent(Class) + return (clscol and clscol.obj) or None + + def make_parametrize_context(self) -> Metafunc: + cls = self._cls_obj + # Compute the function's fixture closure. This drives parametrization and + # is shared with the generated invocations; it is carried by the Metafunc + # rather than stored here -- fixtures belong to the executed items, not + # to this collector. + # TODO(#3926): getfixtureinfo() is item-scoped, but here the definition + # (a collector) stands in for the not-yet-created invocations. Resolve by + # giving fixture-closure computation a node-level entry point instead of + # casting a collector to an item. + fixtureinfo = self.session._fixturemanager.getfixtureinfo( + cast(nodes.Item, self), self.obj, cls + ) + return Metafunc( + definition=self, + fixtureinfo=fixtureinfo, + config=self.config, + cls=cls, + module=self._module_obj, + _ispytest=True, + ) + + def parametrize_hook_extras(self) -> Sequence[Callable[..., object]]: + module = self._module_obj + cls = self._cls_obj + methods = [] + if hasattr(module, "pytest_generate_tests"): + methods.append(module.pytest_generate_tests) + if cls is not None and hasattr(cls, "pytest_generate_tests"): + methods.append(cls().pytest_generate_tests) + return methods + + def finalize_parametrization(self, context: ParametrizeContext) -> None: + super().finalize_parametrization(context) + # Direct parametrizations taking place in module/class-specific + # `metafunc.parametrize` calls may have shadowed some fixtures, so make sure + # we update what the function really needs a.k.a its fixture closure. Note that + # direct parametrizations using `@pytest.mark.parametrize` have already been considered + # into making the closure using `ignore_args` arg to `getfixtureclosure`. + assert isinstance(context, Metafunc) + context._fixtureinfo.prune_dependency_tree() + + def make_item( + self, + parent: nodes.Collector, + *, + name: str, + callspec: CallSpec | None, + nodeid: str, + context: ParametrizeContext, + ) -> Function: + assert isinstance(context, Metafunc) + fixtureinfo = context._fixtureinfo + if callspec is None: + return Function.from_parent( + parent, + name=name, + fixtureinfo=fixtureinfo, + nodeid=nodeid, + ) + return Function.from_parent( + parent, + name=name, + callspec=callspec, + fixtureinfo=fixtureinfo, + keywords={callspec.id: True}, + originalname=self.name, + nodeid=nodeid, + ) def __getattr__(name: str) -> object: diff --git a/src/_pytest/reports.py b/src/_pytest/reports.py index 011a69db001..fb9ce20692a 100644 --- a/src/_pytest/reports.py +++ b/src/_pytest/reports.py @@ -417,8 +417,10 @@ def from_item_and_call(cls, item: Item, call: CallInfo[None]) -> TestReport: ) if excinfo.value._use_item_location: path, line = item.reportinfo()[:2] - assert line is not None - longrepr = (os.fspath(path), line + 1, r.message) + # Items which cannot point at a line -- non-Python items may + # legitimately report None -- get the location without one. + lineno = line + 1 if line is not None else -1 + longrepr = (os.fspath(path), lineno, r.message) else: longrepr = (str(r.path), r.lineno, r.message) elif isinstance(excinfo.value, BaseExceptionGroup) and ( diff --git a/src/_pytest/scope.py b/src/_pytest/scope.py index 68a9da0e6b8..9cc05bc1e90 100644 --- a/src/_pytest/scope.py +++ b/src/_pytest/scope.py @@ -15,7 +15,7 @@ from typing import Literal -ScopeName = Literal["session", "package", "module", "class", "function"] +ScopeName = Literal["session", "package", "module", "class", "definition", "function"] @total_ordering @@ -27,13 +27,19 @@ class Scope(Enum): ->>> higher ->>> - Function < Class < Module < Package < Session + Function < Definition < Class < Module < Package < Session <<<- lower <<<- + + ``Definition`` is the scope of a single test *definition*, shared by all + the (possibly parametrized) invocations generated from it. It requires the + definition to be a node in the collection tree -- see + :confval:`collect_function_definition`. """ # Scopes need to be listed from lower to higher. Function = "function" + Definition = "definition" Class = "class" Module = "module" Package = "package" diff --git a/src/_pytest/setuponly.py b/src/_pytest/setuponly.py index 7e6b46bcdb4..eed5b52e52c 100644 --- a/src/_pytest/setuponly.py +++ b/src/_pytest/setuponly.py @@ -61,6 +61,20 @@ def pytest_fixture_post_finalizer( del fixturedef.cached_param +# Use smaller indentation the higher the scope. Spelled out rather than derived +# from the Scope order, so that adding a scope does not shift the output of the +# existing ones; Definition shares Function's indentation and is told apart by +# the scope letter. +_SCOPE_INDENTS = { + Scope.Session: 0, + Scope.Package: 1, + Scope.Module: 2, + Scope.Class: 3, + Scope.Definition: 4, + Scope.Function: 4, +} + + def _show_fixture_action( fixturedef: FixtureDef[object], config: Config, msg: str ) -> None: @@ -70,9 +84,7 @@ def _show_fixture_action( tw = config.get_terminal_writer() tw.line() - # Use smaller indentation the higher the scope: Session = 0, Package = 1, etc. - scope_indent = list(reversed(Scope)).index(fixturedef._scope) - tw.write(" " * 2 * scope_indent) + tw.write(" " * 2 * _SCOPE_INDENTS[fixturedef._scope]) scopename = fixturedef.scope[0].upper() tw.write(f"{msg:<8} {scopename} {fixturedef.argname}") diff --git a/testing/deprecated_test.py b/testing/deprecated_test.py index 8eed1fb3149..cb36dbe7941 100644 --- a/testing/deprecated_test.py +++ b/testing/deprecated_test.py @@ -316,8 +316,8 @@ def test_it(request): def test_callspec2_renamed() -> None: """Importing/accessing CallSpec2 warns and returns CallSpec.""" + from _pytest.parametrize import CallSpec import _pytest.python as python_mod - from _pytest.python import CallSpec with pytest.warns(pytest.PytestRemovedIn10Warning, match="CallSpec2"): from _pytest.python import CallSpec2 diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index 3dd07a0ffc8..61b0f91f36c 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -21,9 +21,9 @@ from _pytest.compat import NOTSET from _pytest.outcomes import fail from _pytest.outcomes import Failed +from _pytest.parametrize import IdMaker from _pytest.pytester import Pytester from _pytest.python import Function -from _pytest.python import IdMaker from _pytest.scope import Scope import pytest @@ -223,7 +223,7 @@ def func(request): def test_infer_parametrize_scope(self) -> None: """Unit test for _infer_parameterize_scope (#3941).""" - from _pytest.python import _infer_parametrize_scope + from _pytest.parametrize import _infer_parametrize_scope @dataclasses.dataclass class DummyFixtureDef: diff --git a/testing/test_collect_function_definition.py b/testing/test_collect_function_definition.py new file mode 100644 index 00000000000..68e8a486151 --- /dev/null +++ b/testing/test_collect_function_definition.py @@ -0,0 +1,219 @@ +"""Tests for the `collect_function_definition` configuration value.""" + +from __future__ import annotations + +from _pytest.pytester import Pytester +import pytest + + +@pytest.fixture +def sample(pytester: Pytester) -> Pytester: + pytester.makepyfile( + """ + import pytest + + def test_plain(): + pass + + @pytest.mark.parametrize("x", [1, 2]) + def test_param(x): + assert x + + class TestCls: + @pytest.mark.parametrize("y", [3, 4]) + def test_method(self, y): + assert y + + def test_single(self): + pass + """ + ) + return pytester + + +def _set_mode(pytester: Pytester, mode: str) -> None: + pytester.makeini(f"[pytest]\ncollect_function_definition = {mode}\n") + + +FLAT_IDS = [ + "test_plain", + "test_param[1]", + "test_param[2]", + "TestCls::test_method[3]", + "TestCls::test_method[4]", + "TestCls::test_single", +] + +TREE_MODES = ["pedantic", "messy"] +ALL_MODES = ["hidden", *TREE_MODES] + + +def test_hidden_is_default(sample: Pytester) -> None: + """Without the option the definition node is not part of the tree.""" + items, _ = sample.inline_genitems() + for item in items: + assert type(item.parent).__name__ in ("Module", "Class") + + +@pytest.mark.parametrize("mode", TREE_MODES) +def test_definition_node_inserted(sample: Pytester, mode: str) -> None: + """In tree modes a FunctionDefinition collector wraps each test.""" + from _pytest.python import Function + from _pytest.python import FunctionDefinition + + _set_mode(sample, mode) + items, _ = sample.inline_genitems() + assert len(items) == len(FLAT_IDS) + for item in items: + assert isinstance(item, Function) + # Every invocation, including non-parametrized ones, is wrapped. + assert isinstance(item.parent, FunctionDefinition) + + +@pytest.mark.parametrize("mode", ALL_MODES) +def test_nodeids_are_stable(sample: Pytester, mode: str) -> None: + """Invocation nodeids stay flat regardless of the mode.""" + _set_mode(sample, mode) + items, _ = sample.inline_genitems() + suffixes = [item.nodeid.split("::", 1)[1] for item in items] + assert suffixes == FLAT_IDS + + +@pytest.mark.parametrize("mode", ALL_MODES) +def test_tests_run(sample: Pytester, mode: str) -> None: + _set_mode(sample, mode) + result = sample.runpytest() + result.assert_outcomes(passed=6) + + +def test_selection_and_reporting(sample: Pytester) -> None: + """-k selection and failure reporting address the flat nodeid.""" + _set_mode(sample, "pedantic") + result = sample.runpytest("-k", "test_param and 1", "-v") + assert "test_param[1] PASSED" in result.stdout.str() + result.assert_outcomes(passed=1, deselected=5) + + +@pytest.mark.parametrize("mode", ALL_MODES) +@pytest.mark.parametrize( + ("selector", "expected"), + [ + ("test_param[1]", 1), + ("test_param", 2), + ("test_plain", 1), + ("TestCls::test_method[3]", 1), + ("TestCls::test_single", 1), + ("TestCls", 3), + ], +) +def test_selection_by_nodeid( + sample: Pytester, mode: str, selector: str, expected: int +) -> None: + """Addressing a test by its (flat) nodeid works in every mode. + + The definition node carries the bare name while the parametrization lives on + the items below it, so the argument matcher has to look through it. + """ + _set_mode(sample, mode) + module = next(sample.path.glob("test_*.py")).name + result = sample.runpytest(f"{module}::{selector}") + result.assert_outcomes(passed=expected) + + +def test_collect_only_tree(sample: Pytester) -> None: + _set_mode(sample, "pedantic") + result = sample.runpytest("--collect-only") + out = result.stdout.str() + for expected in [ + "", + "", + "", + "", + "", + "", + ]: + assert expected in out + + +@pytest.mark.parametrize("mode", ALL_MODES) +def test_markers_are_not_duplicated(pytester: Pytester, mode: str) -> None: + """A function-level marker resolves exactly once in every mode. + + Regression guard: with the definition node in the tree both it and the + invocation carry the same function-level marks, which must not double up + when walking parents in ``iter_markers``. + """ + pytester.makeini(f"[pytest]\ncollect_function_definition = {mode}\n") + pytester.makepyfile( + """ + import pytest + + @pytest.mark.foo + @pytest.mark.parametrize("x", [1]) + def test_it(x): + pass + """ + ) + items, _ = pytester.inline_genitems() + (item,) = items + assert len(list(item.iter_markers(name="foo"))) == 1 + + +def test_pedantic_scopes_markers_to_definition(pytester: Pytester) -> None: + """In pedantic mode function-level markers live on the definition node.""" + from _pytest.python import FunctionDefinition + + pytester.makeini("[pytest]\ncollect_function_definition = pedantic\n") + pytester.makepyfile( + """ + import pytest + + @pytest.mark.foo + def test_it(): + pass + """ + ) + (item,) = pytester.inline_genitems()[0] + assert isinstance(item.parent, FunctionDefinition) + assert [m.name for m in item.own_markers] == [] + assert [m.name for m in item.parent.own_markers] == ["foo"] + + +def test_messy_transfers_markers_to_invocation(pytester: Pytester) -> None: + """In messy mode markers are transferred back onto the invocation.""" + from _pytest.python import FunctionDefinition + + pytester.makeini("[pytest]\ncollect_function_definition = messy\n") + pytester.makepyfile( + """ + import pytest + + @pytest.mark.foo + def test_it(): + pass + """ + ) + (item,) = pytester.inline_genitems()[0] + assert isinstance(item.parent, FunctionDefinition) + assert [m.name for m in item.own_markers] == ["foo"] + # Transferred away from the definition scope to avoid duplication. + assert [m.name for m in item.parent.own_markers] == [] + + +def test_messy_emits_header_warning(sample: Pytester) -> None: + _set_mode(sample, "messy") + result = sample.runpytest() + result.stdout.fnmatch_lines(["*collect_function_definition=messy*"]) + + +def test_pedantic_has_no_header_warning(sample: Pytester) -> None: + _set_mode(sample, "pedantic") + result = sample.runpytest() + result.stdout.no_fnmatch_line("*collect_function_definition=messy*") + + +def test_invalid_value_errors(sample: Pytester) -> None: + _set_mode(sample, "bogus") + result = sample.runpytest() + result.stderr.fnmatch_lines(["*Unknown collect_function_definition: 'bogus'*"]) + assert result.ret != 0 diff --git a/testing/test_definition_scope.py b/testing/test_definition_scope.py new file mode 100644 index 00000000000..083a80d1007 --- /dev/null +++ b/testing/test_definition_scope.py @@ -0,0 +1,344 @@ +"""Tests for the ``"definition"`` fixture scope. + +A definition-scoped fixture is shared by all the (possibly parametrized) +invocations generated from one test definition. That needs the definition to be +a node of the collection tree, i.e. :confval:`collect_function_definition` set +to something other than the default ``hidden``. +""" + +from __future__ import annotations + +from _pytest.pytester import Pytester +import pytest + + +def _enable_definition_nodes(pytester: Pytester) -> None: + pytester.makeini("[pytest]\ncollect_function_definition = pedantic\n") + + +def test_shared_between_invocations_of_one_definition(pytester: Pytester) -> None: + """One fixture instance per definition, not per invocation.""" + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + setups = [] + + @pytest.fixture(scope="definition") + def counter(request): + setups.append(request.node.nodeid) + return len(setups) + + @pytest.mark.parametrize("n", [1, 2, 3]) + def test_a(counter, n): + assert counter == 1 + + @pytest.mark.parametrize("n", [1, 2]) + def test_b(counter, n): + assert counter == 2 + + def test_setups_are_per_definition(): + assert setups == [ + "test_shared_between_invocations_of_one_definition.py::test_a", + "test_shared_between_invocations_of_one_definition.py::test_b", + ] + """ + ) + pytester.runpytest().assert_outcomes(passed=6) + + +def test_torn_down_when_definition_is_left(pytester: Pytester) -> None: + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + events = [] + + @pytest.fixture(scope="definition") + def res(request): + events.append(f"setup {request.node.name}") + yield + events.append(f"teardown {request.node.name}") + + @pytest.mark.parametrize("n", [1, 2]) + def test_a(res, n): + events.append(f"test_a{n}") + + @pytest.mark.parametrize("n", [1, 2]) + def test_b(res, n): + events.append(f"test_b{n}") + + def test_order(): + assert events == [ + "setup test_a", "test_a1", "test_a2", "teardown test_a", + "setup test_b", "test_b1", "test_b2", "teardown test_b", + ] + """ + ) + pytester.runpytest().assert_outcomes(passed=5) + + +def test_methods_of_a_class(pytester: Pytester) -> None: + """Each method has its own definition, the class is still shared.""" + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + definition_setups = [] + class_setups = [] + + @pytest.fixture(scope="definition") + def per_definition(): + definition_setups.append(1) + + @pytest.fixture(scope="class") + def per_class(): + class_setups.append(1) + + class TestIt: + @pytest.mark.parametrize("n", [1, 2]) + def test_a(self, per_definition, per_class, n): + pass + + @pytest.mark.parametrize("n", [1, 2]) + def test_b(self, per_definition, per_class, n): + pass + + def test_counts(): + assert len(definition_setups) == 2 + assert len(class_setups) == 1 + """ + ) + pytester.runpytest().assert_outcomes(passed=5) + + +def test_unavailable_without_definition_nodes(pytester: Pytester) -> None: + """In the default ``hidden`` mode there is no node to anchor the scope on.""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="definition") + def res(): + return 1 + + def test_it(res): + pass + """ + ) + result = pytester.runpytest() + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines( + [ + "*ScopeUnavailable: fixture 'res' needs the 'definition' scope, but" + " *::test_it has no definition node in the collection tree.", + "*set the collect_function_definition ini option to 'pedantic'*", + ] + ) + + +def test_may_request_higher_scoped_fixture(pytester: Pytester) -> None: + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="module") + def per_module(): + return 1 + + @pytest.fixture(scope="definition") + def per_definition(per_module): + return per_module + 1 + + def test_it(per_definition): + assert per_definition == 2 + """ + ) + pytester.runpytest().assert_outcomes(passed=1) + + +def test_higher_scope_may_not_request_definition_scope(pytester: Pytester) -> None: + """Class scope is higher than definition scope, so this is a ScopeMismatch.""" + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="definition") + def per_definition(): + return 1 + + @pytest.fixture(scope="class") + def per_class(per_definition): + return per_definition + + def test_it(per_class): + pass + """ + ) + result = pytester.runpytest() + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines( + [ + "*ScopeMismatch: You tried to access the definition scoped fixture" + " per_definition with a class scoped request object*" + ] + ) + + +def test_definition_scope_may_request_function_scope_fails( + pytester: Pytester, +) -> None: + """Function scope is lower than definition scope.""" + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def per_function(): + return 1 + + @pytest.fixture(scope="definition") + def per_definition(per_function): + return per_function + + def test_it(per_definition): + pass + """ + ) + result = pytester.runpytest() + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines( + [ + "*ScopeMismatch: You tried to access the function scoped fixture" + " per_function with a definition scoped request object*" + ] + ) + + +def test_setup_show_marks_the_scope(pytester: Pytester) -> None: + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="definition") + def res(): + return 1 + + @pytest.mark.parametrize("n", [1, 2]) + def test_it(res, n): + pass + """ + ) + result = pytester.runpytest("--setup-show") + result.assert_outcomes(passed=2) + result.stdout.fnmatch_lines( + [ + " SETUP D res", + "*test_it[[]1[]]*", + "*test_it[[]2[]]*", + " TEARDOWN D res", + ] + ) + + +def test_parametrize_with_definition_scope(pytester: Pytester) -> None: + """A definition-scoped param is set up once per value within the definition.""" + _enable_definition_nodes(pytester) + pytester.makepyfile( + """ + import pytest + + events = [] + + @pytest.fixture + def outer(request): + events.append(f"setup {request.param}") + yield request.param + events.append(f"teardown {request.param}") + + @pytest.mark.parametrize("outer", ["a", "b"], indirect=True, scope="definition") + @pytest.mark.parametrize("inner", [1, 2]) + def test_it(outer, inner): + events.append(f"{outer}{inner}") + + def test_order(): + assert events == [ + "setup a", "a1", "a2", "teardown a", + "setup b", "b1", "b2", "teardown b", + ] + """ + ) + pytester.runpytest().assert_outcomes(passed=5) + + +def test_parametrize_with_definition_scope_needs_definition_nodes( + pytester: Pytester, +) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize("x", [1, 2], scope="definition") + def test_it(x): + pass + """ + ) + result = pytester.runpytest() + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines( + [ + "*ScopeUnavailable: parametrize(scope='definition') in test_it needs" + " the 'definition' scope*" + ] + ) + + +def test_unknown_scope_still_rejected(pytester: Pytester) -> None: + """Guard against the new scope name loosening validation.""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="definitions") + def res(): + return 1 + + def test_it(res): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*got an unexpected scope value 'definitions'*"]) + + +@pytest.mark.parametrize("scope", ["session", "package", "module", "class"]) +def test_definition_scoped_fixture_may_be_requested_by_tests( + pytester: Pytester, scope: str +) -> None: + """A definition-scoped fixture composes with every higher scope.""" + _enable_definition_nodes(pytester) + pytester.makepyfile( + f""" + import pytest + + @pytest.fixture(scope="{scope}") + def higher(): + return "{scope}" + + @pytest.fixture(scope="definition") + def per_definition(higher): + return higher + + @pytest.mark.parametrize("n", [1, 2]) + def test_it(per_definition, n): + assert per_definition == "{scope}" + """ + ) + pytester.runpytest().assert_outcomes(passed=2) diff --git a/testing/test_item_definition.py b/testing/test_item_definition.py new file mode 100644 index 00000000000..f4c61f5fdcb --- /dev/null +++ b/testing/test_item_definition.py @@ -0,0 +1,199 @@ +"""Tests for :class:`_pytest.nodes.ItemDefinition`. + +The generic definition node lets a non-Python collector take part in +:hook:`pytest_generate_tests`: it declares the names it accepts, the hook +parametrizes it like any Python test, and the chosen values arrive on +``item.callspec.params``. +""" + +from __future__ import annotations + +from _pytest.pytester import Pytester +import pytest + + +# A minimal non-Python collector: every whitespace-separated word in a ``.echo`` +# file is a test definition that can be parametrized on "value". +ECHO_PLUGIN = """ +from _pytest import nodes +import pytest + + +class EchoItem(pytest.Item): + def __init__(self, *, spec, **kw): + super().__init__(**kw) + self.spec = spec + + def runtest(self): + assert self.spec.get("value") != "bad", f"bad value in {self.name}" + + def reportinfo(self): + return self.path, None, self.name + + +class EchoDefinition(nodes.ItemDefinition): + parametrize_argnames = ("value",) + + def make_item(self, parent, *, name, callspec, nodeid, context): + return EchoItem.from_parent( + parent, + name=name, + nodeid=nodeid, + spec=dict(callspec.params) if callspec is not None else {}, + ) + + +class EchoFile(pytest.File): + def collect(self): + for word in self.path.read_text().split(): + yield EchoDefinition.from_parent(self, name=word) + + +def pytest_collect_file(file_path, parent): + if file_path.suffix == ".echo": + return EchoFile.from_parent(parent, path=file_path) +""" + + +@pytest.fixture +def echo(pytester: Pytester) -> Pytester: + pytester.makeconftest(ECHO_PLUGIN) + pytester.makefile(".echo", things="alpha beta") + return pytester + + +def test_definition_without_parametrization(echo: Pytester) -> None: + """No parametrize() call: one item per definition, same name.""" + result = echo.runpytest("--collect-only", "-q") + result.stdout.fnmatch_lines(["things.echo::alpha", "things.echo::beta"]) + echo.runpytest().assert_outcomes(passed=2) + + +def test_generate_tests_hook_parametrizes_the_definition(echo: Pytester) -> None: + """An ordinary pytest_generate_tests impl works on a non-Python definition.""" + echo.makeconftest( + ECHO_PLUGIN + + """ +def pytest_generate_tests(metafunc): + if "value" in metafunc.fixturenames: + metafunc.parametrize("value", ["good", "bad"]) +""" + ) + result = echo.runpytest("--collect-only", "-q") + result.stdout.fnmatch_lines( + [ + "things.echo::alpha[[]good[]]", + "things.echo::alpha[[]bad[]]", + "things.echo::beta[[]good[]]", + "things.echo::beta[[]bad[]]", + ] + ) + result = echo.runpytest() + result.assert_outcomes(passed=2, failed=2) + result.stdout.fnmatch_lines(["*bad value in alpha[[]bad[]]*"]) + + +def test_params_are_delivered_on_the_callspec(echo: Pytester) -> None: + echo.makeconftest( + ECHO_PLUGIN + + """ +def pytest_generate_tests(metafunc): + metafunc.parametrize("value", ["good"]) + +def pytest_collection_modifyitems(items): + for item in items: + assert item.callspec.params == {"value": "good"} + assert item.spec == {"value": "good"} +""" + ) + echo.runpytest().assert_outcomes(passed=2) + + +def test_ids_and_marks_from_pytest_param(echo: Pytester) -> None: + echo.makeconftest( + ECHO_PLUGIN + + """ +import pytest + +def pytest_generate_tests(metafunc): + metafunc.parametrize( + "value", + [ + pytest.param("good", id="ok"), + pytest.param("bad", marks=pytest.mark.skip(reason="nope")), + ], + ) +""" + ) + result = echo.runpytest("-v") + result.assert_outcomes(passed=2, skipped=2) + result.stdout.fnmatch_lines(["*things.echo::alpha[[]ok[]] PASSED*"]) + + +def test_parametrize_of_undeclared_argname_is_rejected(echo: Pytester) -> None: + echo.makeconftest( + ECHO_PLUGIN + + """ +def pytest_generate_tests(metafunc): + metafunc.parametrize("nope", [1]) +""" + ) + result = echo.runpytest() + result.stdout.fnmatch_lines( + ["*In things.echo::alpha: definition uses no argument 'nope'*"] + ) + assert result.ret != 0 + + +def test_parametrize_marker_on_the_definition(echo: Pytester) -> None: + """Markers on the definition drive parametrization, as they do for Python.""" + echo.makeconftest( + ECHO_PLUGIN + + """ +import pytest + +def pytest_collectstart(collector): + if isinstance(collector, EchoDefinition): + collector.add_marker(pytest.mark.parametrize("value", ["good", "bad"])) +""" + ) + result = echo.runpytest() + result.assert_outcomes(passed=2, failed=2) + + +def test_item_selection_by_nodeid(echo: Pytester) -> None: + echo.makeconftest( + ECHO_PLUGIN + + """ +def pytest_generate_tests(metafunc): + metafunc.parametrize("value", ["good", "bad"]) +""" + ) + result = echo.runpytest("things.echo::alpha[good]") + result.assert_outcomes(passed=1) + + +def test_duplicate_parametrization_is_reported(echo: Pytester) -> None: + echo.makeconftest( + ECHO_PLUGIN + + """ +def pytest_generate_tests(metafunc): + metafunc.parametrize("value", ["a"]) + metafunc.parametrize("value", ["b"]) +""" + ) + result = echo.runpytest() + result.stdout.fnmatch_lines(["*duplicate parametrization of 'value'*"]) + assert result.ret != 0 + + +def test_definition_node_is_a_collector(echo: Pytester) -> None: + """The definition shows up in the tree between the file and the items.""" + result = echo.runpytest("--collect-only") + result.stdout.fnmatch_lines( + [ + "*", + "*", + "*", + ] + ) diff --git a/testing/test_scope.py b/testing/test_scope.py index 3cb811469a9..c5d10a73614 100644 --- a/testing/test_scope.py +++ b/testing/test_scope.py @@ -10,21 +10,24 @@ def test_ordering() -> None: assert Scope.Session > Scope.Package assert Scope.Package > Scope.Module assert Scope.Module > Scope.Class - assert Scope.Class > Scope.Function + assert Scope.Class > Scope.Definition + assert Scope.Definition > Scope.Function def test_next_lower() -> None: assert Scope.Session.next_lower() is Scope.Package assert Scope.Package.next_lower() is Scope.Module assert Scope.Module.next_lower() is Scope.Class - assert Scope.Class.next_lower() is Scope.Function + assert Scope.Class.next_lower() is Scope.Definition + assert Scope.Definition.next_lower() is Scope.Function with pytest.raises(ValueError, match="Function is the lower-most scope"): Scope.Function.next_lower() def test_next_higher() -> None: - assert Scope.Function.next_higher() is Scope.Class + assert Scope.Function.next_higher() is Scope.Definition + assert Scope.Definition.next_higher() is Scope.Class assert Scope.Class.next_higher() is Scope.Module assert Scope.Module.next_higher() is Scope.Package assert Scope.Package.next_higher() is Scope.Session