Experimental _pytest.scenario API: nested configs + in-memory collection (PoC)

changelog/14809.feature.rst

@@ -0,0 +1,13 @@
+Added the experimental internal ``_pytest.scenario`` API for testing pytest with pytest:
+build a hermetic nested configuration from declarative data (``ConfigSpec``/``configured()``),
+collect test items from in-memory python objects instead of files on disk
+(``fake_module()``, ``collect_tests()``), run them through the standard runtest protocol,
+and assert on typed report objects (``RunRecord``) instead of glob-matching terminal output.
+
+In support of this:
+
+* ``Config.parse()`` was split into behavior-preserving phase methods, so programmatic
+  config construction shares real code with command line parsing.
+* ``Config.ArgsSource.SPEC`` marks args taken verbatim from a programmatic specification.
+* ``pytest.Class.from_parent()`` no longer silently discards a passed ``obj``.
+* ``_pytest.config.get_config()`` accepts an explicit invocation ``dir=``.

src/_pytest/config/__init__.py

@@ -356,13 +356,14 @@ def get_config(
     plugins: Sequence[str | _PluggyPlugin] | None = None,
     *,
     prog: str | None = None,
+    dir: pathlib.Path | None = None,
 ) -> Config:
     # Subsequent calls to main will create a fresh instance.
     pluginmanager = PytestPluginManager()
     invocation_params = Config.InvocationParams(
         args=args or (),
         plugins=plugins,
-        dir=pathlib.Path.cwd(),
+        dir=dir if dir is not None else pathlib.Path.cwd(),
     )
     config = Config(pluginmanager, invocation_params=invocation_params, prog=prog)
 
@@ -881,7 +882,7 @@ def consider_module(self, mod: types.ModuleType) -> None:
         self._import_plugin_specs(getattr(mod, "pytest_plugins", []))
 
     def _import_plugin_specs(
-        self, spec: None | types.ModuleType | str | Sequence[str]
+        self, spec: types.ModuleType | str | Sequence[str] | None
     ) -> None:
         plugins = _get_plugin_specs_as_list(spec)
         for import_spec in plugins:
@@ -931,7 +932,7 @@ def import_plugin(self, modname: str, consider_entry_points: bool = False) -> No
 
 
 def _get_plugin_specs_as_list(
-    specs: None | types.ModuleType | str | Sequence[str],
+    specs: types.ModuleType | str | Sequence[str] | None,
 ) -> list[str]:
     """Parse a plugins specification into a list of plugin names."""
     # None means empty.
@@ -1100,10 +1101,17 @@ class ArgsSource(enum.Enum):
         INCOVATION_DIR = INVOCATION_DIR  # backwards compatibility alias
         #: 'testpaths' configuration value.
         TESTPATHS = enum.auto()
+        #: Programmatic specification; args are taken verbatim, without
+        #: filesystem-based fallbacks (experimental).
+        SPEC = enum.auto()
 
     # Set by cacheprovider plugin.
     cache: Cache
 
+    #: The parsed command line namespace, including only options known at
+    #: the time of parsing. Set during :meth:`parse`.
+    known_args_namespace: argparse.Namespace
+
     def __init__(
         self,
         pluginmanager: PytestPluginManager,
@@ -1546,41 +1554,37 @@ def _get_unknown_ini_keys(self) -> set[str]:
         known_keys = self._parser._inidict.keys() | self._parser._ini_aliases.keys()
         return self._inicfg.keys() - known_keys
 
-    def parse(self, args: list[str], addopts: bool = True) -> None:
-        # Parse given cmdline arguments into this config object.
-        assert self.args == [], (
-            "can only parse cmdline args at most once per Config object"
-        )
-
-        self.hook.pytest_addhooks.call_historic(
-            kwargs=dict(pluginmanager=self.pluginmanager)
-        )
+    def _preparse_addopts(self, args: list[str]) -> None:
+        """Prepend options from the PYTEST_ADDOPTS environment variable (in place)."""
+        env_addopts = os.environ.get("PYTEST_ADDOPTS", "")
+        if len(env_addopts):
+            args[:] = (
+                self._validate_args(shlex.split(env_addopts), "via PYTEST_ADDOPTS")
+                + args
+            )
 
-        if addopts:
-            env_addopts = os.environ.get("PYTEST_ADDOPTS", "")
-            if len(env_addopts):
-                args[:] = (
-                    self._validate_args(shlex.split(env_addopts), "via PYTEST_ADDOPTS")
-                    + args
-                )
+    def _apply_rootdir(
+        self,
+        *,
+        rootpath: pathlib.Path,
+        inipath: pathlib.Path | None,
+        inicfg: ConfigDict,
+        ignored_config_files: Sequence[str],
+    ) -> None:
+        """Apply an already-determined rootdir/inifile setup to this config.
 
-        # At this point, self.option contains only defaults from the _processopt
-        # callback.
-        ns = self._parser.parse_known_args(args, namespace=copy.copy(self.option))
-        rootpath, inipath, inicfg, ignored_config_files = determine_setup(
-            inifile=ns.inifilename,
-            override_ini=ns.override_ini,
-            args=ns.file_or_dir,
-            rootdir_cmd_arg=ns.rootdir or None,
-            invocation_dir=self.invocation_params.dir,
-        )
+        Normally fed by :func:`determine_setup`, but usable with explicitly
+        constructed values as well (experimental).
+        """
         self._rootpath = rootpath
         self._inipath = inipath
         self._ignored_config_files = ignored_config_files
         self._inicfg = inicfg
         self._parser.extra_info["rootdir"] = str(self.rootpath)
         self._parser.extra_info["inifile"] = str(self.inipath)
 
+    def _register_core_ini_options(self) -> None:
+        """Register the ini options which parse() itself consumes."""
         self._parser.addini("addopts", "Extra command line options", "args")
         self._parser.addini("minversion", "Minimally required pytest version")
         self._parser.addini(
@@ -1593,20 +1597,8 @@ def parse(self, args: list[str], addopts: bool = True) -> None:
             default=[],
         )
 
-        if addopts:
-            args[:] = (
-                self._validate_args(self.getini("addopts"), "via addopts config") + args
-            )
-
-        self.known_args_namespace = self._parser.parse_known_args(
-            args, namespace=copy.copy(self.option)
-        )
-        if addopts:
-            # addopts may have added overrides (especially via OverrideIniAction).
-            # The thing can be endlessly circular but we only do one level (#14442).
-            if overrides := parse_override_ini(self.known_args_namespace.override_ini):
-                self._inicfg.update(overrides)
-                self._inicache.clear()
+    def _load_plugins_phase(self, args: list[str]) -> None:
+        """Load plugins from args, entry points and environment; reparse known args."""
         self._checkversion()
         self._consider_importhook()
         self._configure_python_path()
@@ -1632,6 +1624,8 @@ def parse(self, args: list[str], addopts: bool = True) -> None:
         self._validate_plugins()
         self._warn_about_skipped_plugins()
 
+    def _load_initial_conftests_phase(self, args: list[str]) -> None:
+        """Default confcutdir and fire the pytest_load_initial_conftests hook."""
         if self.known_args_namespace.confcutdir is None:
             if self.inipath is not None:
                 confcutdir = str(self.inipath.parent)
@@ -1653,19 +1647,83 @@ def parse(self, args: list[str], addopts: bool = True) -> None:
             else:
                 raise
 
+    def _finalize_parse(self, args: list[str], *, decide_args: bool = True) -> None:
+        """Fully parse args into self.option and decide the initial args.
+
+        With ``decide_args=False`` (experimental), the positional args are
+        taken verbatim without the testpaths/invocation-dir fallbacks and
+        ``args_source`` is set to :attr:`ArgsSource.SPEC`.
+        """
+        if not hasattr(self, "known_args_namespace"):
+            self.known_args_namespace = self._parser.parse_known_args(
+                args, namespace=copy.copy(self.option)
+            )
         try:
             self._parser.parse(args, namespace=self.option)
         except PrintHelp:
             return
 
-        self.args, self.args_source = self._decide_args(
-            args=getattr(self.option, FILE_OR_DIR),
-            pyargs=self.option.pyargs,
-            testpaths=self.getini("testpaths"),
+        if decide_args:
+            self.args, self.args_source = self._decide_args(
+                args=getattr(self.option, FILE_OR_DIR),
+                pyargs=self.option.pyargs,
+                testpaths=self.getini("testpaths"),
+                invocation_dir=self.invocation_params.dir,
+                rootpath=self.rootpath,
+                warn=True,
+            )
+        else:
+            self.args = list(getattr(self.option, FILE_OR_DIR))
+            self.args_source = Config.ArgsSource.SPEC
+
+    def parse(self, args: list[str], addopts: bool = True) -> None:
+        # Parse given cmdline arguments into this config object.
+        assert self.args == [], (
+            "can only parse cmdline args at most once per Config object"
+        )
+
+        self.hook.pytest_addhooks.call_historic(
+            kwargs=dict(pluginmanager=self.pluginmanager)
+        )
+
+        if addopts:
+            self._preparse_addopts(args)
+
+        # At this point, self.option contains only defaults from the _processopt
+        # callback.
+        ns = self._parser.parse_known_args(args, namespace=copy.copy(self.option))
+        rootpath, inipath, inicfg, ignored_config_files = determine_setup(
+            inifile=ns.inifilename,
+            override_ini=ns.override_ini,
+            args=ns.file_or_dir,
+            rootdir_cmd_arg=ns.rootdir or None,
             invocation_dir=self.invocation_params.dir,
-            rootpath=self.rootpath,
-            warn=True,
         )
+        self._apply_rootdir(
+            rootpath=rootpath,
+            inipath=inipath,
+            inicfg=inicfg,
+            ignored_config_files=ignored_config_files,
+        )
+        self._register_core_ini_options()
+
+        if addopts:
+            args[:] = (
+                self._validate_args(self.getini("addopts"), "via addopts config") + args
+            )
+
+        self.known_args_namespace = self._parser.parse_known_args(
+            args, namespace=copy.copy(self.option)
+        )
+        if addopts:
+            # addopts may have added overrides (especially via OverrideIniAction).
+            # The thing can be endlessly circular but we only do one level (#14442).
+            if overrides := parse_override_ini(self.known_args_namespace.override_ini):
+                self._inicfg.update(overrides)
+                self._inicache.clear()
+        self._load_plugins_phase(args)
+        self._load_initial_conftests_phase(args)
+        self._finalize_parse(args)
 
     def issue_config_time_warning(self, warning: Warning, stacklevel: int) -> None:
         """Issue and handle a warning during the "configure" stage.

src/_pytest/debugging.py

@@ -330,7 +330,7 @@ def wrapper(*args, **kwargs) -> None:
 def maybe_wrap_pytest_function_for_tracing(pyfuncitem) -> None:
     """Wrap the given pytestfunct item for tracing support if --trace was given in
     the command line."""
-    if pyfuncitem.config.getvalue("trace"):
+    if pyfuncitem.config.getoption("trace", False):
         wrap_pytest_function_for_tracing(pyfuncitem)
 
 

src/_pytest/python.py

@@ -238,7 +238,7 @@ def pytest_pycollect_makemodule(module_path: Path, parent) -> Module:
 @hookimpl(trylast=True)
 def pytest_pycollect_makeitem(
     collector: Module | Class, name: str, obj: object
-) -> None | nodes.Item | nodes.Collector | list[nodes.Item | nodes.Collector]:
+) -> nodes.Item | nodes.Collector | list[nodes.Item | nodes.Collector] | None:
     assert isinstance(collector, Class | Module), type(collector)
     # Nothing was collected elsewhere, let's do it here.
     if safe_isclass(obj):
@@ -765,10 +765,22 @@ def _get_first_non_fixture_func(obj: object, names: Iterable[str]) -> object | N
 class Class(PyCollector):
     """Collector for test methods (and nested classes) in a Python class."""
 
+    #: Explicitly provided class object, taking precedence over looking up
+    #: ``name`` on the parent's object (experimental).
+    _given_obj: type | None = None
+
     @classmethod
     def from_parent(cls, parent, *, name, obj=None, **kw) -> Self:  # type: ignore[override]
         """The public constructor."""
-        return super().from_parent(name=name, parent=parent, **kw)
+        node: Self = super().from_parent(name=name, parent=parent, **kw)
+        if obj is not None:
+            node._given_obj = obj
+        return node
+
+    def _getobj(self):
+        if self._given_obj is not None:
+            return self._given_obj
+        return super()._getobj()
 
     def newinstance(self):
         return self.obj()