Deprecate legacy top level exports

docs/changes/newsfragments/8252.breaking

@@ -0,0 +1,14 @@
+Importing the short hand aliases (such as ``Parameter``, ``Measurement``,
+``Instrument``, ``Monitor`` and ``Station``) from the top level ``qcodes``
+namespace is now deprecated and emits a ``QCoDeSDeprecationWarning``. Import
+these names from their respective submodules (for example ``qcodes.parameters``,
+``qcodes.dataset`` or ``qcodes.station``) instead. The submodules themselves
+(such as ``qcodes.dataset`` and ``qcodes.instrument``) remain accessible from the
+top level ``qcodes`` namespace and are not deprecated.
+
+Similarly, importing the parameter classes (such as ``Parameter``, ``ManualParameter`` and
+``DelegateParameter``) from ``qcodes.instrument`` is now deprecated and emits a
+``QCoDeSDeprecationWarning``. Import these names from ``qcodes.parameters``
+instead. The names remain available from ``qcodes.instrument`` at runtime for
+backwards compatibility but are hidden from static type checkers, so importing
+them from ``qcodes.instrument`` is reported as an unknown attribute.

docs/changes/newsfragments/8252.underthehood

@@ -0,0 +1,10 @@
+Importing the top level ``qcodes`` namespace no longer eagerly imports the
+``dataset``, ``instrument``, ``parameters``, ``monitor``, ``station`` and
+``validators`` submodules. The discouraged short hand aliases (such as
+``qcodes.Parameter`` and ``qcodes.Measurement``) as well as ``qcodes.validators``
+are still available at runtime via a lazy module level ``__getattr__``, but they
+are intentionally hidden from static type checkers and are now reported as
+unknown attributes rather than typed as ``Any``; import the names from their
+respective submodules to retain type information. This breaks a large import
+cycle spanning the parameters, dataset and instrument packages that triggered an
+internal error in ``mypy`` 2.2.

docs/examples/Parameters/Legacy_Parameters.ipynb

@@ -32,7 +32,7 @@
     }
    ],
    "source": [
-    "from qcodes.instrument import ArrayParameter\n",
+    "from qcodes.parameters import ArrayParameter\n",
     "\n",
     "\n",
     "class ArrayCounter(ArrayParameter):\n",

src/qcodes/__init__.py

@@ -1,14 +1,8 @@
 """Set up the main qcodes namespace."""
 
-# ruff: noqa: F401, E402
-# This module still contains a lot of short hand imports
-# since these imports are discouraged and they are officially
-# added elsewhere under their respective submodules we cannot add
-# them to __all__ here so silence the warning.
-
-# config
+import importlib
 import warnings
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import qcodes._version
 import qcodes.configuration as qcconfig
@@ -22,62 +16,116 @@
 
 conditionally_start_all_logging()
 
-import atexit
-
-import qcodes.validators
-from qcodes.dataset import (
-    Measurement,
-    ParamSpec,
-    SQLiteSettings,
-    experiments,
-    get_guids_by_run_spec,
-    initialise_database,
-    initialise_or_create_database_at,
-    initialised_database_at,
-    load_by_counter,
-    load_by_guid,
-    load_by_id,
-    load_by_run_spec,
-    load_experiment,
-    load_experiment_by_name,
-    load_last_experiment,
-    load_or_create_experiment,
-    new_data_set,
-    new_experiment,
-)
-from qcodes.instrument import (
-    ChannelList,
-    ChannelTuple,
-    Instrument,
-    InstrumentChannel,
-    IPInstrument,
-    VisaInstrument,
-    find_or_create_instrument,
-)
-from qcodes.monitor import Monitor
-from qcodes.parameters import (
-    ArrayParameter,
-    CombinedParameter,
-    DelegateParameter,
-    Function,
-    ManualParameter,
-    MultiParameter,
-    Parameter,
-    ParameterWithSetpoints,
-    ScaledParameter,
-    SweepFixedValues,
-    SweepValues,
-    combine,
-)
-from qcodes.station import Station
-
-# ensure to close all instruments when interpreter is closed
-atexit.register(Instrument.close_all)
-
 if config.core.import_legacy_api:
     warnings.warn(
         "`core.import_legacy_api` and `gui.plotlib` config option has no effect "
         "and will be removed in the future. "
         "Please avoid setting this in your `qcodesrc.json` config file.",
         QCoDeSDeprecationWarning,
     )
+
+# The following names are re-exported for backwards compatibility as short hand
+# for the objects in their respective submodules. Importing them from the top
+# level ``qcodes`` namespace is deprecated (import them from the submodule
+# instead); they are provided lazily via a module level ``__getattr__`` that
+# emits a ``QCoDeSDeprecationWarning``. Importing them eagerly here would also
+# pull ``qcodes.dataset``, ``qcodes.instrument``, ``qcodes.parameters``,
+# ``qcodes.monitor`` and ``qcodes.station`` into a single large import cycle at
+# type-check time (which triggers an internal error in mypy >= 2.2).
+_LAZY_NAME_TO_MODULE = (
+    {
+        name: "qcodes.dataset"
+        for name in (
+            "Measurement",
+            "ParamSpec",
+            "SQLiteSettings",
+            "experiments",
+            "get_guids_by_run_spec",
+            "initialise_database",
+            "initialise_or_create_database_at",
+            "initialised_database_at",
+            "load_by_counter",
+            "load_by_guid",
+            "load_by_id",
+            "load_by_run_spec",
+            "load_experiment",
+            "load_experiment_by_name",
+            "load_last_experiment",
+            "load_or_create_experiment",
+            "new_data_set",
+            "new_experiment",
+        )
+    }
+    | {
+        name: "qcodes.instrument"
+        for name in (
+            "ChannelList",
+            "ChannelTuple",
+            "Instrument",
+            "InstrumentChannel",
+            "IPInstrument",
+            "VisaInstrument",
+            "find_or_create_instrument",
+        )
+    }
+    | {
+        name: "qcodes.parameters"
+        for name in (
+            "ArrayParameter",
+            "CombinedParameter",
+            "DelegateParameter",
+            "Function",
+            "ManualParameter",
+            "MultiParameter",
+            "Parameter",
+            "ParameterWithSetpoints",
+            "ScaledParameter",
+            "SweepFixedValues",
+            "SweepValues",
+            "combine",
+        )
+    }
+    | {
+        "Monitor": "qcodes.monitor",
+        "Station": "qcodes.station",
+    }
+)
+
+
+# These public submodules are exposed lazily so that ``import qcodes`` stays
+# cheap and does not force the submodules (and their dependencies) to be
+# imported. Accessing them is not deprecated. The submodules that the deprecated
+# short hands above live in are included here so that ``qcodes.dataset`` and
+# friends keep working as submodule accessors once the deprecated short hands are
+# eventually removed.
+_LAZY_SUBMODULES = frozenset(
+    {
+        "validators",
+        "dataset",
+        "instrument",
+        "parameters",
+        "monitor",
+        "station",
+    }
+)
+
+
+# The lazy ``__getattr__`` is intentionally hidden from static type checkers via
+# ``if not TYPE_CHECKING`` so that these discouraged short hands are reported as
+# unknown attributes (rather than typed as ``Any``); import the names from their
+# respective submodules to get proper type information.
+if not TYPE_CHECKING:
+
+    def __getattr__(name: str) -> Any:
+        if name in _LAZY_SUBMODULES:
+            return importlib.import_module(f"{__name__}.{name}")
+        module_name = _LAZY_NAME_TO_MODULE.get(name)
+        if module_name is not None:
+            warnings.warn(
+                f"Importing {name!r} from the top level {__name__!r} namespace "
+                f"is deprecated. Import it from {module_name!r} instead.",
+                QCoDeSDeprecationWarning,
+                stacklevel=2,
+            )
+            return getattr(importlib.import_module(module_name), name)
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

src/qcodes/instrument/__init__.py

@@ -1,26 +1,21 @@
-# left here for backwards compatibility
-# but not part of the api officially
-from qcodes.parameters import (  # noqa: F401
-    ArrayParameter,
-    CombinedParameter,
-    DelegateParameter,
-    Function,
-    ManualParameter,
-    MultiParameter,
-    Parameter,
-    ParameterWithSetpoints,
-    ScaledParameter,
-    SweepFixedValues,
-    SweepValues,
-    combine,
-)
+import atexit
+import importlib
+import warnings
+from typing import TYPE_CHECKING, Any
+
+from qcodes.utils import QCoDeSDeprecationWarning
 
 from .channel import ChannelList, ChannelTuple, InstrumentChannel, InstrumentModule
 from .instrument import Instrument, find_or_create_instrument
 from .instrument_base import InstrumentBase, InstrumentBaseKWArgs
 from .ip import IPInstrument
 from .visa import VisaInstrument, VisaInstrumentKWArgs
 
+# ensure that all instruments are closed when the interpreter is shut down.
+# this is registered here rather than in ``qcodes.__init__`` so that importing
+# the top level ``qcodes`` package does not eagerly import ``qcodes.instrument``.
+atexit.register(Instrument.close_all)
+
 __all__ = [
     "ChannelList",
     "ChannelTuple",
@@ -34,3 +29,43 @@
     "VisaInstrumentKWArgs",
     "find_or_create_instrument",
 ]
+
+# The following parameter classes used to be re-exported from
+# ``qcodes.instrument`` for backwards compatibility. They now live in
+# ``qcodes.parameters`` and importing them from here is deprecated. They are
+# provided lazily via a module level ``__getattr__`` that emits a deprecation
+# warning; import them from ``qcodes.parameters`` instead.
+_DEPRECATED_PARAMETER_NAMES = frozenset(
+    {
+        "ArrayParameter",
+        "CombinedParameter",
+        "DelegateParameter",
+        "Function",
+        "ManualParameter",
+        "MultiParameter",
+        "Parameter",
+        "ParameterWithSetpoints",
+        "ScaledParameter",
+        "SweepFixedValues",
+        "SweepValues",
+        "combine",
+    }
+)
+
+
+# The lazy ``__getattr__`` is intentionally hidden from static type checkers via
+# ``if not TYPE_CHECKING`` so that importing these deprecated names from
+# ``qcodes.instrument`` is reported as an unknown attribute; import them from
+# ``qcodes.parameters`` instead.
+if not TYPE_CHECKING:
+
+    def __getattr__(name: str) -> Any:
+        if name in _DEPRECATED_PARAMETER_NAMES:
+            warnings.warn(
+                f"Importing {name!r} from {__name__!r} is deprecated. "
+                f"Import it from 'qcodes.parameters' instead.",
+                QCoDeSDeprecationWarning,
+                stacklevel=2,
+            )
+            return getattr(importlib.import_module("qcodes.parameters"), name)
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

src/qcodes/instrument/instrument.py

@@ -180,7 +180,10 @@ def close(self) -> None:
         self.remove_instance(self)
 
     @classmethod
-    def close_all(cls) -> None:
+    def close_all(
+        cls,
+        log_status: bool = False,
+    ) -> None:
         """
         Try to close all instruments registered in
         ``_all_instruments`` This is handy for use with atexit to
@@ -190,15 +193,22 @@ def close_all(cls) -> None:
         Examples:
             >>> atexit.register(qc.Instrument.close_all())
 
+        Args:
+            log_status: If True, log the status of closing each instrument. Set this to False
+              if you want to avoid logging during interpreter shutdown, which can cause errors.
+
         """
-        log.info("Closing all registered instruments")
+        if log_status:
+            log.info("Closing all registered instruments")
         for inststr in list(cls._all_instruments):
             try:
                 inst: Instrument = cls.find_instrument(inststr)
-                log.info("Closing %s", inststr)
+                if log_status:
+                    log.info("Closing %s", inststr)
                 inst.close()
             except Exception:
-                log.exception("Failed to close %s, ignored", inststr)
+                if log_status:
+                    log.exception("Failed to close %s, ignored", inststr)
 
     @classmethod
     def record_instance(cls, instance: Instrument) -> None: