From fd0670a80d64471b0b923c2e5d1adb7752e309d3 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:20:14 +0000 Subject: [PATCH 1/5] Graduate harness agent --- python/packages/core/AGENTS.md | 1 + .../core/agent_framework/_harness/_agent.py | 67 +++++++++- .../core/tests/core/test_harness_agent.py | 121 ++++++++++++++++++ 3 files changed, 187 insertions(+), 2 deletions(-) diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index 84f8d04701..9517fadcff 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -126,6 +126,7 @@ agent_framework/ - **Descriptions & index** - `file_memory_write` accepts an optional `description`, stored in a companion `_description.md` sidecar. After each write/delete the provider rebuilds a capped (50-entry) `memories.md` index, and `before_run` injects that index as a `user` context message so the model knows what memories exist. Sidecars and the index are internal files hidden from `file_memory_ls`/`file_memory_grep` and rejected as write targets. - **`DEFAULT_FILE_MEMORY_SOURCE_ID`** / **`DEFAULT_FILE_MEMORY_INSTRUCTIONS`** - Public defaults for the provider's source id and instruction banner. - **Harness wiring** - `create_harness_agent` includes the `FileMemoryProvider` by default; the `FileAccessProvider` is opt-in and added only when a `file_access_store` is supplied (no implicit `{cwd}/working` store is created). Disable file memory via `disable_file_memory`; override its backing store via `file_memory_store`. When no file-memory store is supplied, the default is `FileSystemAgentFileStore` rooted at `{cwd}/agent-file-memory`. `create_harness_agent` also wires in `MessageInjectionMiddleware` by default (mirroring the .NET harness's `UseMessageInjection`); it is always on with no opt-out because it is a no-op when no messages are queued for the session. +- **Experimental-feature gating** - `create_harness_agent` itself is released (no longer `@experimental`), but a few features it can wire in remain experimental: **background agents** (`background_agents`), **file access** (`file_access_store`), and **looping** (`loop_should_continue`). Enabling any of them emits a single `ExperimentalWarning` (naming the responsible parameter) and seeds the shared `HARNESS` dedup key so the downstream experimental provider does not warn a second time. ### Tool Approval Harness (`_harness/_tool_approval.py`) diff --git a/python/packages/core/agent_framework/_harness/_agent.py b/python/packages/core/agent_framework/_harness/_agent.py index a29472e1b8..25045103d4 100644 --- a/python/packages/core/agent_framework/_harness/_agent.py +++ b/python/packages/core/agent_framework/_harness/_agent.py @@ -12,6 +12,7 @@ import logging import sys +import warnings from collections.abc import Callable, Sequence from pathlib import Path from typing import TYPE_CHECKING, Any, TypedDict @@ -19,7 +20,7 @@ from .._agents import Agent, SupportsAgentRun from .._clients import SupportsShellTool, SupportsWebSearchTool from .._compaction import CompactionProvider, ContextWindowCompactionStrategy -from .._feature_stage import ExperimentalFeature, experimental +from .._feature_stage import _WARNED_FEATURES, ExperimentalFeature, ExperimentalWarning from .._sessions import ContextProvider, HistoryProvider, InMemoryHistoryProvider, MessageInjectionMiddleware from .._skills import SkillsProvider from .._types import ChatOptions @@ -269,8 +270,40 @@ def _assemble_shell( default="ChatOptions[None]", ) +# Dedup label for the pre-release shell tooling (provided by the alpha-stage +# agent-framework-tools package). It is not a feature-stage-decorated feature, so it uses a +# label distinct from ExperimentalFeature.HARNESS to avoid suppressing unrelated HARNESS warnings. +_SHELL_TOOLING_FEATURE_ID = "SHELL_TOOLING" + + +def _warn_experimental_harness_params( + param_names: Sequence[str], + *, + feature_id: str, + detail: str, +) -> None: + """Emit a single ExperimentalWarning when experimental harness features are enabled. + + ``create_harness_agent`` itself is released, but some of the features it can wire in + remain experimental or pre-release. When a caller opts into one of those features, warn + once (pointing at the caller's call site) naming the responsible parameter(s), and seed a + per-feature dedup key so the downstream experimental provider does not warn again. + """ + if not param_names: + return + dedup_key = (ExperimentalWarning, feature_id) + if dedup_key in _WARNED_FEATURES: + return + joined = ", ".join(repr(name) for name in param_names) + warnings.warn( + f"[{feature_id}] create_harness_agent parameter(s) {joined} enable " + f"{detail} that may change or be removed in future versions without notice.", + ExperimentalWarning, + stacklevel=3, + ) + _WARNED_FEATURES.add(dedup_key) + -@experimental(feature_id=ExperimentalFeature.HARNESS) def create_harness_agent( client: SupportsChatGetResponse[OptionsCoT], *, @@ -334,6 +367,14 @@ def create_harness_agent( Each feature can be disabled or customized via keyword arguments. + .. note:: Experimental features + + ``create_harness_agent`` is released, but a few of the features it can wire in are + still experimental or pre-release: **background agents** (``background_agents``), + **file access** (``file_access_store``), **looping** (``loop_should_continue``), and + the **shell tooling** (``shell_executor``, provided by the pre-release + ``agent-framework-tools`` package). Enabling any of them emits an ``ExperimentalWarning``. + Examples: Basic usage: @@ -502,6 +543,28 @@ def create_harness_agent( ): raise ValueError("max_output_tokens must be less than max_context_window_tokens.") + # Warn when opting into harness features that are still experimental. create_harness_agent + # itself is released, but background agents, file access, and looping remain experimental, + # and the shell tooling is provided by the pre-release agent-framework-tools package. + experimental_params: list[str] = [] + if background_agents: + experimental_params.append("background_agents") + if file_access_store is not None: + experimental_params.append("file_access_store") + if loop_should_continue is not None: + experimental_params.append("loop_should_continue") + _warn_experimental_harness_params( + experimental_params, + feature_id=ExperimentalFeature.HARNESS.value, + detail="experimental harness features", + ) + if shell_executor is not None: + _warn_experimental_harness_params( + ["shell_executor"], + feature_id=_SHELL_TOOLING_FEATURE_ID, + detail="pre-release shell tooling from the agent-framework-tools package", + ) + # Build history provider. resolved_history = history_provider or InMemoryHistoryProvider() diff --git a/python/packages/core/tests/core/test_harness_agent.py b/python/packages/core/tests/core/test_harness_agent.py index ecb0c9025f..893e9f3af7 100644 --- a/python/packages/core/tests/core/test_harness_agent.py +++ b/python/packages/core/tests/core/test_harness_agent.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib.util +import warnings from collections.abc import AsyncIterable, Awaitable, Mapping, Sequence from pathlib import Path from typing import Any @@ -1271,3 +1272,123 @@ def _next_message(**kwargs: Any) -> Any: loop_max_iterations=5, ) assert _find_loop_middleware(agent) is None + + +# --- Experimental graduation / gating Tests --- + + +def _clear_harness_experimental_dedup() -> None: + """Drop the shared HARNESS dedup key so ExperimentalWarning can fire again in a test.""" + from agent_framework._feature_stage import _WARNED_FEATURES, ExperimentalFeature, ExperimentalWarning + + _WARNED_FEATURES.discard((ExperimentalWarning, ExperimentalFeature.HARNESS.value)) + + +def _clear_harness_shell_dedup() -> None: + """Drop the shell-tooling dedup key so ExperimentalWarning can fire again in a test.""" + from agent_framework._feature_stage import _WARNED_FEATURES, ExperimentalWarning + from agent_framework._harness._agent import _SHELL_TOOLING_FEATURE_ID + + _WARNED_FEATURES.discard((ExperimentalWarning, _SHELL_TOOLING_FEATURE_ID)) + + +def test_create_harness_agent_is_not_experimental() -> None: + """create_harness_agent is graduated and should no longer carry experimental metadata.""" + assert getattr(create_harness_agent, "__feature_stage__", None) is None + assert getattr(create_harness_agent, "__feature_id__", None) is None + + +def test_create_harness_agent_graduated_features_emit_no_experimental_warning() -> None: + """Using only graduated features must not emit an ExperimentalWarning.""" + from agent_framework._feature_stage import ExperimentalWarning + + _clear_harness_experimental_dedup() + with warnings.catch_warnings(): + warnings.simplefilter("error", ExperimentalWarning) + create_harness_agent( + client=_FakeChatClient(), + max_context_window_tokens=128_000, + max_output_tokens=16_384, + disable_web_search=True, + disable_file_memory=True, + ) + + +def test_create_harness_agent_background_agents_emits_experimental_warning() -> None: + """Opting into background agents (still experimental) should warn and still wire the provider.""" + from agent_framework._feature_stage import ExperimentalWarning + from agent_framework._harness._background_agents import BackgroundAgentsProvider + + _clear_harness_experimental_dedup() + bg_agent = _FakeBackgroundAgent("WebSearcher", "Searches the web") + with pytest.warns(ExperimentalWarning, match="background_agents"): + agent = create_harness_agent( + client=_FakeChatClient(), + max_context_window_tokens=128_000, + max_output_tokens=16_384, + disable_web_search=True, + disable_file_memory=True, + background_agents=[bg_agent], # type: ignore[list-item] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + ) + assert any(isinstance(p, BackgroundAgentsProvider) for p in (agent.context_providers or [])) + + +def test_create_harness_agent_file_access_store_emits_experimental_warning() -> None: + """Opting into file access (still experimental) should warn and still wire the provider.""" + from agent_framework._feature_stage import ExperimentalWarning + + _clear_harness_experimental_dedup() + store = InMemoryAgentFileStore() + _clear_harness_experimental_dedup() + with pytest.warns(ExperimentalWarning, match="file_access_store"): + agent = create_harness_agent( + client=_FakeChatClient(), + max_context_window_tokens=128_000, + max_output_tokens=16_384, + disable_web_search=True, + disable_file_memory=True, + file_access_store=store, + ) + assert any(isinstance(p, FileAccessProvider) for p in (agent.context_providers or [])) + + +def test_create_harness_agent_loop_should_continue_emits_experimental_warning() -> None: + """Opting into looping (still experimental) should warn and still wire the loop middleware.""" + from agent_framework import AgentLoopMiddleware + from agent_framework._feature_stage import ExperimentalWarning + + def _should_continue(**kwargs: Any) -> bool: + return False + + _clear_harness_experimental_dedup() + with pytest.warns(ExperimentalWarning, match="loop_should_continue"): + agent = create_harness_agent( + client=_FakeChatClient(), # type: ignore[arg-type] + max_context_window_tokens=128_000, + max_output_tokens=16_384, + disable_web_search=True, + disable_file_memory=True, + loop_should_continue=_should_continue, + ) + assert agent.middleware is not None + assert isinstance(agent.middleware[0], AgentLoopMiddleware) + + +@_requires_shell_tools +def test_create_harness_agent_shell_executor_emits_experimental_warning() -> None: + """Opting into shell tooling (pre-release) should warn and still wire the shell tool.""" + from agent_framework._feature_stage import ExperimentalWarning + + _clear_harness_experimental_dedup() + _clear_harness_shell_dedup() + client = _FakeShellClient() + with pytest.warns(ExperimentalWarning, match="shell_executor"): + agent = create_harness_agent( + client=client, + max_context_window_tokens=128_000, + max_output_tokens=16_384, + disable_web_search=True, + disable_file_memory=True, + shell_executor=_FakeShellTool(), + ) + assert "shell_tool_instance" in agent.default_options.get("tools", []) From 9532bf7b86f30e9ed2b651123ced23e20b02dbcb Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:20:42 +0000 Subject: [PATCH 2/5] Add agents.md update --- python/packages/core/AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index 9517fadcff..9327e9d4ad 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -126,7 +126,7 @@ agent_framework/ - **Descriptions & index** - `file_memory_write` accepts an optional `description`, stored in a companion `_description.md` sidecar. After each write/delete the provider rebuilds a capped (50-entry) `memories.md` index, and `before_run` injects that index as a `user` context message so the model knows what memories exist. Sidecars and the index are internal files hidden from `file_memory_ls`/`file_memory_grep` and rejected as write targets. - **`DEFAULT_FILE_MEMORY_SOURCE_ID`** / **`DEFAULT_FILE_MEMORY_INSTRUCTIONS`** - Public defaults for the provider's source id and instruction banner. - **Harness wiring** - `create_harness_agent` includes the `FileMemoryProvider` by default; the `FileAccessProvider` is opt-in and added only when a `file_access_store` is supplied (no implicit `{cwd}/working` store is created). Disable file memory via `disable_file_memory`; override its backing store via `file_memory_store`. When no file-memory store is supplied, the default is `FileSystemAgentFileStore` rooted at `{cwd}/agent-file-memory`. `create_harness_agent` also wires in `MessageInjectionMiddleware` by default (mirroring the .NET harness's `UseMessageInjection`); it is always on with no opt-out because it is a no-op when no messages are queued for the session. -- **Experimental-feature gating** - `create_harness_agent` itself is released (no longer `@experimental`), but a few features it can wire in remain experimental: **background agents** (`background_agents`), **file access** (`file_access_store`), and **looping** (`loop_should_continue`). Enabling any of them emits a single `ExperimentalWarning` (naming the responsible parameter) and seeds the shared `HARNESS` dedup key so the downstream experimental provider does not warn a second time. +- **Experimental-feature gating** - `create_harness_agent` itself is released (no longer `@experimental`), but a few features it can wire in remain experimental or pre-release: **background agents** (`background_agents`), **file access** (`file_access_store`), **looping** (`loop_should_continue`), and the **shell tooling** (`shell_executor`, from the pre-release `agent-framework-tools` package). Enabling any of them emits a single `ExperimentalWarning` (naming the responsible parameter). The three experimental harness providers share one `HARNESS` dedup key so the downstream experimental provider does not warn a second time; the shell tooling uses a separate dedup key (it is not a `HARNESS`-decorated feature). ### Tool Approval Harness (`_harness/_tool_approval.py`) From e75d9bd53bf718182caceeb9f0e922dbf4691aad Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:41:00 +0000 Subject: [PATCH 3/5] Fix build errors --- .../core/agent_framework/_feature_stage.py | 39 +++++++++++++++++++ .../core/agent_framework/_harness/_agent.py | 12 ++---- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/python/packages/core/agent_framework/_feature_stage.py b/python/packages/core/agent_framework/_feature_stage.py index 2d7c97c949..39f9c74f4b 100644 --- a/python/packages/core/agent_framework/_feature_stage.py +++ b/python/packages/core/agent_framework/_feature_stage.py @@ -251,6 +251,45 @@ def _warn_on_feature_use( _WARNED_FEATURES.add(warning_key) +def warn_experimental_feature( + message: str, + *, + feature_id: str | Enum, + category: type[Warning] = ExperimentalWarning, +) -> bool: + """Emit a one-time feature-stage warning for a feature not gated by a decorator. + + Some released APIs opt callers into experimental behaviour through individual + parameters, which Python cannot decorate on their own. Call this to warn once per + ``feature_id`` with a custom ``message`` (pointing at the caller's call site) and to + seed the shared dedup registry, so a downstream decorated provider for the same + ``feature_id`` does not warn a second time. + + Returns ``True`` when a warning was emitted, ``False`` when it was already emitted for + this ``feature_id``/``category`` earlier in the process. + """ + normalized_feature_id = _normalize_feature_id(feature_id) + warning_key = (category, normalized_feature_id) + if warning_key in _WARNED_FEATURES: + return False + + user_frame = _resolve_user_frame() + if user_frame is None: + # Last-resort fallback: emit at the immediate caller of this helper. + warnings.warn(message, category=category, stacklevel=2) + else: + filename, lineno, module = user_frame + warnings.warn_explicit( + message, + category=category, + filename=filename, + lineno=lineno, + module=module, + ) + _WARNED_FEATURES.add(warning_key) + return True + + def _add_runtime_warning( obj: FeatureStageT, *, diff --git a/python/packages/core/agent_framework/_harness/_agent.py b/python/packages/core/agent_framework/_harness/_agent.py index 25045103d4..7e17a3d350 100644 --- a/python/packages/core/agent_framework/_harness/_agent.py +++ b/python/packages/core/agent_framework/_harness/_agent.py @@ -12,7 +12,6 @@ import logging import sys -import warnings from collections.abc import Callable, Sequence from pathlib import Path from typing import TYPE_CHECKING, Any, TypedDict @@ -20,7 +19,7 @@ from .._agents import Agent, SupportsAgentRun from .._clients import SupportsShellTool, SupportsWebSearchTool from .._compaction import CompactionProvider, ContextWindowCompactionStrategy -from .._feature_stage import _WARNED_FEATURES, ExperimentalFeature, ExperimentalWarning +from .._feature_stage import ExperimentalFeature, warn_experimental_feature from .._sessions import ContextProvider, HistoryProvider, InMemoryHistoryProvider, MessageInjectionMiddleware from .._skills import SkillsProvider from .._types import ChatOptions @@ -291,17 +290,12 @@ def _warn_experimental_harness_params( """ if not param_names: return - dedup_key = (ExperimentalWarning, feature_id) - if dedup_key in _WARNED_FEATURES: - return joined = ", ".join(repr(name) for name in param_names) - warnings.warn( + warn_experimental_feature( f"[{feature_id}] create_harness_agent parameter(s) {joined} enable " f"{detail} that may change or be removed in future versions without notice.", - ExperimentalWarning, - stacklevel=3, + feature_id=feature_id, ) - _WARNED_FEATURES.add(dedup_key) def create_harness_agent( From 62cdabe6fea378d32b52a2c37dbc291269772bde Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:46:26 +0000 Subject: [PATCH 4/5] Address PR comments --- .../core/tests/core/test_harness_agent.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/python/packages/core/tests/core/test_harness_agent.py b/python/packages/core/tests/core/test_harness_agent.py index 893e9f3af7..fb8cab0506 100644 --- a/python/packages/core/tests/core/test_harness_agent.py +++ b/python/packages/core/tests/core/test_harness_agent.py @@ -1392,3 +1392,41 @@ def test_create_harness_agent_shell_executor_emits_experimental_warning() -> Non shell_executor=_FakeShellTool(), ) assert "shell_tool_instance" in agent.default_options.get("tools", []) + + +@_requires_shell_tools +def test_create_harness_agent_shell_dedup_does_not_suppress_harness_warning() -> None: + """The shell tooling uses a dedup key separate from HARNESS. + + Enabling shell tooling must only seed the SHELL_TOOLING dedup key, so a subsequent + opt-in to an experimental HARNESS feature (e.g. ``background_agents``) must still warn. + This guards the separate-dedup-key strategy against regressions. + """ + from agent_framework._feature_stage import ExperimentalWarning + + _clear_harness_experimental_dedup() + _clear_harness_shell_dedup() + + # Enabling shell tooling seeds only the SHELL_TOOLING dedup key, not HARNESS. + with pytest.warns(ExperimentalWarning, match="shell_executor"): + create_harness_agent( + client=_FakeShellClient(), + max_context_window_tokens=128_000, + max_output_tokens=16_384, + disable_web_search=True, + disable_file_memory=True, + shell_executor=_FakeShellTool(), + ) + + # HARNESS was never seeded by the shell warning, so opting into an experimental + # HARNESS feature afterwards must still emit its own ExperimentalWarning. + bg_agent = _FakeBackgroundAgent("WebSearcher", "Searches the web") + with pytest.warns(ExperimentalWarning, match="background_agents"): + create_harness_agent( + client=_FakeChatClient(), + max_context_window_tokens=128_000, + max_output_tokens=16_384, + disable_web_search=True, + disable_file_memory=True, + background_agents=[bg_agent], # type: ignore[list-item] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + ) From e5d95565b2d897fd2138334fb96e6ba1c621dbeb Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:29:03 +0000 Subject: [PATCH 5/5] Fix build error --- python/packages/core/agent_framework/_harness/_agent.pyi | 1 + 1 file changed, 1 insertion(+) diff --git a/python/packages/core/agent_framework/_harness/_agent.pyi b/python/packages/core/agent_framework/_harness/_agent.pyi index 3a9906e917..26a57288a7 100644 --- a/python/packages/core/agent_framework/_harness/_agent.pyi +++ b/python/packages/core/agent_framework/_harness/_agent.pyi @@ -22,6 +22,7 @@ from ._tool_approval import ToolApprovalRuleCallback DEFAULT_HARNESS_INSTRUCTIONS: str HARNESS_AGENT_PROVIDER_NAME: str +_SHELL_TOOLING_FEATURE_ID: str OptionsCoT = TypeVar( "OptionsCoT",