diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_validators/_conversation_validator.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_validators/_conversation_validator.py index 377258aa7d19..bf88c9fce756 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_validators/_conversation_validator.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_validators/_conversation_validator.py @@ -30,8 +30,9 @@ class ConversationValidator(ValidatorInterface): # ``azure_ai_search``, ``azure_fabric``, and ``sharepoint_grounding`` are # accepted by default because the shared formatter JSON-encodes their # structured tool_result payloads (see ``_stringify_tool_result``). - # ``GroundednessConversationValidator`` overrides this list to keep - # rejecting them until a context-extractor helper lands. + # ``GroundednessConversationValidator`` now also allows + # ``bing_grounding``, ``bing_custom_search``, and ``openapi_call`` while + # keeping stricter guardrails for the remaining unsupported tool families. UNSUPPORTED_TOOLS: List[str] = [ "bing_custom_search", "bing_grounding", @@ -478,22 +479,17 @@ class GroundednessConversationValidator(ConversationValidator): """ ConversationValidator override used by ``GroundednessEvaluator``. - Groundedness still rejects ``azure_ai_search``, ``azure_fabric``, and - ``sharepoint_grounding`` tool calls pending a helper that can extract - document bodies out of structured ``tool_result`` envelopes and feed - them in as the ``context`` input the groundedness judge prompt scores - against. Without that helper the judge would receive empty or wrong - context and return a silently low groundedness score, so the - enablement performed for ``ToolCallSuccessEvaluator`` and - ``ToolOutputUtilizationEvaluator`` is intentionally not extended here. - - Once the context-extractor helper lands, this subclass can be deleted - and ``GroundednessEvaluator`` can use ``ConversationValidator`` - directly. + Groundedness now accepts the same structured grounding tools enabled in + the shared ``ConversationValidator`` path, and additionally allows + ``bing_grounding``, ``bing_custom_search``, and ``openapi_call``. + + A dedicated subclass is still kept so Groundedness can preserve stricter + guardrails for the remaining unsupported tool families without changing + behavior of other evaluators. """ - UNSUPPORTED_TOOLS: List[str] = ConversationValidator.UNSUPPORTED_TOOLS + [ - "azure_ai_search", - "azure_fabric", - "sharepoint_grounding", + UNSUPPORTED_TOOLS: List[str] = [ + tool_name + for tool_name in ConversationValidator.UNSUPPORTED_TOOLS + if tool_name not in {"bing_custom_search", "bing_grounding", "openapi_call"} ] diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py index d4c3efabc463..f716cb6e8d74 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py @@ -118,9 +118,9 @@ def __init__(self, model_config, *, threshold=3, credential=None, **kwargs): self._higher_is_better = True # Initialize input validator. ``GroundednessConversationValidator`` - # carries a wider ``UNSUPPORTED_TOOLS`` list than the shared base: - # ``azure_ai_search``, ``azure_fabric``, and ``sharepoint_grounding`` - # are still rejected here pending a context-extractor helper. + # keeps stricter guardrails than the shared base for the remaining + # unsupported tool families, while allowing grounding/search/openapi + # tool calls supported by this evaluator. self._validator = GroundednessConversationValidator( error_target=ErrorTarget.GROUNDEDNESS_EVALUATOR, requires_query=False, diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_common_validators.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_common_validators.py index 9dc831ddd638..75211dfee417 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_common_validators.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_common_validators.py @@ -616,9 +616,9 @@ def test_query_response_fallback_no_enforce_tool_definitions(self): # ``ToolCallsValidator`` inherit that narrowed default, so TCS and TOU # accept those tool calls. # -# ``GroundednessConversationValidator`` keeps the wider list -- Groundedness -# still rejects those three tools pending a context-extractor helper that -# can derive a grounding ``context`` from structured ``tool_result`` payloads. +# ``GroundednessConversationValidator`` now narrows further and allows +# ``bing_grounding``, ``bing_custom_search``, and ``openapi_call`` while +# preserving stricter guardrails only for the remaining unsupported families. NEWLY_ENABLED_TOOLS = [ "azure_ai_search", @@ -636,6 +636,19 @@ def test_query_response_fallback_no_enforce_tool_definitions(self): "web_search", ] +GROUNDEDNESS_NEWLY_ENABLED_TOOLS = [ + "bing_grounding", + "bing_custom_search", + "openapi_call", +] + +GROUNDEDNESS_STILL_UNSUPPORTED_TOOLS = [ + "browser_automation", + "code_interpreter_call", + "computer_call", + "web_search", +] + @pytest.mark.unittest class TestUnsupportedToolsListConversationValidator: @@ -661,23 +674,30 @@ def test_tool_definitions_validator_inherits_same_list(self): @pytest.mark.unittest class TestUnsupportedToolsListGroundednessValidator: - """Groundedness keeps the wider list via its dedicated subclass.""" + """Groundedness keeps a dedicated but narrower unsupported-tools list.""" - def test_full_unsupported_list_contains_newly_enabled_tools(self): - for tool_name in NEWLY_ENABLED_TOOLS: - assert tool_name in GroundednessConversationValidator.UNSUPPORTED_TOOLS + def test_groundedness_unsupported_list_does_not_include_newly_enabled_tools(self): + for tool_name in NEWLY_ENABLED_TOOLS + GROUNDEDNESS_NEWLY_ENABLED_TOOLS: + assert tool_name not in GroundednessConversationValidator.UNSUPPORTED_TOOLS - def test_full_unsupported_list_contains_still_unsupported_tools(self): - for tool_name in STILL_UNSUPPORTED_TOOLS: + def test_groundedness_unsupported_list_contains_still_unsupported_tools(self): + for tool_name in GROUNDEDNESS_STILL_UNSUPPORTED_TOOLS: assert tool_name in GroundednessConversationValidator.UNSUPPORTED_TOOLS - def test_full_unsupported_list_exact_match(self): - assert set(GroundednessConversationValidator.UNSUPPORTED_TOOLS) == set( - NEWLY_ENABLED_TOOLS + STILL_UNSUPPORTED_TOOLS - ) + def test_groundedness_unsupported_list_exact_match(self): + assert set(GroundednessConversationValidator.UNSUPPORTED_TOOLS) == set(GROUNDEDNESS_STILL_UNSUPPORTED_TOOLS) - @pytest.mark.parametrize("tool_name", NEWLY_ENABLED_TOOLS) - def test_groundedness_validator_still_rejects_newly_enabled_tools(self, tool_name): + @pytest.mark.parametrize("tool_name", NEWLY_ENABLED_TOOLS + GROUNDEDNESS_NEWLY_ENABLED_TOOLS) + def test_groundedness_validator_accepts_newly_enabled_tools(self, tool_name): + validator = GroundednessConversationValidator(error_target=TARGET, check_for_unsupported_tools=True) + assistant = { + "role": "assistant", + "content": [_tool_call_content_item(name=tool_name)], + } + assert validator.validate_eval_input({"query": [_user_message()], "response": [assistant]}) is True + + @pytest.mark.parametrize("tool_name", GROUNDEDNESS_STILL_UNSUPPORTED_TOOLS) + def test_groundedness_validator_rejects_still_unsupported_tools(self, tool_name): validator = GroundednessConversationValidator(error_target=TARGET, check_for_unsupported_tools=True) assistant = { "role": "assistant",