diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_validators/_conversation_validator.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_validators/_conversation_validator.py index 377258aa7d19..8750daefbde6 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_validators/_conversation_validator.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_validators/_conversation_validator.py @@ -129,7 +129,9 @@ def _validate_dict_field( ) return None - def _validate_text_content_item(self, content_item: Dict[str, Any], role: str) -> Optional[EvaluationException]: + def _validate_text_content_item( + self, content_item: Dict[str, Any], role: str + ) -> Optional[EvaluationException]: """Validate a text content item.""" if "text" not in content_item: return EvaluationException( @@ -148,7 +150,9 @@ def _validate_text_content_item(self, content_item: Dict[str, Any], role: str) - ) return None - def _validate_tool_call_content_item(self, content_item: Dict[str, Any]) -> Optional[EvaluationException]: + def _validate_tool_call_content_item( + self, content_item: Dict[str, Any] + ) -> Optional[EvaluationException]: """Validate a tool_call content item.""" valid_tool_call_content_types = [ ContentType.TOOL_CALL, @@ -156,8 +160,13 @@ def _validate_tool_call_content_item(self, content_item: Dict[str, Any]) -> Opti ContentType.OPENAPI_CALL, ContentType.MCP_APPROVAL_REQUEST, ] - valid_tool_call_content_types_as_strings = [t.value for t in valid_tool_call_content_types] - if "type" not in content_item or content_item["type"] not in valid_tool_call_content_types: + valid_tool_call_content_types_as_strings = [ + t.value for t in valid_tool_call_content_types + ] + if ( + "type" not in content_item + or content_item["type"] not in valid_tool_call_content_types + ): return EvaluationException( message=f"The content item must be of type {valid_tool_call_content_types_as_strings} in tool_call content item.", blame=ErrorBlame.USER_ERROR, @@ -168,21 +177,29 @@ def _validate_tool_call_content_item(self, content_item: Dict[str, Any]) -> Opti if content_item["type"] == ContentType.MCP_APPROVAL_REQUEST: return None - error = self._validate_string_field(content_item, "name", "tool_call content items") + error = self._validate_string_field( + content_item, "name", "tool_call content items" + ) if error: return error - error = self._validate_dict_field(content_item, "arguments", "tool_call content items") + error = self._validate_dict_field( + content_item, "arguments", "tool_call content items" + ) if error: return error - error = self._validate_string_field(content_item, "tool_call_id", "tool_call content items") + error = self._validate_string_field( + content_item, "tool_call_id", "tool_call content items" + ) if error: return error return None - def _validate_user_or_system_message(self, message: Dict[str, Any], role: str) -> Optional[EvaluationException]: + def _validate_user_or_system_message( + self, message: Dict[str, Any], role: str + ) -> Optional[EvaluationException]: """Validate user or system message content.""" content = message["content"] @@ -203,7 +220,9 @@ def _validate_user_or_system_message(self, message: Dict[str, Any], role: str) - return error return None - def _validate_assistant_message(self, message: Dict[str, Any]) -> Optional[EvaluationException]: + def _validate_assistant_message( + self, message: Dict[str, Any] + ) -> Optional[EvaluationException]: """Validate assistant message content.""" content = message["content"] @@ -216,7 +235,9 @@ def _validate_assistant_message(self, message: Dict[str, Any]) -> Optional[Evalu ContentType.MCP_APPROVAL_REQUEST, ContentType.OPENAPI_CALL, ] - valid_assistant_content_types_as_strings = [t.value for t in valid_assistant_content_types] + valid_assistant_content_types_as_strings = [ + t.value for t in valid_assistant_content_types + ] for content_item in content: content_type = content_item["type"] if content_type not in valid_assistant_content_types: @@ -228,7 +249,9 @@ def _validate_assistant_message(self, message: Dict[str, Any]) -> Optional[Evalu ) if content_type in [ContentType.TEXT, ContentType.OUTPUT_TEXT]: - error = self._validate_text_content_item(content_item, MessageRole.ASSISTANT) + error = self._validate_text_content_item( + content_item, MessageRole.ASSISTANT + ) if error: return error elif content_type in [ @@ -242,9 +265,14 @@ def _validate_assistant_message(self, message: Dict[str, Any]) -> Optional[Evalu # Raise error in case of unsupported tools for evaluators that enabled check_for_unsupported_tools if self.check_for_unsupported_tools: - if content_type == ContentType.TOOL_CALL or content_type == ContentType.OPENAPI_CALL: + if ( + content_type == ContentType.TOOL_CALL + or content_type == ContentType.OPENAPI_CALL + ): name = ( - "openapi_call" if content_type == ContentType.OPENAPI_CALL else content_item["name"].lower() + "openapi_call" + if content_type == ContentType.OPENAPI_CALL + else content_item["name"].lower() ) if name in self.UNSUPPORTED_TOOLS: return EvaluationException( @@ -255,7 +283,9 @@ def _validate_assistant_message(self, message: Dict[str, Any]) -> Optional[Evalu ) return None - def _validate_tool_message(self, message: Dict[str, Any]) -> Optional[EvaluationException]: + def _validate_tool_message( + self, message: Dict[str, Any] + ) -> Optional[EvaluationException]: """Validate tool message content.""" content = message["content"] @@ -281,7 +311,9 @@ def _validate_tool_message(self, message: Dict[str, Any]) -> Optional[Evaluation ContentType.MCP_APPROVAL_RESPONSE, ContentType.OPENAPI_CALL_OUTPUT, ] - valid_tool_content_types_as_strings = [t.value for t in valid_tool_content_types] + valid_tool_content_types_as_strings = [ + t.value for t in valid_tool_content_types + ] for content_item in content: content_type = content_item["type"] if content_type not in valid_tool_content_types: @@ -307,7 +339,9 @@ def _validate_tool_message(self, message: Dict[str, Any]) -> Optional[Evaluation return None - def _validate_message_dict(self, message: Dict[str, Any]) -> Optional[EvaluationException]: + def _validate_message_dict( + self, message: Dict[str, Any] + ) -> Optional[EvaluationException]: """Validate a single message dictionary.""" if "role" not in message: return EvaluationException( @@ -329,7 +363,8 @@ def _validate_message_dict(self, message: Dict[str, Any]) -> Optional[Evaluation content = message["content"] content_is_string_or_list_of_dicts = isinstance(content, str) or ( - isinstance(content, list) and all(item and isinstance(item, dict) for item in content) + isinstance(content, list) + and all(item and isinstance(item, dict) for item in content) ) if not content_is_string_or_list_of_dicts: return EvaluationException( @@ -372,7 +407,9 @@ def _validate_message_dict(self, message: Dict[str, Any]) -> Optional[Evaluation return None - def _validate_input_messages_list(self, input_messages: Any, input_name: str) -> Optional[EvaluationException]: + def _validate_input_messages_list( + self, input_messages: Any, input_name: str + ) -> Optional[EvaluationException]: if input_messages is None: return EvaluationException( message=f"{input_name} is a required input and cannot be None.", @@ -423,7 +460,9 @@ def _validate_input_messages_list(self, input_messages: Any, input_name: str) -> return None - def _validate_conversation(self, conversation: Any) -> Optional[EvaluationException]: + def _validate_conversation( + self, conversation: Any + ) -> Optional[EvaluationException]: """Validate the conversation input.""" if not isinstance(conversation, dict): return EvaluationException( @@ -456,7 +495,9 @@ def validate_eval_input(self, eval_input: Dict[str, Any]) -> bool: """Validate the evaluation input dictionary.""" conversation = eval_input.get("conversation") if conversation: - conversation_validation_exception = self._validate_conversation(conversation) + conversation_validation_exception = self._validate_conversation( + conversation + ) if conversation_validation_exception: raise conversation_validation_exception return True @@ -478,22 +519,17 @@ class GroundednessConversationValidator(ConversationValidator): """ ConversationValidator override used by ``GroundednessEvaluator``. - Groundedness still rejects ``azure_ai_search``, ``azure_fabric``, and - ``sharepoint_grounding`` tool calls pending a helper that can extract - document bodies out of structured ``tool_result`` envelopes and feed - them in as the ``context`` input the groundedness judge prompt scores - against. Without that helper the judge would receive empty or wrong - context and return a silently low groundedness score, so the - enablement performed for ``ToolCallSuccessEvaluator`` and - ``ToolOutputUtilizationEvaluator`` is intentionally not extended here. - - Once the context-extractor helper lands, this subclass can be deleted - and ``GroundednessEvaluator`` can use ``ConversationValidator`` - directly. + Groundedness now accepts the same structured grounding tools enabled in + the shared ``ConversationValidator`` path, and additionally allows + ``bing_grounding``, ``bing_custom_search``, and ``openapi_call``. + + A dedicated subclass is still kept so Groundedness can preserve stricter + guardrails for the remaining unsupported tool families without changing + behavior of other evaluators. """ - UNSUPPORTED_TOOLS: List[str] = ConversationValidator.UNSUPPORTED_TOOLS + [ - "azure_ai_search", - "azure_fabric", - "sharepoint_grounding", + UNSUPPORTED_TOOLS: List[str] = [ + tool_name + for tool_name in ConversationValidator.UNSUPPORTED_TOOLS + if tool_name not in {"bing_custom_search", "bing_grounding", "openapi_call"} ] diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py index d4c3efabc463..9ba5009d0393 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py @@ -113,14 +113,16 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]): @override def __init__(self, model_config, *, threshold=3, credential=None, **kwargs): current_dir = os.path.dirname(__file__) - prompty_path = os.path.join(current_dir, self._PROMPTY_FILE_NO_QUERY) # Default to no query + prompty_path = os.path.join( + current_dir, self._PROMPTY_FILE_NO_QUERY + ) # Default to no query self._higher_is_better = True # Initialize input validator. ``GroundednessConversationValidator`` - # carries a wider ``UNSUPPORTED_TOOLS`` list than the shared base: - # ``azure_ai_search``, ``azure_fabric``, and ``sharepoint_grounding`` - # are still rejected here pending a context-extractor helper. + # keeps stricter guardrails than the shared base for the remaining + # unsupported tool families, while allowing grounding/search/openapi + # tool calls supported by this evaluator. self._validator = GroundednessConversationValidator( error_target=ErrorTarget.GROUNDEDNESS_EVALUATOR, requires_query=False, @@ -306,8 +308,12 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: contains_context = self._has_context(eval_input) - simplified_query = simplify_messages(eval_input["query"], drop_tool_calls=contains_context) - simplified_response = simplify_messages(eval_input["response"], drop_tool_calls=False) + simplified_query = simplify_messages( + eval_input["query"], drop_tool_calls=contains_context + ) + simplified_response = simplify_messages( + eval_input["response"], drop_tool_calls=False + ) # Build simplified input simplified_eval_input = { @@ -371,11 +377,17 @@ def _convert_kwargs_to_eval_input(self, **kwargs): # If response is a string, we can skip the context extraction and just return the eval input if response and isinstance(response, str): - return super()._convert_kwargs_to_eval_input(query=query, response=response, context=response) + return super()._convert_kwargs_to_eval_input( + query=query, response=response, context=response + ) context = self._get_context_from_agent_response(response, tool_definitions) - if not self._validate_context(context) and self._is_single_entry(response) and self._is_single_entry(query): + if ( + not self._validate_context(context) + and self._is_single_entry(response) + and self._is_single_entry(query) + ): msg = ( f"{type(self).__name__}: No valid context provided or could be extracted from the query or response. " "Please either provide valid context or pass the full items list for 'response' and 'query' " @@ -388,14 +400,26 @@ def _convert_kwargs_to_eval_input(self, **kwargs): target=ErrorTarget.GROUNDEDNESS_EVALUATOR, ) - filtered_response = self._filter_file_search_results(response) if self._validate_context(context) else response - return super()._convert_kwargs_to_eval_input(response=filtered_response, context=context, query=query) + filtered_response = ( + self._filter_file_search_results(response) + if self._validate_context(context) + else response + ) + return super()._convert_kwargs_to_eval_input( + response=filtered_response, context=context, query=query + ) - def _filter_file_search_results(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + def _filter_file_search_results( + self, messages: List[Dict[str, Any]] + ) -> List[Dict[str, Any]]: """Filter out file_search tool results from the messages.""" file_search_ids = self._get_file_search_tool_call_ids(messages) return [ - msg for msg in messages if not (msg.get("role") == "tool" and msg.get("tool_call_id") in file_search_ids) + msg + for msg in messages + if not ( + msg.get("role") == "tool" and msg.get("tool_call_id") in file_search_ids + ) ] def _get_context_from_agent_response(self, response, tool_definitions): @@ -405,14 +429,20 @@ def _get_context_from_agent_response(self, response, tool_definitions): try: logger.debug("Extracting context from response") tool_calls = self._parse_tools_from_response(response=response) - logger.debug("Tool calls parsed successfully: count=%d", len(tool_calls) if tool_calls else 0) + logger.debug( + "Tool calls parsed successfully: count=%d", + len(tool_calls) if tool_calls else 0, + ) if not tool_calls: return NO_CONTEXT context_lines = [] for tool_call in tool_calls: - if not isinstance(tool_call, dict) or tool_call.get("type") != "tool_call": + if ( + not isinstance(tool_call, dict) + or tool_call.get("type") != "tool_call" + ): continue tool_name = tool_call.get("name") @@ -441,4 +471,8 @@ def _get_context_from_agent_response(self, response, tool_definitions): def _get_file_search_tool_call_ids(self, query_or_response): """Return a list of tool_call_ids for file search tool calls.""" tool_calls = self._parse_tools_from_response(query_or_response) - return [tc.get("tool_call_id") for tc in tool_calls if tc.get("name") == "file_search"] + return [ + tc.get("tool_call_id") + for tc in tool_calls + if tc.get("name") == "file_search" + ] diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_common_validators.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_common_validators.py index 9dc831ddd638..661bab082e1f 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_common_validators.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_common_validators.py @@ -77,7 +77,9 @@ def test_valid_string_query_response(self): def test_valid_conversation(self): validator = ConversationValidator(error_target=TARGET) - eval_input = {"conversation": {"messages": [_user_message(), _assistant_message()]}} + eval_input = { + "conversation": {"messages": [_user_message(), _assistant_message()]} + } assert validator.validate_eval_input(eval_input) is True def test_query_not_required(self): @@ -94,34 +96,46 @@ def test_missing_query_raises(self): def test_empty_query_list_raises(self): validator = ConversationValidator(error_target=TARGET) with pytest.raises(EvaluationException) as exc_info: - validator.validate_eval_input({"query": [], "response": [_assistant_message()]}) + validator.validate_eval_input( + {"query": [], "response": [_assistant_message()]} + ) assert exc_info.value.category == ErrorCategory.MISSING_FIELD def test_empty_query_string_raises(self): validator = ConversationValidator(error_target=TARGET) with pytest.raises(EvaluationException): - validator.validate_eval_input({"query": "", "response": [_assistant_message()]}) + validator.validate_eval_input( + {"query": "", "response": [_assistant_message()]} + ) def test_query_wrong_type_raises(self): validator = ConversationValidator(error_target=TARGET) with pytest.raises(EvaluationException) as exc_info: - validator.validate_eval_input({"query": 123, "response": [_assistant_message()]}) + validator.validate_eval_input( + {"query": 123, "response": [_assistant_message()]} + ) assert exc_info.value.category == ErrorCategory.INVALID_VALUE def test_message_not_dict_raises(self): validator = ConversationValidator(error_target=TARGET) with pytest.raises(EvaluationException): - validator.validate_eval_input({"query": ["not a dict"], "response": [_assistant_message()]}) + validator.validate_eval_input( + {"query": ["not a dict"], "response": [_assistant_message()]} + ) def test_message_missing_role_raises(self): validator = ConversationValidator(error_target=TARGET) with pytest.raises(EvaluationException): - validator.validate_eval_input({"query": [{"content": "hi"}], "response": [_assistant_message()]}) + validator.validate_eval_input( + {"query": [{"content": "hi"}], "response": [_assistant_message()]} + ) def test_message_missing_content_raises(self): validator = ConversationValidator(error_target=TARGET) with pytest.raises(EvaluationException): - validator.validate_eval_input({"query": [{"role": "user"}], "response": [_assistant_message()]}) + validator.validate_eval_input( + {"query": [{"role": "user"}], "response": [_assistant_message()]} + ) def test_empty_content_raises(self): validator = ConversationValidator(error_target=TARGET) @@ -137,13 +151,17 @@ def test_content_list_item_missing_type_raises(self): validator = ConversationValidator(error_target=TARGET) bad = {"role": "user", "content": [{"text": "hi"}]} with pytest.raises(EvaluationException): - validator.validate_eval_input({"query": [bad], "response": [_assistant_message()]}) + validator.validate_eval_input( + {"query": [bad], "response": [_assistant_message()]} + ) def test_user_message_invalid_content_type_raises(self): validator = ConversationValidator(error_target=TARGET) bad = {"role": "user", "content": [{"type": "tool_call", "text": "hi"}]} with pytest.raises(EvaluationException): - validator.validate_eval_input({"query": [bad], "response": [_assistant_message()]}) + validator.validate_eval_input( + {"query": [bad], "response": [_assistant_message()]} + ) def test_assistant_message_with_tool_call(self): validator = ConversationValidator(error_target=TARGET) @@ -156,18 +174,26 @@ def test_assistant_tool_call_missing_name_raises(self): bad_item = {"type": "tool_call", "arguments": {}, "tool_call_id": "1"} assistant = {"role": "assistant", "content": [bad_item]} with pytest.raises(EvaluationException): - validator.validate_eval_input({"query": [_user_message()], "response": [assistant]}) + validator.validate_eval_input( + {"query": [_user_message()], "response": [assistant]} + ) def test_unsupported_tool_raises_when_enabled(self): - validator = ConversationValidator(error_target=TARGET, check_for_unsupported_tools=True) + validator = ConversationValidator( + error_target=TARGET, check_for_unsupported_tools=True + ) unsupported = _tool_call_content_item(name="bing_grounding") assistant = {"role": "assistant", "content": [unsupported]} with pytest.raises(EvaluationException) as exc_info: - validator.validate_eval_input({"query": [_user_message()], "response": [assistant]}) + validator.validate_eval_input( + {"query": [_user_message()], "response": [assistant]} + ) assert exc_info.value.category == ErrorCategory.NOT_APPLICABLE def test_unsupported_tool_allowed_when_disabled(self): - validator = ConversationValidator(error_target=TARGET, check_for_unsupported_tools=False) + validator = ConversationValidator( + error_target=TARGET, check_for_unsupported_tools=False + ) unsupported = _tool_call_content_item(name="bing_grounding") assistant = {"role": "assistant", "content": [unsupported]} eval_input = {"query": [_user_message()], "response": [assistant]} @@ -200,18 +226,24 @@ def test_tool_message_content_not_list_raises(self): validator = ConversationValidator(error_target=TARGET) tool_msg = {"role": "tool", "tool_call_id": "call_1", "content": "result"} with pytest.raises(EvaluationException): - validator.validate_eval_input({"query": [tool_msg], "response": [_assistant_message()]}) + validator.validate_eval_input( + {"query": [tool_msg], "response": [_assistant_message()]} + ) @pytest.mark.unittest class TestToolDefinitionsValidator: def test_optional_tool_definitions_absent_ok(self): - validator = ToolDefinitionsValidator(error_target=TARGET, optional_tool_definitions=True) + validator = ToolDefinitionsValidator( + error_target=TARGET, optional_tool_definitions=True + ) eval_input = {"query": [_user_message()], "response": [_assistant_message()]} assert validator.validate_eval_input(eval_input) is True def test_required_tool_definitions_absent_raises(self): - validator = ToolDefinitionsValidator(error_target=TARGET, optional_tool_definitions=False) + validator = ToolDefinitionsValidator( + error_target=TARGET, optional_tool_definitions=False + ) eval_input = {"query": [_user_message()], "response": [_assistant_message()]} with pytest.raises(EvaluationException) as exc_info: validator.validate_eval_input(eval_input) @@ -262,7 +294,9 @@ def test_openapi_tool_definition_valid(self): eval_input = { "query": [_user_message()], "response": [_assistant_message()], - "tool_definitions": [{"type": "openapi", "functions": [_tool_definition()]}], + "tool_definitions": [ + {"type": "openapi", "functions": [_tool_definition()]} + ], } assert validator.validate_eval_input(eval_input) is True @@ -451,13 +485,17 @@ def test_json_string_inputs_parsed(self): def test_response_none_raises(self): validator = TaskNavigationEfficiencyValidator(error_target=TARGET) with pytest.raises(EvaluationException) as exc_info: - validator.validate_eval_input({"response": None, "ground_truth": ["search"]}) + validator.validate_eval_input( + {"response": None, "ground_truth": ["search"]} + ) assert exc_info.value.category == ErrorCategory.MISSING_FIELD def test_response_not_list_raises(self): validator = TaskNavigationEfficiencyValidator(error_target=TARGET) with pytest.raises(EvaluationException): - validator.validate_eval_input({"response": {"role": "user"}, "ground_truth": ["search"]}) + validator.validate_eval_input( + {"response": {"role": "user"}, "ground_truth": ["search"]} + ) def test_action_missing_role_raises(self): validator = TaskNavigationEfficiencyValidator(error_target=TARGET) @@ -487,22 +525,30 @@ def test_tool_call_missing_name_raises(self): def test_ground_truth_empty_raises(self): validator = TaskNavigationEfficiencyValidator(error_target=TARGET) with pytest.raises(EvaluationException): - validator.validate_eval_input({"response": self._response(), "ground_truth": []}) + validator.validate_eval_input( + {"response": self._response(), "ground_truth": []} + ) def test_ground_truth_wrong_type_raises(self): validator = TaskNavigationEfficiencyValidator(error_target=TARGET) with pytest.raises(EvaluationException): - validator.validate_eval_input({"response": self._response(), "ground_truth": 123}) + validator.validate_eval_input( + {"response": self._response(), "ground_truth": 123} + ) def test_ground_truth_list_non_string_raises(self): validator = TaskNavigationEfficiencyValidator(error_target=TARGET) with pytest.raises(EvaluationException): - validator.validate_eval_input({"response": self._response(), "ground_truth": ["search", 1]}) + validator.validate_eval_input( + {"response": self._response(), "ground_truth": ["search", 1]} + ) def test_ground_truth_tuple_wrong_length_raises(self): validator = TaskNavigationEfficiencyValidator(error_target=TARGET) with pytest.raises(EvaluationException): - validator.validate_eval_input({"response": self._response(), "ground_truth": (["search"],)}) + validator.validate_eval_input( + {"response": self._response(), "ground_truth": (["search"],)} + ) def test_ground_truth_tuple_params_not_dict_raises(self): validator = TaskNavigationEfficiencyValidator(error_target=TARGET) @@ -519,14 +565,28 @@ def test_ground_truth_tuple_params_not_dict_raises(self): # 'response'/'actions' missing (None). ({"response": None, "ground_truth": ["search"]}, "response", "actions"), # 'response'/'actions' wrong type. - ({"response": "not a list", "ground_truth": ["search"]}, "response", "actions"), + ( + {"response": "not a list", "ground_truth": ["search"]}, + "response", + "actions", + ), # 'ground_truth'/'expected_actions' empty -- the reported UX case. - ({"response": [{"role": "user", "content": "x"}], "ground_truth": []}, "ground_truth", "expected_actions"), + ( + {"response": [{"role": "user", "content": "x"}], "ground_truth": []}, + "ground_truth", + "expected_actions", + ), # 'ground_truth'/'expected_actions' wrong type. - ({"response": [{"role": "user", "content": "x"}], "ground_truth": 123}, "ground_truth", "expected_actions"), + ( + {"response": [{"role": "user", "content": "x"}], "ground_truth": 123}, + "ground_truth", + "expected_actions", + ), ], ) - def test_error_messages_name_both_canonical_and_alias(self, eval_input, canonical_name, alias_name): + def test_error_messages_name_both_canonical_and_alias( + self, eval_input, canonical_name, alias_name + ): # Regression guard: user-facing validation errors must cite both the SDK input name and # its azureml-assets alias, so a caller never sees a parameter name they did not supply. validator = TaskNavigationEfficiencyValidator(error_target=TARGET) @@ -598,11 +658,15 @@ def test_enforce_tool_definitions_required(self): validator.validate_eval_input({"messages": self._messages()}) def test_no_enforce_tool_definitions_ok(self): - validator = MessagesOrQueryResponseInputValidator(error_target=TARGET, enforce_tool_definitions=False) + validator = MessagesOrQueryResponseInputValidator( + error_target=TARGET, enforce_tool_definitions=False + ) assert validator.validate_eval_input({"messages": self._messages()}) is True def test_query_response_fallback_no_enforce_tool_definitions(self): - validator = MessagesOrQueryResponseInputValidator(error_target=TARGET, enforce_tool_definitions=False) + validator = MessagesOrQueryResponseInputValidator( + error_target=TARGET, enforce_tool_definitions=False + ) eval_input = {"query": [_user_message()], "response": [_assistant_message()]} assert validator.validate_eval_input(eval_input) is True @@ -616,9 +680,9 @@ def test_query_response_fallback_no_enforce_tool_definitions(self): # ``ToolCallsValidator`` inherit that narrowed default, so TCS and TOU # accept those tool calls. # -# ``GroundednessConversationValidator`` keeps the wider list -- Groundedness -# still rejects those three tools pending a context-extractor helper that -# can derive a grounding ``context`` from structured ``tool_result`` payloads. +# ``GroundednessConversationValidator`` now narrows further and allows +# ``bing_grounding``, ``bing_custom_search``, and ``openapi_call`` while +# preserving stricter guardrails only for the remaining unsupported families. NEWLY_ENABLED_TOOLS = [ "azure_ai_search", @@ -636,6 +700,19 @@ def test_query_response_fallback_no_enforce_tool_definitions(self): "web_search", ] +GROUNDEDNESS_NEWLY_ENABLED_TOOLS = [ + "bing_grounding", + "bing_custom_search", + "openapi_call", +] + +GROUNDEDNESS_STILL_UNSUPPORTED_TOOLS = [ + "browser_automation", + "code_interpreter_call", + "computer_call", + "web_search", +] + @pytest.mark.unittest class TestUnsupportedToolsListConversationValidator: @@ -651,40 +728,67 @@ def test_still_unsupported_tools_remain_in_list(self): def test_unsupported_list_exact_match(self): # Defensive: keep the list explicit so future additions surface here. - assert set(ConversationValidator.UNSUPPORTED_TOOLS) == set(STILL_UNSUPPORTED_TOOLS) + assert set(ConversationValidator.UNSUPPORTED_TOOLS) == set( + STILL_UNSUPPORTED_TOOLS + ) def test_tool_definitions_validator_inherits_same_list(self): # TCS / TOU use ``ToolDefinitionsValidator``; it must not silently # diverge from the shared default. - assert ToolDefinitionsValidator.UNSUPPORTED_TOOLS is ConversationValidator.UNSUPPORTED_TOOLS + assert ( + ToolDefinitionsValidator.UNSUPPORTED_TOOLS + is ConversationValidator.UNSUPPORTED_TOOLS + ) @pytest.mark.unittest class TestUnsupportedToolsListGroundednessValidator: - """Groundedness keeps the wider list via its dedicated subclass.""" + """Groundedness keeps a dedicated but narrower unsupported-tools list.""" - def test_full_unsupported_list_contains_newly_enabled_tools(self): - for tool_name in NEWLY_ENABLED_TOOLS: - assert tool_name in GroundednessConversationValidator.UNSUPPORTED_TOOLS + def test_groundedness_unsupported_list_does_not_include_newly_enabled_tools(self): + for tool_name in NEWLY_ENABLED_TOOLS + GROUNDEDNESS_NEWLY_ENABLED_TOOLS: + assert tool_name not in GroundednessConversationValidator.UNSUPPORTED_TOOLS - def test_full_unsupported_list_contains_still_unsupported_tools(self): - for tool_name in STILL_UNSUPPORTED_TOOLS: + def test_groundedness_unsupported_list_contains_still_unsupported_tools(self): + for tool_name in GROUNDEDNESS_STILL_UNSUPPORTED_TOOLS: assert tool_name in GroundednessConversationValidator.UNSUPPORTED_TOOLS - def test_full_unsupported_list_exact_match(self): + def test_groundedness_unsupported_list_exact_match(self): assert set(GroundednessConversationValidator.UNSUPPORTED_TOOLS) == set( - NEWLY_ENABLED_TOOLS + STILL_UNSUPPORTED_TOOLS + GROUNDEDNESS_STILL_UNSUPPORTED_TOOLS ) - @pytest.mark.parametrize("tool_name", NEWLY_ENABLED_TOOLS) - def test_groundedness_validator_still_rejects_newly_enabled_tools(self, tool_name): - validator = GroundednessConversationValidator(error_target=TARGET, check_for_unsupported_tools=True) + @pytest.mark.parametrize( + "tool_name", NEWLY_ENABLED_TOOLS + GROUNDEDNESS_NEWLY_ENABLED_TOOLS + ) + def test_groundedness_validator_accepts_newly_enabled_tools(self, tool_name): + validator = GroundednessConversationValidator( + error_target=TARGET, check_for_unsupported_tools=True + ) + assistant = { + "role": "assistant", + "content": [_tool_call_content_item(name=tool_name)], + } + assert ( + validator.validate_eval_input( + {"query": [_user_message()], "response": [assistant]} + ) + is True + ) + + @pytest.mark.parametrize("tool_name", GROUNDEDNESS_STILL_UNSUPPORTED_TOOLS) + def test_groundedness_validator_rejects_still_unsupported_tools(self, tool_name): + validator = GroundednessConversationValidator( + error_target=TARGET, check_for_unsupported_tools=True + ) assistant = { "role": "assistant", "content": [_tool_call_content_item(name=tool_name)], } with pytest.raises(EvaluationException) as exc_info: - validator.validate_eval_input({"query": [_user_message()], "response": [assistant]}) + validator.validate_eval_input( + {"query": [_user_message()], "response": [assistant]} + ) assert exc_info.value.category == ErrorCategory.NOT_APPLICABLE @@ -694,7 +798,9 @@ class TestConversationValidatorAcceptsNewlyEnabledTools: @pytest.mark.parametrize("tool_name", NEWLY_ENABLED_TOOLS) def test_validate_eval_input_accepts_tool(self, tool_name): - validator = ConversationValidator(error_target=TARGET, check_for_unsupported_tools=True) + validator = ConversationValidator( + error_target=TARGET, check_for_unsupported_tools=True + ) assistant = { "role": "assistant", "content": [_tool_call_content_item(name=tool_name)], @@ -704,7 +810,9 @@ def test_validate_eval_input_accepts_tool(self, tool_name): @pytest.mark.parametrize("tool_name", NEWLY_ENABLED_TOOLS) def test_tool_definitions_validator_accepts_tool(self, tool_name): - validator = ToolDefinitionsValidator(error_target=TARGET, check_for_unsupported_tools=True) + validator = ToolDefinitionsValidator( + error_target=TARGET, check_for_unsupported_tools=True + ) assistant = { "role": "assistant", "content": [_tool_call_content_item(name=tool_name)], @@ -719,13 +827,17 @@ class TestConversationValidatorRejectsStillUnsupportedTools: @pytest.mark.parametrize("tool_name", STILL_UNSUPPORTED_TOOLS) def test_validate_eval_input_rejects_tool(self, tool_name): - validator = ConversationValidator(error_target=TARGET, check_for_unsupported_tools=True) + validator = ConversationValidator( + error_target=TARGET, check_for_unsupported_tools=True + ) assistant = { "role": "assistant", "content": [_tool_call_content_item(name=tool_name)], } with pytest.raises(EvaluationException) as exc_info: - validator.validate_eval_input({"query": [_user_message()], "response": [assistant]}) + validator.validate_eval_input( + {"query": [_user_message()], "response": [assistant]} + ) assert exc_info.value.category == ErrorCategory.NOT_APPLICABLE @@ -744,7 +856,9 @@ class TestUnsupportedToolCheckOptOut: NEWLY_ENABLED_TOOLS + STILL_UNSUPPORTED_TOOLS, ) def test_opt_out_skips_unsupported_tool_check(self, tool_name): - validator = ToolDefinitionsValidator(error_target=TARGET, check_for_unsupported_tools=False) + validator = ToolDefinitionsValidator( + error_target=TARGET, check_for_unsupported_tools=False + ) assistant = { "role": "assistant", "content": [_tool_call_content_item(name=tool_name)],