diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index 608d5a1db7..f4dd5fa341 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -575,6 +575,90 @@ def _is_gemini_agent_resource(agent: str) -> bool: ) +def _get_resolved_location(api_client: Any) -> Optional[str]: + """Returns the location configured on the API client.""" + return getattr(api_client, "location", None) + + +def _normalize_interaction_resource( + interaction: str, agent: str, location: Optional[str] +) -> str: + """Normalizes an interaction id into a full resource name. + + A bare interaction id is expanded to + `projects/{project}/locations/{location}/interactions/{id}` using the + project and location parsed from the agent resource name. Fully-qualified + interaction resource names are returned unchanged. + """ + if interaction.startswith("projects/"): + return interaction + parts = agent.split("/") + project = parts[1] + agent_location = parts[3] if len(parts) > 3 else (location or "global") + return f"projects/{project}/locations/{agent_location}/interactions/{interaction}" + + +def _build_interaction_id_dataset( + loaded_data: list[dict[str, Any]], + agent: Optional[str], + location: Optional[str], +) -> Optional[types.EvaluationDataset]: + """Builds an EvaluationDataset from rows that carry an `interaction_id`. + + When the dataset contains an `interaction_id` column, each row is turned + into an EvalCase whose `interactions_data_source` references the interaction + and the Gemini agent. The backend resolves the interaction trace and agent + config; no client-side prompt/response is required. Returns None if the + data does not contain interaction ids. + """ + has_interaction_id = bool(loaded_data) and any( + _evals_constant.INTERACTION_ID in row for row in loaded_data + ) + if not has_interaction_id: + if agent: + raise ValueError( + "An `agent` was provided but the dataset does not contain an" + " `interaction_id` column. The `agent` argument is only used to" + " resolve an `interaction_id` dataset column (so the backend can" + " fetch the interaction trace and Agent config). To evaluate" + " with an agent, provide a dataset with an `interaction_id`" + " column; otherwise omit `agent`." + ) + return None + + if not agent: + raise ValueError( + "An `agent` resource name is required when the dataset contains an" + " `interaction_id` column, so the backend can resolve the Agent" + " config for each interaction." + ) + if not _is_gemini_agent_resource(agent): + raise ValueError( + "`agent` must be a Gemini Agents API resource name of the form" + " projects/{project}/locations/{location}/agents/{agent} when" + " evaluating interaction ids. Got: %s" % agent + ) + + gemini_agent_config = types.GeminiAgentConfig(gemini_agent=agent) + eval_cases = [] + for i, row in enumerate(loaded_data): + interaction = row.get(_evals_constant.INTERACTION_ID) + if not interaction: + raise ValueError("Missing `interaction_id` value for row %d." % i) + eval_cases.append( + types.EvalCase( + eval_case_id="eval_case_%s" % i, + interactions_data_source=types.InteractionsDataSource( + interaction=_normalize_interaction_resource( + str(interaction), agent, location + ), + gemini_agent_config=gemini_agent_config, + ), + ) + ) + return types.EvaluationDataset(eval_cases=eval_cases) + + def _add_evaluation_run_labels( labels: Optional[dict[str, str]] = None, agent: Optional[str] = None, @@ -1591,6 +1675,8 @@ def _resolve_dataset_inputs( dataset_schema: Optional[Literal["GEMINI", "FLATTEN", "OPENAI"]], loader: "_evals_utils.EvalDatasetLoader", agent_info: Optional[types.evals.AgentInfo] = None, + agent: Optional[str] = None, + api_client: Any = None, ) -> tuple[types.EvaluationDataset, int]: """Loads and processes single or multiple datasets for evaluation. @@ -1640,6 +1726,13 @@ def _resolve_dataset_inputs( ds_source_for_loader = _get_dataset_source(ds_item) current_loaded_data = loader.load(ds_source_for_loader) + interaction_dataset = _build_interaction_id_dataset( + current_loaded_data, agent, _get_resolved_location(api_client) + ) + if interaction_dataset is not None: + parsed_evaluation_datasets.append(interaction_dataset) + continue + if dataset_schema: current_schema = _evals_data_converters.EvalDatasetSchema(dataset_schema) else: @@ -1797,6 +1890,7 @@ def _execute_evaluation( # type: ignore[no-untyped-def] api_client: Any, dataset: Union[types.EvaluationDataset, list[types.EvaluationDataset]], metrics: list[types.Metric], + agent: Optional[str] = None, dataset_schema: Optional[Literal["GEMINI", "FLATTEN", "OPENAI"]] = None, dest: Optional[str] = None, location: Optional[str] = None, @@ -1877,6 +1971,8 @@ def _execute_evaluation( # type: ignore[no-untyped-def] dataset_schema=dataset_schema, loader=loader, agent_info=validated_agent_info, + agent=agent, + api_client=api_client, ) resolved_metrics = _resolve_metrics(metrics, api_client) diff --git a/agentplatform/_genai/_evals_constant.py b/agentplatform/_genai/_evals_constant.py index 822f8e685a..f94a5e26b4 100644 --- a/agentplatform/_genai/_evals_constant.py +++ b/agentplatform/_genai/_evals_constant.py @@ -57,6 +57,7 @@ PARTS = "parts" USER_AUTHOR = "user" AGENT_DATA = "agent_data" +INTERACTION_ID = "interaction_id" STARTING_PROMPT = "starting_prompt" CONVERSATION_PLAN = "conversation_plan" HISTORY = "history" @@ -74,5 +75,6 @@ STARTING_PROMPT, CONVERSATION_PLAN, AGENT_DATA, + INTERACTION_ID, } ) diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index 36d99198a8..08d2963759 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -2214,6 +2214,7 @@ def evaluate( list[types.EvaluationDatasetOrDict], ], metrics: Optional[list[types.MetricOrDict]] = None, + agent: Optional[str] = None, location: Optional[str] = None, config: Optional[types.EvaluateMethodConfigOrDict] = None, **kwargs: Any, @@ -2222,8 +2223,15 @@ def evaluate( Args: dataset: The dataset(s) to evaluate. Can be a pandas DataFrame, a single - `types.EvaluationDataset` or a list of `types.EvaluationDataset`. + `types.EvaluationDataset` or a list of `types.EvaluationDataset`. To + evaluate existing interactions, provide a dataset with an + `interaction_id` column; each interaction is resolved by the backend + using `agent` to populate the agent data for evaluation. metrics: The list of metrics to use for evaluation. + agent: Optional Gemini Agents API agent resource name + (`projects/{project}/locations/{location}/agents/{agent}`). Required + when the dataset contains an `interaction_id` column: the backend uses + it to resolve the Agent config for each referenced interaction. location: The location to use for the evaluation service. If not specified, the location configured in the client will be used. If specified, this will override the location set in `agentplatform.Client` only for @@ -2274,6 +2282,7 @@ def evaluate( api_client=self._api_client, dataset=dataset, metrics=metrics, + agent=agent, dataset_schema=config.dataset_schema, dest=config.dest, location=location, diff --git a/tests/unit/agentplatform/genai/replays/test_evaluate.py b/tests/unit/agentplatform/genai/replays/test_evaluate.py index 410485a072..fe25760b06 100644 --- a/tests/unit/agentplatform/genai/replays/test_evaluate.py +++ b/tests/unit/agentplatform/genai/replays/test_evaluate.py @@ -582,6 +582,33 @@ def test_evaluation_single_turn_agent_data(client): assert len(evaluation_result.eval_case_results) == 1 +def test_evaluation_with_interaction_id(client): + """Tests evaluate() an interaction_id dataset with the `agent` parameter.""" + client._api_client._http_options.api_version = "v1beta1" + eval_dataset = types.EvaluationDataset( + eval_dataset_df=pd.DataFrame( + {"interaction_id": ["ChA5YTc2MWEzZmIxNWQyY2Y2EAgaATAqBG1haW4"]} + ) + ) + + evaluation_result = client.evals.evaluate( + dataset=eval_dataset, + agent=("projects/model-evaluation-dev/locations/global/agents/test-agent-eval"), + metrics=[types.RubricMetric.MULTI_TURN_TASK_SUCCESS], + ) + + assert isinstance(evaluation_result, types.EvaluationResult) + assert evaluation_result.summary_metrics is not None + assert len(evaluation_result.summary_metrics) > 0 + for summary in evaluation_result.summary_metrics: + assert isinstance(summary, types.AggregatedMetricResult) + assert summary.metric_name is not None + assert summary.mean_score is not None + + assert evaluation_result.eval_case_results is not None + assert len(evaluation_result.eval_case_results) == 1 + + pytestmark = pytest_helper.setup( file=__file__, globals_for_file=globals(), diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index eda21f7e31..a29e853b53 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -9905,6 +9905,79 @@ def test_evaluate_instances_sends_interactions_data_source(self): assert data_source["gemini_agent_config"]["gemini_agent"] == _TEST_GEMINI_AGENT +class TestEvaluateInteractionIdDataset: + """CUJ1 via evaluate(): interaction_id column + agent -> data source.""" + + def test_build_interaction_id_dataset_from_column(self): + loaded = [ + {"interaction_id": "abc123"}, + {"interaction_id": ("projects/p/locations/global/interactions/def456")}, + ] + dataset = _evals_common._build_interaction_id_dataset( + loaded, _TEST_GEMINI_AGENT, "global" + ) + + assert dataset is not None + assert len(dataset.eval_cases) == 2 + ds0 = dataset.eval_cases[0].interactions_data_source + assert ds0.gemini_agent_config.gemini_agent == _TEST_GEMINI_AGENT + assert ds0.interaction == ( + "projects/test-project/locations/us-central1/interactions/abc123" + ) + assert ( + dataset.eval_cases[1].interactions_data_source.interaction + == "projects/p/locations/global/interactions/def456" + ) + assert dataset.eval_cases[0].agent_data is None + + def test_build_interaction_id_dataset_requires_agent(self): + with pytest.raises(ValueError, match="agent.*required"): + _evals_common._build_interaction_id_dataset( + [{"interaction_id": "abc123"}], None, "global" + ) + + def test_build_interaction_id_dataset_rejects_non_gemini_agent(self): + with pytest.raises(ValueError, match="Gemini Agents API resource name"): + _evals_common._build_interaction_id_dataset( + [{"interaction_id": "abc123"}], + "projects/p/locations/us-central1/reasoningEngines/123", + "global", + ) + + def test_build_interaction_id_dataset_none_without_column_and_no_agent(self): + assert ( + _evals_common._build_interaction_id_dataset( + [{"prompt": "hi", "response": "yo"}], None, "global" + ) + is None + ) + + def test_build_interaction_id_dataset_agent_without_column_raises(self): + with pytest.raises(ValueError, match="interaction_id"): + _evals_common._build_interaction_id_dataset( + [{"prompt": "hi", "response": "yo"}], _TEST_GEMINI_AGENT, "global" + ) + + @mock.patch.object(_evals_utils, "EvalDatasetLoader") + def test_evaluate_agent_without_interaction_id_column_raises( + self, mock_eval_dataset_loader + ): + agentplatform.init(project=_TEST_PROJECT, location=_TEST_LOCATION) + client = agentplatform.Client(project=_TEST_PROJECT, location=_TEST_LOCATION) + mock_df = pd.DataFrame([{"prompt": "p1", "response": "r1"}]) + mock_eval_dataset_loader.return_value.load.return_value = mock_df.to_dict( + orient="records" + ) + dataset = agentplatform_genai_types.EvaluationDataset(eval_dataset_df=mock_df) + + with pytest.raises(ValueError, match="interaction_id"): + client.evals.evaluate( + dataset=dataset, + metrics=[agentplatform_genai_types.Metric(name="exact_match")], + agent=_TEST_GEMINI_AGENT, + ) + + class TestCreateEvaluationRunGeminiAgent: """CUJ2: scrape a Gemini agent via create_evaluation_run."""