From 000f4d0fe1aa3dc1b8a4f00d91ae60c2e75a41aa Mon Sep 17 00:00:00 2001 From: Rabah B <134166186+RabsB@users.noreply.github.com> Date: Fri, 24 Jul 2026 10:55:06 -0700 Subject: [PATCH 1/5] Add Entra evaluation writeback support Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2c234682-eb8a-47b6-add7-ba0edb243d8a --- .../ai/evaluation/_evaluate/_evaluate.py | 69 +++++-- .../ai/evaluation/_model_configurations.py | 3 + .../tests/unittests/test_evaluate.py | 177 +++++++++++++++++- 3 files changed, 237 insertions(+), 12 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index 5e266e94adee..135ea4e47c8d 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -18,6 +18,7 @@ from azure.ai.evaluation._legacy._adapters.entities import Run import pandas as pd +from azure.core.credentials import AccessToken, TokenCredential from azure.ai.evaluation._common.math import list_mean_nan_safe, apply_transform_nan_safe from azure.ai.evaluation._common.utils import validate_azure_ai_project, is_onedp_project from azure.ai.evaluation._evaluators._common._base_eval import EvaluatorBase @@ -64,6 +65,7 @@ ) LOGGER = logging.getLogger(__name__) +AZURE_MONITOR_SCOPE = "https://monitor.azure.com/.default" # For metrics (aggregates) whose metric names intentionally differ from their # originating column name, usually because the aggregation of the original value @@ -1383,6 +1385,15 @@ def emit_eval_result_events_to_app_insights( :type results: List[Dict] """ + if not results: + LOGGER.debug("No results to log to App Insights") + return + + exporter_options = _get_app_insights_exporter_options(app_insights_config) + use_entra_authentication = ( + app_insights_config.get("credential_type") == "ProjectManagedIdentity" + ) + from opentelemetry import _logs from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk._logs.export import BatchLogRecordProcessor @@ -1392,10 +1403,6 @@ def emit_eval_result_events_to_app_insights( from opentelemetry._events import get_event_logger from opentelemetry.sdk._events import EventLoggerProvider - if not results: - LOGGER.debug("No results to log to App Insights") - return - logger_provider = None try: # Configure OpenTelemetry logging with anonymized Resource attributes @@ -1414,7 +1421,7 @@ def emit_eval_result_events_to_app_insights( _logs.set_logger_provider(logger_provider) # Create Azure Monitor log exporter - azure_log_exporter = AzureMonitorLogExporter(connection_string=app_insights_config["connection_string"]) + azure_log_exporter = AzureMonitorLogExporter(**exporter_options) # Add the Azure Monitor exporter to the logger provider # Set export_timeout_millis to prevent individual batch exports from hanging @@ -1454,16 +1461,21 @@ def emit_eval_result_events_to_app_insights( # Force flush to ensure events are sent, with a timeout to prevent hanging flush_timeout_millis = 60000 # 60 seconds flush_success = logger_provider.force_flush(timeout_millis=flush_timeout_millis) - if flush_success: - LOGGER.info(f"Successfully logged {len(results)} evaluation results to App Insights") - else: - LOGGER.warning( + if not flush_success: + timeout_message = ( f"App Insights force_flush timed out after {flush_timeout_millis}ms. " "Some evaluation events may not have been sent." ) + if use_entra_authentication: + raise TimeoutError(timeout_message) + LOGGER.warning(timeout_message) + else: + LOGGER.info(f"Successfully logged {len(results)} evaluation results to App Insights") - except Exception as e: - LOGGER.error(f"Failed to emit evaluation results to App Insights: {e}") + except Exception as ex: + if use_entra_authentication: + raise + LOGGER.error("Failed to emit evaluation results to App Insights: %s", ex) finally: # Shut down the logger provider to stop background threads (e.g. OneSettings # configuration poller) that would otherwise keep the process alive indefinitely. @@ -1474,6 +1486,41 @@ def emit_eval_result_events_to_app_insights( pass +class _AzureMonitorScopedCredential: # pylint: disable=too-few-public-methods + """Delegate token acquisition with Azure Monitor's required scope.""" + + def __init__(self, credential: TokenCredential) -> None: + self._credential = credential + + def get_token(self, *_scopes: str, **kwargs: Any) -> AccessToken: + """Request a fresh Azure Monitor token from the configured credential.""" + return self._credential.get_token(AZURE_MONITOR_SCOPE, **kwargs) + + +def _get_app_insights_exporter_options( + app_insights_config: AppInsightsConfig, +) -> Dict[str, Any]: + exporter_options: Dict[str, Any] = { + "connection_string": app_insights_config["connection_string"] + } + credential_type = app_insights_config.get("credential_type") + if credential_type is None or credential_type == "ApiKey": + return exporter_options + if credential_type != "ProjectManagedIdentity": + raise ValueError( + f"Unsupported App Insights credential type: {credential_type}." + ) + + credential = app_insights_config.get("credential") + if credential is None: + raise ValueError( + "App Insights ProjectManagedIdentity authentication requires a TokenCredential." + ) + exporter_options["credential"] = _AzureMonitorScopedCredential(credential) + exporter_options["credential_scopes"] = [AZURE_MONITOR_SCOPE] + return exporter_options + + def _preprocess_data( data: Union[str, os.PathLike], evaluators_and_graders: Dict[str, Union[Callable, AzureOpenAIGrader]], diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py index d922d597fb93..32eb70130d93 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py @@ -5,6 +5,7 @@ from typing import Any, Dict, List, Literal, TypedDict, Union from typing_extensions import NotRequired +from azure.core.credentials import TokenCredential from ._evaluator_definition import EvaluatorDefinition from typing import Dict, List, Optional, Any @@ -175,6 +176,8 @@ class EvaluationResult(TypedDict): class AppInsightsConfig(TypedDict): connection_string: str + credential_type: NotRequired[Literal["ApiKey", "ProjectManagedIdentity"]] + credential: NotRequired[TokenCredential] project_id: NotRequired[str] run_type: NotRequired[str] schedule_type: NotRequired[str] diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 2ac7d282f237..485a221c7ed2 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -5,8 +5,12 @@ import os import pathlib import numpy as np -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch +from azure.core.credentials import AccessToken, TokenCredential +from azure.core.pipeline import PipelineContext, PipelineRequest +from azure.core.pipeline.policies import BearerTokenCredentialPolicy +from azure.core.rest import HttpRequest import pandas as pd import pytest from pandas.testing import assert_frame_equal @@ -53,6 +57,7 @@ _is_inverse_metric, _create_result_object, emit_eval_result_events_to_app_insights, + _get_app_insights_exporter_options, ) from azure.ai.evaluation._evaluate._utils import _convert_name_map_into_property_entries from azure.ai.evaluation._evaluate._utils import _apply_column_mapping, _trace_destination_from_project_scope @@ -3613,3 +3618,173 @@ def test_no_shutdown_when_results_empty(self, mock_lp_cls): emit_eval_result_events_to_app_insights(config, []) mock_lp_cls.assert_not_called() + + +@pytest.mark.skipif( + MISSING_OPENTELEMETRY, reason="This test requires the opentelemetry package" +) +class TestAppInsightsAuthentication: + """Tests for Application Insights exporter authentication configuration.""" + + _RESULTS = [ + { + "results": [{"metric": "coherence", "score": 4.5}], + "datasource_item": {}, + } + ] + + @patch("opentelemetry.sdk._logs.LoggerProvider") + def test_project_managed_identity_credential_and_scope_are_passed_to_exporter( + self, mock_lp_cls + ): + mock_lp_cls.return_value.force_flush.return_value = True + credential = MagicMock(spec=TokenCredential) + exporter_module = MagicMock() + config = { + "connection_string": "InstrumentationKey=fake-key", + "credential_type": "ProjectManagedIdentity", + "credential": credential, + } + + with patch.dict( + "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} + ): + emit_eval_result_events_to_app_insights(config, self._RESULTS) + + exporter_options = exporter_module.AzureMonitorLogExporter.call_args.kwargs + assert exporter_options["connection_string"] == "InstrumentationKey=fake-key" + assert exporter_options["credential_scopes"] == [ + "https://monitor.azure.com/.default" + ] + assert exporter_options["credential"] is not credential + assert "token" not in config + + def test_exporter_credential_refresh_uses_monitor_scope(self): + credential = MagicMock(spec=TokenCredential) + credential.get_token.side_effect = [ + AccessToken("token-1", 0), + AccessToken("token-2", 0), + ] + exporter_options = _get_app_insights_exporter_options( + { + "connection_string": "InstrumentationKey=fake-key", + "credential_type": "ProjectManagedIdentity", + "credential": credential, + } + ) + policy = BearerTokenCredentialPolicy( + exporter_options["credential"], + "https://monitor.azure.com//.default", + ) + request = PipelineRequest( + HttpRequest("POST", "https://example.test"), + PipelineContext(None), + ) + + policy.on_request(request) + policy.on_request(request) + + assert credential.get_token.call_args_list == [ + call("https://monitor.azure.com/.default"), + call("https://monitor.azure.com/.default"), + ] + + @patch("opentelemetry.sdk._logs.LoggerProvider") + def test_project_managed_identity_exporter_failure_is_surfaced(self, mock_lp_cls): + credential = MagicMock(spec=TokenCredential) + exporter_module = MagicMock() + exporter_module.AzureMonitorLogExporter.side_effect = RuntimeError( + "authentication failed" + ) + + with patch.dict( + "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} + ): + with pytest.raises(RuntimeError, match="authentication failed"): + emit_eval_result_events_to_app_insights( + { + "connection_string": "InstrumentationKey=fake-key", + "credential_type": "ProjectManagedIdentity", + "credential": credential, + }, + self._RESULTS, + ) + + mock_lp_cls.return_value.shutdown.assert_called_once() + + @patch("opentelemetry.sdk._logs.LoggerProvider") + def test_project_managed_identity_flush_timeout_is_surfaced(self, mock_lp_cls): + mock_lp_cls.return_value.force_flush.return_value = False + credential = MagicMock(spec=TokenCredential) + exporter_module = MagicMock() + + with patch.dict( + "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} + ): + with pytest.raises(TimeoutError, match="force_flush timed out"): + emit_eval_result_events_to_app_insights( + { + "connection_string": "InstrumentationKey=fake-key", + "credential_type": "ProjectManagedIdentity", + "credential": credential, + }, + self._RESULTS, + ) + + mock_lp_cls.return_value.shutdown.assert_called_once() + + @patch("azure.identity.DefaultAzureCredential") + def test_entra_authentication_requires_credential_without_fallback( + self, mock_default_credential + ): + environment = { + "APPLICATIONINSIGHTS_CONNECTION_STRING": "InstrumentationKey=environment-key", + "AZURE_CLIENT_ID": "environment-client-id", + } + with patch.dict(os.environ, environment, clear=False): + original_environment = os.environ.copy() + with pytest.raises( + ValueError, + match="ProjectManagedIdentity authentication requires a TokenCredential", + ): + emit_eval_result_events_to_app_insights( + { + "connection_string": "InstrumentationKey=fake-key", + "credential_type": "ProjectManagedIdentity", + }, + self._RESULTS, + ) + assert os.environ == original_environment + mock_default_credential.assert_not_called() + + def test_unknown_credential_type_does_not_fall_back_to_api_key(self): + with pytest.raises( + ValueError, match="Unsupported App Insights credential type" + ): + emit_eval_result_events_to_app_insights( + { + "connection_string": "InstrumentationKey=fake-key", + "credential_type": "FutureCredential", + }, + self._RESULTS, + ) + + @pytest.mark.parametrize("credential_type", [None, "ApiKey"]) + @patch("opentelemetry.sdk._logs.LoggerProvider") + def test_api_key_configuration_remains_compatible( + self, mock_lp_cls, credential_type + ): + mock_lp_cls.return_value.force_flush.return_value = True + exporter_module = MagicMock() + config = {"connection_string": "InstrumentationKey=fake-key"} + if credential_type is not None: + config["credential_type"] = credential_type + + with patch.dict( + "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} + ): + emit_eval_result_events_to_app_insights(config, self._RESULTS) + + exporter_module.AzureMonitorLogExporter.assert_called_once_with( + connection_string="InstrumentationKey=fake-key" + ) From 804b498b9df02566e36519fc3ee0bdca3288499e Mon Sep 17 00:00:00 2001 From: Rabah B <134166186+RabsB@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:37:23 -0700 Subject: [PATCH 2/5] Surface App Insights batch export failures Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2c234682-eb8a-47b6-add7-ba0edb243d8a --- .../ai/evaluation/_evaluate/_evaluate.py | 48 ++++++++++++++++++- .../tests/unittests/test_evaluate.py | 39 +++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index 135ea4e47c8d..437cd4f904ea 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -11,6 +11,7 @@ import tempfile import json import time +from threading import Lock from typing import Any, Callable, Dict, Iterable, Iterator, List, Literal, Optional, Set, Tuple, TypedDict, Union, cast from openai import OpenAI, AzureOpenAI @@ -1396,7 +1397,7 @@ def emit_eval_result_events_to_app_insights( from opentelemetry import _logs from opentelemetry.sdk._logs import LoggerProvider - from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, LogExportResult from opentelemetry.sdk.resources import Resource from opentelemetry.semconv.resource import ResourceAttributes from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter @@ -1422,11 +1423,18 @@ def emit_eval_result_events_to_app_insights( # Create Azure Monitor log exporter azure_log_exporter = AzureMonitorLogExporter(**exporter_options) + export_result_tracker: Optional["_ExportResultTrackingLogExporter"] = None + log_exporter: Any = azure_log_exporter + if use_entra_authentication: + export_result_tracker = _ExportResultTrackingLogExporter( + azure_log_exporter, LogExportResult.FAILURE + ) + log_exporter = export_result_tracker # Add the Azure Monitor exporter to the logger provider # Set export_timeout_millis to prevent individual batch exports from hanging logger_provider.add_log_record_processor( - BatchLogRecordProcessor(azure_log_exporter, export_timeout_millis=60000) + BatchLogRecordProcessor(log_exporter, export_timeout_millis=60000) ) # Create event logger @@ -1461,6 +1469,8 @@ def emit_eval_result_events_to_app_insights( # Force flush to ensure events are sent, with a timeout to prevent hanging flush_timeout_millis = 60000 # 60 seconds flush_success = logger_provider.force_flush(timeout_millis=flush_timeout_millis) + if export_result_tracker is not None and export_result_tracker.export_failed: + raise RuntimeError("Failed to export evaluation results to App Insights.") if not flush_success: timeout_message = ( f"App Insights force_flush timed out after {flush_timeout_millis}ms. " @@ -1497,6 +1507,40 @@ def get_token(self, *_scopes: str, **kwargs: Any) -> AccessToken: return self._credential.get_token(AZURE_MONITOR_SCOPE, **kwargs) +class _ExportResultTrackingLogExporter: # pylint: disable=too-few-public-methods + """Track failures returned by a log exporter running on a worker thread.""" + + def __init__(self, exporter: Any, failure_result: Any) -> None: + self._exporter = exporter + self._failure_result = failure_result + self._export_failed = False + self._lock = Lock() + + @property + def export_failed(self) -> bool: + """Return whether any batch export failed.""" + with self._lock: + return self._export_failed + + def export(self, batch: Any) -> Any: + """Delegate a batch export and retain its failure status.""" + try: + result = self._exporter.export(batch) + except Exception: + with self._lock: + self._export_failed = True + raise + + if result == self._failure_result: + with self._lock: + self._export_failed = True + return result + + def shutdown(self) -> None: + """Shut down the wrapped exporter.""" + self._exporter.shutdown() + + def _get_app_insights_exporter_options( app_insights_config: AppInsightsConfig, ) -> Dict[str, Any]: diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 485a221c7ed2..afbb96dfac6d 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -3712,6 +3712,45 @@ def test_project_managed_identity_exporter_failure_is_surfaced(self, mock_lp_cls mock_lp_cls.return_value.shutdown.assert_called_once() + @patch("opentelemetry.sdk._logs.LoggerProvider") + def test_project_managed_identity_batch_export_failure_is_surfaced( + self, mock_lp_cls + ): + from opentelemetry.sdk._logs.export import LogExportResult + + mock_lp_cls.return_value.force_flush.return_value = True + credential = MagicMock(spec=TokenCredential) + exporter = MagicMock() + exporter.export.return_value = LogExportResult.FAILURE + exporter_module = MagicMock() + exporter_module.AzureMonitorLogExporter.return_value = exporter + + def create_processor(tracked_exporter, **_kwargs): + tracked_exporter.export([]) + return MagicMock() + + with patch.dict( + "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} + ), patch( + "opentelemetry.sdk._logs.export.BatchLogRecordProcessor", + side_effect=create_processor, + ): + with pytest.raises( + RuntimeError, match="Failed to export evaluation results" + ): + emit_eval_result_events_to_app_insights( + { + "connection_string": "InstrumentationKey=fake-key", + "credential_type": "ProjectManagedIdentity", + "credential": credential, + }, + self._RESULTS, + ) + + exporter.export.assert_called_once_with([]) + mock_lp_cls.return_value.force_flush.assert_called_once() + mock_lp_cls.return_value.shutdown.assert_called_once() + @patch("opentelemetry.sdk._logs.LoggerProvider") def test_project_managed_identity_flush_timeout_is_surfaced(self, mock_lp_cls): mock_lp_cls.return_value.force_flush.return_value = False From 8c8dfeef1499ad46f2e222c369e0267ba826ff33 Mon Sep 17 00:00:00 2001 From: Rabah B <134166186+RabsB@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:18:07 -0700 Subject: [PATCH 3/5] Format Entra writeback changes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2c234682-eb8a-47b6-add7-ba0edb243d8a --- .../ai/evaluation/_evaluate/_evaluate.py | 24 ++------ .../tests/unittests/test_evaluate.py | 60 +++++-------------- 2 files changed, 21 insertions(+), 63 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index 437cd4f904ea..f5d0486767d8 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -1391,9 +1391,7 @@ def emit_eval_result_events_to_app_insights( return exporter_options = _get_app_insights_exporter_options(app_insights_config) - use_entra_authentication = ( - app_insights_config.get("credential_type") == "ProjectManagedIdentity" - ) + use_entra_authentication = app_insights_config.get("credential_type") == "ProjectManagedIdentity" from opentelemetry import _logs from opentelemetry.sdk._logs import LoggerProvider @@ -1426,16 +1424,12 @@ def emit_eval_result_events_to_app_insights( export_result_tracker: Optional["_ExportResultTrackingLogExporter"] = None log_exporter: Any = azure_log_exporter if use_entra_authentication: - export_result_tracker = _ExportResultTrackingLogExporter( - azure_log_exporter, LogExportResult.FAILURE - ) + export_result_tracker = _ExportResultTrackingLogExporter(azure_log_exporter, LogExportResult.FAILURE) log_exporter = export_result_tracker # Add the Azure Monitor exporter to the logger provider # Set export_timeout_millis to prevent individual batch exports from hanging - logger_provider.add_log_record_processor( - BatchLogRecordProcessor(log_exporter, export_timeout_millis=60000) - ) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter, export_timeout_millis=60000)) # Create event logger event_provider = EventLoggerProvider(logger_provider) @@ -1544,22 +1538,16 @@ def shutdown(self) -> None: def _get_app_insights_exporter_options( app_insights_config: AppInsightsConfig, ) -> Dict[str, Any]: - exporter_options: Dict[str, Any] = { - "connection_string": app_insights_config["connection_string"] - } + exporter_options: Dict[str, Any] = {"connection_string": app_insights_config["connection_string"]} credential_type = app_insights_config.get("credential_type") if credential_type is None or credential_type == "ApiKey": return exporter_options if credential_type != "ProjectManagedIdentity": - raise ValueError( - f"Unsupported App Insights credential type: {credential_type}." - ) + raise ValueError(f"Unsupported App Insights credential type: {credential_type}.") credential = app_insights_config.get("credential") if credential is None: - raise ValueError( - "App Insights ProjectManagedIdentity authentication requires a TokenCredential." - ) + raise ValueError("App Insights ProjectManagedIdentity authentication requires a TokenCredential.") exporter_options["credential"] = _AzureMonitorScopedCredential(credential) exporter_options["credential_scopes"] = [AZURE_MONITOR_SCOPE] return exporter_options diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index afbb96dfac6d..1aec628b4da4 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -3620,9 +3620,7 @@ def test_no_shutdown_when_results_empty(self, mock_lp_cls): mock_lp_cls.assert_not_called() -@pytest.mark.skipif( - MISSING_OPENTELEMETRY, reason="This test requires the opentelemetry package" -) +@pytest.mark.skipif(MISSING_OPENTELEMETRY, reason="This test requires the opentelemetry package") class TestAppInsightsAuthentication: """Tests for Application Insights exporter authentication configuration.""" @@ -3634,9 +3632,7 @@ class TestAppInsightsAuthentication: ] @patch("opentelemetry.sdk._logs.LoggerProvider") - def test_project_managed_identity_credential_and_scope_are_passed_to_exporter( - self, mock_lp_cls - ): + def test_project_managed_identity_credential_and_scope_are_passed_to_exporter(self, mock_lp_cls): mock_lp_cls.return_value.force_flush.return_value = True credential = MagicMock(spec=TokenCredential) exporter_module = MagicMock() @@ -3646,16 +3642,12 @@ def test_project_managed_identity_credential_and_scope_are_passed_to_exporter( "credential": credential, } - with patch.dict( - "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} - ): + with patch.dict("sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module}): emit_eval_result_events_to_app_insights(config, self._RESULTS) exporter_options = exporter_module.AzureMonitorLogExporter.call_args.kwargs assert exporter_options["connection_string"] == "InstrumentationKey=fake-key" - assert exporter_options["credential_scopes"] == [ - "https://monitor.azure.com/.default" - ] + assert exporter_options["credential_scopes"] == ["https://monitor.azure.com/.default"] assert exporter_options["credential"] is not credential assert "token" not in config @@ -3693,13 +3685,9 @@ def test_exporter_credential_refresh_uses_monitor_scope(self): def test_project_managed_identity_exporter_failure_is_surfaced(self, mock_lp_cls): credential = MagicMock(spec=TokenCredential) exporter_module = MagicMock() - exporter_module.AzureMonitorLogExporter.side_effect = RuntimeError( - "authentication failed" - ) + exporter_module.AzureMonitorLogExporter.side_effect = RuntimeError("authentication failed") - with patch.dict( - "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} - ): + with patch.dict("sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module}): with pytest.raises(RuntimeError, match="authentication failed"): emit_eval_result_events_to_app_insights( { @@ -3713,9 +3701,7 @@ def test_project_managed_identity_exporter_failure_is_surfaced(self, mock_lp_cls mock_lp_cls.return_value.shutdown.assert_called_once() @patch("opentelemetry.sdk._logs.LoggerProvider") - def test_project_managed_identity_batch_export_failure_is_surfaced( - self, mock_lp_cls - ): + def test_project_managed_identity_batch_export_failure_is_surfaced(self, mock_lp_cls): from opentelemetry.sdk._logs.export import LogExportResult mock_lp_cls.return_value.force_flush.return_value = True @@ -3729,15 +3715,11 @@ def create_processor(tracked_exporter, **_kwargs): tracked_exporter.export([]) return MagicMock() - with patch.dict( - "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} - ), patch( + with patch.dict("sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module}), patch( "opentelemetry.sdk._logs.export.BatchLogRecordProcessor", side_effect=create_processor, ): - with pytest.raises( - RuntimeError, match="Failed to export evaluation results" - ): + with pytest.raises(RuntimeError, match="Failed to export evaluation results"): emit_eval_result_events_to_app_insights( { "connection_string": "InstrumentationKey=fake-key", @@ -3757,9 +3739,7 @@ def test_project_managed_identity_flush_timeout_is_surfaced(self, mock_lp_cls): credential = MagicMock(spec=TokenCredential) exporter_module = MagicMock() - with patch.dict( - "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} - ): + with patch.dict("sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module}): with pytest.raises(TimeoutError, match="force_flush timed out"): emit_eval_result_events_to_app_insights( { @@ -3773,9 +3753,7 @@ def test_project_managed_identity_flush_timeout_is_surfaced(self, mock_lp_cls): mock_lp_cls.return_value.shutdown.assert_called_once() @patch("azure.identity.DefaultAzureCredential") - def test_entra_authentication_requires_credential_without_fallback( - self, mock_default_credential - ): + def test_entra_authentication_requires_credential_without_fallback(self, mock_default_credential): environment = { "APPLICATIONINSIGHTS_CONNECTION_STRING": "InstrumentationKey=environment-key", "AZURE_CLIENT_ID": "environment-client-id", @@ -3797,9 +3775,7 @@ def test_entra_authentication_requires_credential_without_fallback( mock_default_credential.assert_not_called() def test_unknown_credential_type_does_not_fall_back_to_api_key(self): - with pytest.raises( - ValueError, match="Unsupported App Insights credential type" - ): + with pytest.raises(ValueError, match="Unsupported App Insights credential type"): emit_eval_result_events_to_app_insights( { "connection_string": "InstrumentationKey=fake-key", @@ -3810,20 +3786,14 @@ def test_unknown_credential_type_does_not_fall_back_to_api_key(self): @pytest.mark.parametrize("credential_type", [None, "ApiKey"]) @patch("opentelemetry.sdk._logs.LoggerProvider") - def test_api_key_configuration_remains_compatible( - self, mock_lp_cls, credential_type - ): + def test_api_key_configuration_remains_compatible(self, mock_lp_cls, credential_type): mock_lp_cls.return_value.force_flush.return_value = True exporter_module = MagicMock() config = {"connection_string": "InstrumentationKey=fake-key"} if credential_type is not None: config["credential_type"] = credential_type - with patch.dict( - "sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module} - ): + with patch.dict("sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module}): emit_eval_result_events_to_app_insights(config, self._RESULTS) - exporter_module.AzureMonitorLogExporter.assert_called_once_with( - connection_string="InstrumentationKey=fake-key" - ) + exporter_module.AzureMonitorLogExporter.assert_called_once_with(connection_string="InstrumentationKey=fake-key") From 932c091dccef2eb4c7a2b8cb228b2261cedc0952 Mon Sep 17 00:00:00 2001 From: Rabah B <134166186+RabsB@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:50:36 -0700 Subject: [PATCH 4/5] Use evaluation error for invalid App Insights auth Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2c234682-eb8a-47b6-add7-ba0edb243d8a --- .../azure/ai/evaluation/_evaluate/_evaluate.py | 7 ++++++- .../azure-ai-evaluation/tests/unittests/test_evaluate.py | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index f5d0486767d8..2bf36eebf0f6 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -1543,7 +1543,12 @@ def _get_app_insights_exporter_options( if credential_type is None or credential_type == "ApiKey": return exporter_options if credential_type != "ProjectManagedIdentity": - raise ValueError(f"Unsupported App Insights credential type: {credential_type}.") + raise EvaluationException( + message=f"Unsupported App Insights credential type: {credential_type}.", + target=ErrorTarget.EVALUATE, + category=ErrorCategory.INVALID_VALUE, + blame=ErrorBlame.SYSTEM_ERROR, + ) credential = app_insights_config.get("credential") if credential is None: diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 1aec628b4da4..2635277a999b 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -62,7 +62,7 @@ from azure.ai.evaluation._evaluate._utils import _convert_name_map_into_property_entries from azure.ai.evaluation._evaluate._utils import _apply_column_mapping, _trace_destination_from_project_scope from azure.ai.evaluation._evaluators._eci._eci import ECIEvaluator -from azure.ai.evaluation._exceptions import EvaluationException +from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from azure.ai.evaluation._legacy._adapters._check import MISSING_LEGACY_SDK @@ -3775,7 +3775,7 @@ def test_entra_authentication_requires_credential_without_fallback(self, mock_de mock_default_credential.assert_not_called() def test_unknown_credential_type_does_not_fall_back_to_api_key(self): - with pytest.raises(ValueError, match="Unsupported App Insights credential type"): + with pytest.raises(EvaluationException, match="Unsupported App Insights credential type") as exc_info: emit_eval_result_events_to_app_insights( { "connection_string": "InstrumentationKey=fake-key", @@ -3783,6 +3783,9 @@ def test_unknown_credential_type_does_not_fall_back_to_api_key(self): }, self._RESULTS, ) + assert exc_info.value.target == ErrorTarget.EVALUATE + assert exc_info.value.category == ErrorCategory.INVALID_VALUE + assert exc_info.value.blame == ErrorBlame.SYSTEM_ERROR @pytest.mark.parametrize("credential_type", [None, "ApiKey"]) @patch("opentelemetry.sdk._logs.LoggerProvider") From f071a7f6a61cc783c81d23fdf1122744ee06c249 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:10:56 +0000 Subject: [PATCH 5/5] fix(evaluation): disable AI auth fallback for explicit ApiKey App Insights mode Co-authored-by: RabsB <134166186+RabsB@users.noreply.github.com> --- .../ai/evaluation/_evaluate/_evaluate.py | 5 +++- .../tests/unittests/test_evaluate.py | 26 ++++++++++++++++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index 2bf36eebf0f6..28be6d5a718d 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -1540,7 +1540,10 @@ def _get_app_insights_exporter_options( ) -> Dict[str, Any]: exporter_options: Dict[str, Any] = {"connection_string": app_insights_config["connection_string"]} credential_type = app_insights_config.get("credential_type") - if credential_type is None or credential_type == "ApiKey": + if credential_type is None: + return exporter_options + if credential_type == "ApiKey": + exporter_options["credential"] = None return exporter_options if credential_type != "ProjectManagedIdentity": raise EvaluationException( diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 2635277a999b..edb93f9176f4 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -3787,16 +3787,34 @@ def test_unknown_credential_type_does_not_fall_back_to_api_key(self): assert exc_info.value.category == ErrorCategory.INVALID_VALUE assert exc_info.value.blame == ErrorBlame.SYSTEM_ERROR - @pytest.mark.parametrize("credential_type", [None, "ApiKey"]) @patch("opentelemetry.sdk._logs.LoggerProvider") - def test_api_key_configuration_remains_compatible(self, mock_lp_cls, credential_type): + def test_missing_credential_type_configuration_remains_compatible(self, mock_lp_cls): mock_lp_cls.return_value.force_flush.return_value = True exporter_module = MagicMock() config = {"connection_string": "InstrumentationKey=fake-key"} - if credential_type is not None: - config["credential_type"] = credential_type with patch.dict("sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module}): emit_eval_result_events_to_app_insights(config, self._RESULTS) exporter_module.AzureMonitorLogExporter.assert_called_once_with(connection_string="InstrumentationKey=fake-key") + + @patch("opentelemetry.sdk._logs.LoggerProvider") + def test_api_key_configuration_disables_environment_fallback(self, mock_lp_cls): + mock_lp_cls.return_value.force_flush.return_value = True + exporter_module = MagicMock() + config = { + "connection_string": "InstrumentationKey=fake-key", + "credential_type": "ApiKey", + } + + with patch.dict( + os.environ, + {"APPLICATIONINSIGHTS_AUTHENTICATION_STRING": "Authorization=AAD"}, + clear=False, + ), patch.dict("sys.modules", {"azure.monitor.opentelemetry.exporter": exporter_module}): + emit_eval_result_events_to_app_insights(config, self._RESULTS) + + exporter_module.AzureMonitorLogExporter.assert_called_once_with( + connection_string="InstrumentationKey=fake-key", + credential=None, + )