From 7b086bbbb97c1cae6cbe237893589e6526ebf0cc Mon Sep 17 00:00:00 2001 From: Erica Pisani Date: Fri, 14 Aug 2026 11:48:38 -0400 Subject: [PATCH 1/2] fix(langgraph): Gate `gen_ai.response.tool_calls` on outputs, not inputs Refs PY-2734 Refs #7200 --- sentry_sdk/integrations/langgraph.py | 2 - .../integrations/langgraph/test_langgraph.py | 59 ++++++++----------- 2 files changed, 25 insertions(+), 36 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 09152959dd..af287b0fd4 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -507,8 +507,6 @@ def _set_response_attributes( else: set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, result) - # Tool calls are an input to the model, so they're gated on inputs - if _should_record_inputs(integration): tool_calls = _extract_tool_calls(new_messages) if tool_calls: set_data_normalized( diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py index f16bd620a2..1070cb4104 100644 --- a/tests/integrations/langgraph/test_langgraph.py +++ b/tests/integrations/langgraph/test_langgraph.py @@ -1,4 +1,5 @@ import asyncio +import json import sys from unittest.mock import MagicMock, patch @@ -347,8 +348,6 @@ def original_invoke(self, *args, **kwargs): ] if isinstance(request_messages, str): - import json - request_messages = json.loads(request_messages) assert len(request_messages) == 2 assert request_messages[0]["content"] == "Hello, can you help me?" @@ -363,8 +362,6 @@ def original_invoke(self, *args, **kwargs): ] if isinstance(tool_calls_data, str): - import json - tool_calls_data = json.loads(tool_calls_data) assert len(tool_calls_data) == 1 @@ -411,8 +408,6 @@ def original_invoke(self, *args, **kwargs): request_messages = invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] if isinstance(request_messages, str): - import json - request_messages = json.loads(request_messages) assert len(request_messages) == 1 assert request_messages[0]["content"] == "Of course! How can I assist you?" @@ -424,8 +419,6 @@ def original_invoke(self, *args, **kwargs): tool_calls_data = invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] if isinstance(tool_calls_data, str): - import json - tool_calls_data = json.loads(tool_calls_data) assert len(tool_calls_data) == 1 @@ -534,8 +527,6 @@ async def run_test(): ] if isinstance(tool_calls_data, str): - import json - tool_calls_data = json.loads(tool_calls_data) assert len(tool_calls_data) == 1 @@ -583,8 +574,6 @@ async def run_test(): tool_calls_data = invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] if isinstance(tool_calls_data, str): - import json - tool_calls_data = json.loads(tool_calls_data) assert len(tool_calls_data) == 1 @@ -1917,7 +1906,6 @@ def original_invoke(self, *args, **kwargs): assert response_text == "Final response" assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in invoke_span["attributes"] - import json tool_calls_data = invoke_span["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] else: @@ -1941,7 +1929,6 @@ def original_invoke(self, *args, **kwargs): assert response_text == "Final response" assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in invoke_span["data"] - import json tool_calls_data = invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] @@ -2012,8 +1999,6 @@ def __init__(self, content, message_type="human"): # If messages were captured, verify role mapping if SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"]: - import json - stored_messages = json.loads( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] ) @@ -2037,8 +2022,6 @@ def __init__(self, content, message_type="human"): # If messages were captured, verify role mapping if SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"]: - import json - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) # Find messages with specific content to verify role mapping @@ -2065,7 +2048,6 @@ def __init__(self, content, message_type="human"): def test_langgraph_message_truncation(sentry_init, capture_events): """Test that large messages are truncated properly in Langgraph integration.""" - import json sentry_init( integrations=[LanggraphIntegration(include_prompts=True)], @@ -2196,7 +2178,7 @@ def _invoke_span_data(items_or_events, span_streaming): ), ], ) -def test_pregel_invoke_gates_request_messages_and_tool_calls_on_inputs_setting( +def test_pregel_invoke_gates_request_messages_on_inputs_setting( sentry_init, capture_events, capture_items, @@ -2219,13 +2201,6 @@ def test_pregel_invoke_gates_request_messages_and_tool_calls_on_inputs_setting( test_state = {"messages": [MockMessage("Hello, can you help me?", name="user")]} pregel = MockPregelInstance("test_graph") - expected_tool_calls = [ - { - "id": "call_test_123", - "type": "function", - "function": {"name": "search_tool", "arguments": '{"query": "help"}'}, - } - ] def original_invoke(self, *args, **kwargs): return { @@ -2234,7 +2209,6 @@ def original_invoke(self, *args, **kwargs): MockMessage( content="I'll help you with that task!", name="assistant", - tool_calls=expected_tool_calls, ) ] } @@ -2249,10 +2223,8 @@ def original_invoke(self, *args, **kwargs): if expect_inputs: assert SPANDATA.GEN_AI_REQUEST_MESSAGES in data - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in data else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in data - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -2297,7 +2269,7 @@ def original_invoke(self, *args, **kwargs): ), ], ) -def test_pregel_invoke_gates_response_text_on_outputs_setting( +def test_pregel_invoke_gates_response_text_and_tool_calls_on_outputs_setting( sentry_init, capture_events, capture_items, @@ -2321,11 +2293,24 @@ def test_pregel_invoke_gates_response_text_on_outputs_setting( test_state = {"messages": [MockMessage("Hello, can you help me?", name="user")]} pregel = MockPregelInstance("test_graph") expected_assistant_response = "I'll help you with that task!" + expected_tool_calls = [ + { + "id": "call_test_123", + "type": "function", + "function": {"name": "search_tool", "arguments": '{"query": "help"}'}, + } + ] def original_invoke(self, *args, **kwargs): return { "messages": args[0].get("messages", []) - + [MockMessage(content=expected_assistant_response, name="assistant")] + + [ + MockMessage( + content=expected_assistant_response, + name="assistant", + tool_calls=expected_tool_calls, + ) + ] } captured = capture_items("span") if span_streaming else capture_events() @@ -2338,8 +2323,12 @@ def original_invoke(self, *args, **kwargs): if expect_outputs: assert data[SPANDATA.GEN_AI_RESPONSE_TEXT] == expected_assistant_response + assert ( + json.loads(data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == expected_tool_calls + ) else: assert SPANDATA.GEN_AI_RESPONSE_TEXT not in data + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -2434,15 +2423,17 @@ async def run_test(): if expect_inputs: assert SPANDATA.GEN_AI_REQUEST_MESSAGES in data - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in data else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in data - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in data if expect_outputs: assert data[SPANDATA.GEN_AI_RESPONSE_TEXT] == expected_assistant_response + assert ( + json.loads(data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == expected_tool_calls + ) else: assert SPANDATA.GEN_AI_RESPONSE_TEXT not in data + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in data @pytest.mark.parametrize("span_streaming", [True, False]) From 6a7ebed74c16e6491dbc747bb2a20d6ae894fb6b Mon Sep 17 00:00:00 2001 From: Erica Pisani Date: Fri, 14 Aug 2026 12:08:43 -0400 Subject: [PATCH 2/2] fix(huggingface_hub): Gate `gen_ai.response.tool_calls` on outputs, not inputs Refs PY-2734 Refs #7200 --- sentry_sdk/integrations/huggingface_hub.py | 4 ++-- .../huggingface_hub/test_huggingface_hub.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/integrations/huggingface_hub.py b/sentry_sdk/integrations/huggingface_hub.py index fa6dae5214..5980c298c0 100644 --- a/sentry_sdk/integrations/huggingface_hub.py +++ b/sentry_sdk/integrations/huggingface_hub.py @@ -225,7 +225,7 @@ def new_huggingface_task(*args: "Any", **kwargs: "Any") -> "Any": if tool_calls is not None and len(tool_calls) > 0: if has_data_collection_enabled(client.options): - if client.options["data_collection"]["gen_ai"]["inputs"]: + if client.options["data_collection"]["gen_ai"]["outputs"]: set_data_normalized( span, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, @@ -402,7 +402,7 @@ def new_iterator() -> "Iterable[ChatCompletionStreamOutput]": if tool_calls is not None and len(tool_calls) > 0: if has_data_collection_enabled(client.options): if client.options["data_collection"]["gen_ai"][ - "inputs" + "outputs" ]: set_data_normalized( span, diff --git a/tests/integrations/huggingface_hub/test_huggingface_hub.py b/tests/integrations/huggingface_hub/test_huggingface_hub.py index 9e7a77f66e..8d8e6a9b2e 100644 --- a/tests/integrations/huggingface_hub/test_huggingface_hub.py +++ b/tests/integrations/huggingface_hub/test_huggingface_hub.py @@ -1921,20 +1921,22 @@ def test_text_generation_streaming_data_collection( False, [ SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, ], - [], + [ + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, + ], id="gen-ai-inputs-enabled-outputs-disabled", ), pytest.param( {"gen_ai": {"inputs": False, "outputs": True}}, False, False, - [], [ - SPANDATA.GEN_AI_REQUEST_MESSAGES, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, + ], + [ + SPANDATA.GEN_AI_REQUEST_MESSAGES, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, ], id="gen-ai-outputs-enabled-inputs-disabled", @@ -2090,11 +2092,11 @@ def test_chat_completion_data_collection_tools( False, [ SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, ], [ SPANDATA.GEN_AI_RESPONSE_TEXT, + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, ], id="gen-ai-inputs-enabled-outputs-disabled", ), @@ -2104,10 +2106,10 @@ def test_chat_completion_data_collection_tools( False, [ SPANDATA.GEN_AI_RESPONSE_TEXT, + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, ], [ SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, ], id="gen-ai-outputs-enabled-inputs-disabled",