From cd3cb168e475e62a9f54c51be8aa22d9da54c27a Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Wed, 29 Jul 2026 09:54:45 -0700 Subject: [PATCH 1/4] Mark gen_ai.tool.description and gen_ai.tool.definitions as sensitive params --- .../_genai/_langchain/_tracer.py | 2 +- .../opentelemetry/_genai/_langchain/_utils.py | 11 ++- tests/langchain/test_tracer.py | 52 ++++++++++- tests/langchain/test_utils.py | 88 ++++++++++++++++++- 4 files changed, 143 insertions(+), 10 deletions(-) diff --git a/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py b/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py index dd81fcce..ec8e2bde 100644 --- a/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py +++ b/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py @@ -508,7 +508,7 @@ def _aggregate_into_parent(self, run: Run) -> None: # pylint: disable=too-many- break if run_type in ("llm", "chat_model"): - for key, val in invocation_parameters(run): + for key, val in invocation_parameters(run, self._enable_sensitive_data): if key == GEN_AI_REQUEST_CHOICE_COUNT_KEY and isinstance(val, int) and val > 0: previous = content.get("request_choice_count") if not isinstance(previous, int) or val > previous: diff --git a/src/microsoft/opentelemetry/_genai/_langchain/_utils.py b/src/microsoft/opentelemetry/_genai/_langchain/_utils.py index 5eb6955b..62a5f0f5 100644 --- a/src/microsoft/opentelemetry/_genai/_langchain/_utils.py +++ b/src/microsoft/opentelemetry/_genai/_langchain/_utils.py @@ -389,7 +389,9 @@ def output_messages( @stop_on_exception -def invocation_parameters(run: Run) -> Iterator[tuple[str, AttributeValue]]: # pylint: disable=too-many-statements +def invocation_parameters( # pylint: disable=too-many-statements + run: Run, enable_sensitive_data: bool = False +) -> Iterator[tuple[str, AttributeValue]]: if run.run_type.lower() not in ("llm", "chat_model"): return if not (extra := run.extra): @@ -429,7 +431,7 @@ def _first_param(*keys: str) -> Any: tool_list = source.get(source_key, []) if isinstance(tool_list, list): tool_defs.extend(tool_list) - if tool_defs: + if tool_defs and _should_capture_content_on_spans(enable_sensitive_data): yield GEN_AI_TOOL_DEFINITIONS_KEY, safe_json_dumps(tool_defs) # gen_ai.request.choice_count (OpenAI/Anthropic "n") @@ -869,7 +871,7 @@ def function_calls(outputs: Mapping[str, Any] | None, enable_sensitive_data: boo if isinstance(name, str): yield GEN_AI_TOOL_NAME_KEY, name desc = fc.get("description") - if isinstance(desc, str): + if isinstance(desc, str) and _should_capture_content_on_spans(enable_sensitive_data): yield GEN_AI_TOOL_DESCRIPTION_KEY, desc call_id = fc.get("id") if isinstance(call_id, str): @@ -902,7 +904,8 @@ def tools(run: Run, enable_sensitive_data: bool = False) -> Iterator[tuple[str, if name := serialized.get("name"): yield GEN_AI_TOOL_NAME_KEY, name if description := serialized.get("description"): - yield GEN_AI_TOOL_DESCRIPTION_KEY, description + if _should_capture_content_on_spans(enable_sensitive_data): + yield GEN_AI_TOOL_DESCRIPTION_KEY, description if run.extra and hasattr(run.extra, "get"): if tool_call_id := run.extra.get("tool_call_id"): yield GEN_AI_TOOL_CALL_ID_KEY, tool_call_id diff --git a/tests/langchain/test_tracer.py b/tests/langchain/test_tracer.py index 1f0ef8ab..42150810 100644 --- a/tests/langchain/test_tracer.py +++ b/tests/langchain/test_tracer.py @@ -1544,8 +1544,12 @@ def test_structured_output_llm_excluded_from_transcript(self, mock_ctx): class TestAggregateToolDefinitions(TestCase): """Aggregate gen_ai.tool.definitions from LLM children onto the wrapper.""" + @patch( + "microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", + return_value=True, + ) @patch("microsoft.opentelemetry._genai._langchain._tracer.context_api") - def test_captures_tool_definitions_from_invocation_params(self, mock_ctx): + def test_captures_tool_definitions_from_invocation_params(self, mock_ctx, _mock_capture): mock_ctx.get_value.return_value = None tracer, otel_tracer, _ = _make_tracer() wrapper = MagicMock() @@ -1580,15 +1584,59 @@ def test_captures_tool_definitions_from_invocation_params(self, mock_ctx): self.assertIn("tool_definitions", content) self.assertIn("get_weather", content["tool_definitions"]) + @patch( + "microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", + return_value=False, + ) + @patch("microsoft.opentelemetry._genai._langchain._tracer.context_api") + def test_omits_tool_definitions_when_content_capture_disabled(self, mock_ctx, _mock_capture): + mock_ctx.get_value.return_value = None + tracer, otel_tracer, _ = _make_tracer() + wrapper = MagicMock() + inner = MagicMock() + otel_tracer.start_span.side_effect = [wrapper, inner] + + agent_run = _make_run(run_type="chain", name="LangGraph") + tracer._start_trace(agent_run) + + tool_defs = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": {"type": "object", "properties": {"location": {"type": "string"}}}, + }, + } + ] + llm_run = _make_run( + run_type="chat_model", + name="gpt-4o", + parent_run_id=agent_run.id, + extra={"invocation_params": {"model": "gpt-4o", "tools": tool_defs}}, + outputs={"generations": []}, + inputs=None, + ) + tracer.run_map[str(llm_run.id)] = llm_run + tracer._aggregate_into_parent(llm_run) + + content = tracer._agent_content[agent_run.id] + # Tool definitions must not leak into agent content when capture is off. + self.assertNotIn("tool_definitions", content) + class TestFinalizeAgentSpanAttributes(TestCase): """End-to-end finalize: assert the wrapper invoke_agent span actually receives ``gen_ai.input.messages``, ``gen_ai.output.messages``, and ``gen_ai.tool.definitions`` attributes with the correct shape.""" + @patch( + "microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", + return_value=True, + ) @patch("microsoft.opentelemetry._genai._langchain._tracer._should_capture_content_on_spans") @patch("microsoft.opentelemetry._genai._langchain._tracer.context_api") - def test_finalize_writes_input_output_and_tool_definitions(self, mock_ctx, mock_capture): + def test_finalize_writes_input_output_and_tool_definitions(self, mock_ctx, mock_capture, _mock_utils_capture): import json mock_ctx.get_value.return_value = None diff --git a/tests/langchain/test_utils.py b/tests/langchain/test_utils.py index 65fdc157..a3382e1b 100644 --- a/tests/langchain/test_utils.py +++ b/tests/langchain/test_utils.py @@ -651,7 +651,8 @@ def test_extracts_tools(self): run_type="llm", extra={"invocation_params": {"tools": [{"name": "get_weather"}]}}, ) - result = dict(invocation_parameters(run)) + # Tool definitions are gated content; opt in via enable_sensitive_data. + result = dict(invocation_parameters(run, enable_sensitive_data=True)) self.assertEqual(len(result), 1) self.assertIn("get_weather", result[GEN_AI_TOOL_DEFINITIONS_KEY]) @@ -660,10 +661,32 @@ def test_extracts_tools_for_chat_model(self): run_type="chat_model", extra={"invocation_params": {"functions": [{"name": "get_weather"}]}}, ) - result = dict(invocation_parameters(run)) + result = dict(invocation_parameters(run, enable_sensitive_data=True)) self.assertEqual(len(result), 1) self.assertIn("get_weather", result[GEN_AI_TOOL_DEFINITIONS_KEY]) + @patch("microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", return_value=False) + def test_omits_tool_definitions_when_content_capture_disabled(self, _mock_capture): + run = _make_run( + run_type="llm", + extra={"invocation_params": {"tools": [{"name": "get_weather"}]}}, + ) + # enable_sensitive_data defaults to False and content capture is off, so + # developer-authored tool definitions must not leak onto the span. + result = dict(invocation_parameters(run)) + self.assertNotIn(GEN_AI_TOOL_DEFINITIONS_KEY, result) + + @patch("microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", return_value=True) + def test_emits_tool_definitions_when_env_content_capture_enabled(self, _mock_capture): + run = _make_run( + run_type="llm", + extra={"invocation_params": {"tools": [{"name": "get_weather"}]}}, + ) + # Even without the flag, the upstream env-var/experimental content-capture + # check opting in should surface the attribute. + result = dict(invocation_parameters(run)) + self.assertIn("get_weather", result[GEN_AI_TOOL_DEFINITIONS_KEY]) + def test_skips_non_llm(self): run = _make_run(run_type="chain", extra={"invocation_params": {"tools": []}}) self.assertEqual(list(invocation_parameters(run)), []) @@ -810,6 +833,54 @@ def test_extracts_function_call_content_when_enabled(self, _mock_capture): self.assertEqual(result[GEN_AI_TOOL_ARGS_KEY], '{"city":"NYC"}') self.assertEqual(result[GEN_AI_TOOL_CALL_RESULT_KEY], '{"temperature":"72F"}') + @patch("microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", return_value=False) + def test_omits_description_when_content_capture_disabled(self, _mock_capture): + outputs = { + "generations": [ + [ + { + "message": { + "kwargs": { + "additional_kwargs": { + "function_call": { + "name": "get_weather", + "description": "Fetches current weather", + } + } + } + } + } + ] + ] + } + result = dict(function_calls(outputs)) + self.assertEqual(result[GEN_AI_TOOL_NAME_KEY], "get_weather") + # Developer-authored tool description must not leak when capture is off. + self.assertNotIn(GEN_AI_TOOL_DESCRIPTION_KEY, result) + + @patch("microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", return_value=True) + def test_emits_description_when_content_capture_enabled(self, _mock_capture): + outputs = { + "generations": [ + [ + { + "message": { + "kwargs": { + "additional_kwargs": { + "function_call": { + "name": "get_weather", + "description": "Fetches current weather", + } + } + } + } + } + ] + ] + } + result = dict(function_calls(outputs)) + self.assertEqual(result[GEN_AI_TOOL_DESCRIPTION_KEY], "Fetches current weather") + def test_returns_empty_on_none(self): self.assertEqual(list(function_calls(None)), []) @@ -952,7 +1023,8 @@ def test_extracts_tool_info(self, _mock_capture): ) result = dict(tools(run)) self.assertEqual(result[GEN_AI_TOOL_NAME_KEY], "calculator") - self.assertEqual(result[GEN_AI_TOOL_DESCRIPTION_KEY], "Does math") + # Description is developer-authored content and must be gated. + self.assertNotIn(GEN_AI_TOOL_DESCRIPTION_KEY, result) self.assertEqual(result[GEN_AI_TOOL_TYPE_KEY], "function") self.assertNotIn(GEN_AI_TOOL_ARGS_KEY, result) self.assertNotIn(GEN_AI_TOOL_CALL_RESULT_KEY, result) @@ -969,9 +1041,19 @@ def test_extracts_tool_payloads_when_content_capture_enabled(self, _mock_capture ) result = dict(tools(run)) self.assertEqual(result[GEN_AI_TOOL_TYPE_KEY], "function") + self.assertEqual(result[GEN_AI_TOOL_DESCRIPTION_KEY], "Does math") self.assertEqual(result[GEN_AI_TOOL_ARGS_KEY], "2+2") self.assertEqual(result[GEN_AI_TOOL_CALL_RESULT_KEY], "4") + @patch("microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", return_value=True) + def test_emits_tool_description_via_enable_sensitive_data_flag(self, _mock_capture): + run = _make_run( + run_type="tool", + serialized={"name": "calculator", "description": "Does math"}, + ) + result = dict(tools(run, enable_sensitive_data=True)) + self.assertEqual(result[GEN_AI_TOOL_DESCRIPTION_KEY], "Does math") + def test_skips_non_tool(self): run = _make_run(run_type="llm", serialized={"name": "calc"}) self.assertEqual(list(tools(run)), []) From ee631b5b7faa41e151f1d49874a0c44f0f706524 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 4 Aug 2026 13:55:56 -0700 Subject: [PATCH 2/4] Ensure gate consistency --- .../_genai/_langchain/_tracer.py | 2 +- tests/langchain/test_tracer.py | 77 +++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py b/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py index ec8e2bde..f26074ba 100644 --- a/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py +++ b/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py @@ -736,7 +736,7 @@ def _update_span(span: Span, run: Run, enable_sensitive_data: bool = False) -> L ) # Extras not covered by LLMInvocation extras = [ - invocation_parameters(run), + invocation_parameters(run, enable_sensitive_data), metadata(run), ] diff --git a/tests/langchain/test_tracer.py b/tests/langchain/test_tracer.py index 42150810..74bafb50 100644 --- a/tests/langchain/test_tracer.py +++ b/tests/langchain/test_tracer.py @@ -1039,6 +1039,35 @@ def test_legacy_only_emits_tool_attributes(self): self.assertEqual(merged.get(GEN_AI_PROVIDER_NAME_KEY), "openai") + def test_child_llm_emits_tool_definitions_when_sensitive_data_enabled(self): + """The agent wrapper aggregates tool definitions, but the child LLM + span must retain the same opted-in attribute as well.""" + span = MagicMock() + run = _make_run( + run_type="chat_model", + name="gpt-4o", + extra={ + "invocation_params": { + "model": "gpt-4o", + "tools": [ + { + "type": "function", + "function": {"name": "get_weather", "parameters": {}}, + } + ], + } + }, + inputs=None, + outputs={"generations": [[{"message": {"kwargs": {"content": "Sunny."}}}]]}, + ) + + _update_span(span, run, enable_sensitive_data=True) + + self.assertIn( + "get_weather", + self._merged_attrs(span)[GEN_AI_TOOL_DEFINITIONS_KEY], + ) + def test_modern_only_no_tool_leak_and_keeps_extras(self): """Modern OpenAI/Anthropic case: only tool_calls -> no gen_ai.tool.*.""" span = MagicMock() @@ -1544,6 +1573,54 @@ def test_structured_output_llm_excluded_from_transcript(self, mock_ctx): class TestAggregateToolDefinitions(TestCase): """Aggregate gen_ai.tool.definitions from LLM children onto the wrapper.""" + @staticmethod + def _all_attributes(span): + """Merge attributes applied through either OTel setter method.""" + attributes = _captured_attrs(span) + for call in span.set_attributes.call_args_list: + if call.args and isinstance(call.args[0], dict): + attributes.update(call.args[0]) + return attributes + + @patch("microsoft.opentelemetry._genai._langchain._tracer.context_api") + def test_sensitive_data_emits_tool_definitions_on_agent_and_llm_spans(self, mock_ctx): + """A single opted-in run must place tool definitions on the LLM span + and on the agent span that aggregates its children.""" + mock_ctx.get_value.return_value = None + tracer, otel_tracer, _ = _make_tracer(enable_sensitive_data=True) + agent_span = MagicMock(name="agent") + llm_span = MagicMock(name="llm") + otel_tracer.start_span.side_effect = [agent_span, llm_span] + + agent_run = _make_run(run_type="chain", name="LangGraph") + tracer._start_trace(agent_run) + llm_run = _make_run( + run_type="chat_model", + name="gpt-4o", + parent_run_id=agent_run.id, + extra={ + "invocation_params": { + "model": "gpt-4o", + "tools": [ + { + "type": "function", + "function": {"name": "get_weather", "parameters": {}}, + } + ], + } + }, + inputs=None, + outputs={"generations": [[{"message": {"kwargs": {"content": "Sunny."}}}]]}, + ) + tracer._start_trace(llm_run) + tracer._end_trace(llm_run) + tracer._end_trace(agent_run) + + for span in (agent_span, llm_span): + attributes = self._all_attributes(span) + self.assertIn(GEN_AI_TOOL_DEFINITIONS_KEY, attributes) + self.assertIn("get_weather", attributes[GEN_AI_TOOL_DEFINITIONS_KEY]) + @patch( "microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", return_value=True, From ffa5780da0bd44a1b6872b0f7b2c470114bc7ef4 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 4 Aug 2026 13:59:47 -0700 Subject: [PATCH 3/4] Mark gen_ai.tool.description and gen_ai.tool.definitions as sensitive attributes per [GenAI Spec](https://github.com/open-telemetry/semantic-conventions-genai/pull/431) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d258cba7..d407428c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ # 1.3.7 (Unreleased) ### Features Added +- Mark `gen_ai.tool.description` and `gen_ai.tool.definitions` as sensitive attributes per [GenAI Spec](https://github.com/open-telemetry/semantic-conventions-genai/pull/431) - Respect RAPI headers in order to populate the `gen_ai.response.model` with the served model if available ([#234](https://github.com/microsoft/opentelemetry-distro-python/pull/234)) - Capture the agent's system prompt as the `gen_ai.system_instructions` span attribute in the LangChain instrumentation From c665f5661d5278ccd1d870c766b28d6ca1ea8c60 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 4 Aug 2026 14:15:40 -0700 Subject: [PATCH 4/4] Add PR link --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d407428c..bde2b6c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ # 1.3.7 (Unreleased) ### Features Added - Mark `gen_ai.tool.description` and `gen_ai.tool.definitions` as sensitive attributes per [GenAI Spec](https://github.com/open-telemetry/semantic-conventions-genai/pull/431) + ([#237](https://github.com/microsoft/opentelemetry-distro-python/pull/237)) - Respect RAPI headers in order to populate the `gen_ai.response.model` with the served model if available ([#234](https://github.com/microsoft/opentelemetry-distro-python/pull/234)) - Capture the agent's system prompt as the `gen_ai.system_instructions` span attribute in the LangChain instrumentation