diff --git a/logfire/_internal/integrations/llm_providers/openai.py b/logfire/_internal/integrations/llm_providers/openai.py index 39e708aeb..7f3c21f75 100644 --- a/logfire/_internal/integrations/llm_providers/openai.py +++ b/logfire/_internal/integrations/llm_providers/openai.py @@ -282,21 +282,45 @@ def _convert_content_part(part: object) -> MessagePart: part = cast('dict[str, Any]', part) part_type = part.get('type', 'unknown') - if part_type in ('text', 'output_text'): + if part_type in ('text', 'output_text', 'input_text'): return TextPart(type='text', content=part.get('text', '')) - elif part_type == 'image_url': # pragma: no cover - url = part.get('image_url', {}).get('url', '') - return UriPart(type='uri', uri=url, modality='image') + elif part_type in ('image_url', 'input_image'): # pragma: no cover + if uri := _media_part_uri(part, 'image_url'): + return UriPart(type='uri', uri=uri, modality='image') + if data := part.get('file_data'): + return BlobPart(type='blob', content=data, modality='image') elif part_type == 'input_audio': # pragma: no cover return BlobPart( type='blob', content=part.get('input_audio', {}).get('data', ''), modality='audio', ) + elif part_type == 'input_file': # pragma: no cover + if uri := _media_part_uri(part, 'file_url'): + return UriPart(type='uri', uri=uri, modality='document') + if data := part.get('file_data'): + return BlobPart(type='blob', content=data, modality='document') else: # pragma: no cover # Return as generic dict for unknown types return {**part, 'type': part_type} + return {**part, 'type': part_type} + + +def _media_part_uri(part: dict[str, Any], url_key: str) -> str: + value = part.get(url_key) + if isinstance(value, str): + return value + if isinstance(value, dict): + value = cast('dict[str, Any]', value) + url = value.get('url') + if isinstance(url, str): + return url + if isinstance(file_id := part.get('file_id'), str): + # Keep file references traceable when the Responses API gives an ID instead of a URL. + return f'openai://file/{file_id}' + return '' + def convert_responses_inputs_to_semconv( inputs: str | list[dict[str, Any]] | None, instructions: str | None @@ -727,11 +751,10 @@ def input_to_events(inp: dict[str, Any], tool_call_id_to_name: dict[str, str]): events.append({'event.name': event_name, 'content': content, 'role': role}) else: for content_item in content: - with contextlib.suppress(KeyError): - if content_item['type'] == 'output_text': - events.append({'event.name': event_name, 'content': content_item['text'], 'role': role}) - continue - events.append(unknown_event(content_item)) # pragma: no cover + if event := _content_part_to_event(content_item, event_name, role): + events.append(event) + else: + events.append(unknown_event(content_item)) # pragma: no cover elif typ == 'function_call': tool_call_id_to_name[inp['call_id']] = inp['name'] events.append( @@ -772,3 +795,24 @@ def unknown_event(inp: dict[str, Any]): 'content': f'{inp.get("type")}\n\nSee JSON for details', 'data': inp, } + + +def _content_part_to_event(content_item: Any, event_name: str, role: str) -> dict[str, Any] | None: + if not isinstance(content_item, dict): # pragma: no cover + return None + + content_item = cast('dict[str, Any]', content_item) + content_type = content_item.get('type') + if content_type in ('output_text', 'input_text'): + return {'event.name': event_name, 'content': content_item.get('text', ''), 'role': role} + if content_type in ('image_url', 'input_image'): + if uri := _media_part_uri(content_item, 'image_url'): + return {'event.name': event_name, 'content': uri, 'role': role} + if content_type == 'input_file': + if uri := _media_part_uri(content_item, 'file_url'): + return {'event.name': event_name, 'content': uri, 'role': role} + if isinstance(filename := content_item.get('filename'), str): + return {'event.name': event_name, 'content': filename, 'role': role} + if content_item.get('file_data') is not None: + return {'event.name': event_name, 'content': '[file_data]', 'role': role} + return None diff --git a/tests/otel_integrations/test_openai.py b/tests/otel_integrations/test_openai.py index 59515106d..667b853af 100644 --- a/tests/otel_integrations/test_openai.py +++ b/tests/otel_integrations/test_openai.py @@ -4235,6 +4235,49 @@ def test_input_to_events_unknown_type_no_role() -> None: ) +def test_input_to_events_responses_api_content_parts() -> None: + """Test Responses API content parts on the legacy events path.""" + from logfire._internal.integrations.llm_providers.openai import input_to_events + + inp: dict[str, Any] = { + 'type': 'message', + 'role': 'user', + 'content': [ + {'type': 'input_text', 'text': 'What is in these files?'}, + {'type': 'input_image', 'image_url': 'data:image/png;base64,abc123'}, + {'type': 'input_image', 'file_data': 'ignored-on-events-path'}, + {'type': 'input_file', 'file_id': 'file_123'}, + {'type': 'input_file', 'file_data': 'JVBERi0x'}, + {'type': 'input_file', 'filename': 'report.pdf', 'file_data': 'JVBERi0x'}, + {'type': 'input_file'}, + ], + } + + events = input_to_events(inp, {}) + + assert events == snapshot( + [ + {'event.name': 'gen_ai.user.message', 'content': 'What is in these files?', 'role': 'user'}, + {'event.name': 'gen_ai.user.message', 'content': 'data:image/png;base64,abc123', 'role': 'user'}, + { + 'event.name': 'gen_ai.unknown', + 'role': 'unknown', + 'content': 'input_image\n\nSee JSON for details', + 'data': {'type': 'input_image', 'file_data': 'ignored-on-events-path'}, + }, + {'event.name': 'gen_ai.user.message', 'content': 'openai://file/file_123', 'role': 'user'}, + {'event.name': 'gen_ai.user.message', 'content': '[file_data]', 'role': 'user'}, + {'event.name': 'gen_ai.user.message', 'content': 'report.pdf', 'role': 'user'}, + { + 'event.name': 'gen_ai.unknown', + 'role': 'unknown', + 'content': 'input_file\n\nSee JSON for details', + 'data': {'type': 'input_file'}, + }, + ] + ) + + def test_inputs_to_events_no_inputs() -> None: """Test inputs_to_events with no inputs.""" from logfire._internal.integrations.llm_providers.openai import inputs_to_events @@ -4368,6 +4411,52 @@ def test_convert_responses_inputs_unknown_type() -> None: ) +def test_convert_responses_inputs_multimodal_content_parts() -> None: + """Test Responses API text, image, and file input parts.""" + from logfire._internal.integrations.llm_providers.openai import convert_responses_inputs_to_semconv + + inputs: list[dict[str, Any]] = [ + { + 'role': 'user', + 'content': [ + {'type': 'input_text', 'text': 'Summarize these inputs'}, + {'type': 'image_url', 'image_url': {'url': 'https://example.com/chat-image.png'}}, + {'type': 'image_url', 'image_url': {}, 'file_id': 'file_chat_image_123'}, + {'type': 'input_image', 'image_url': 'https://example.com/image.png'}, + {'type': 'input_image', 'file_id': 'file_image_123'}, + {'type': 'input_file', 'file_url': 'https://example.com/report.pdf'}, + {'type': 'input_file', 'file_id': 'file_doc_123'}, + {'type': 'input_file', 'file_data': 'JVBERi0x', 'filename': 'inline.pdf'}, + {'type': 'input_file', 'filename': 'missing-data.pdf'}, + ], + } + ] + + input_messages, system_instructions = convert_responses_inputs_to_semconv(inputs, None) + + assert (input_messages, system_instructions) == snapshot( + ( + [ + { + 'role': 'user', + 'parts': [ + {'type': 'text', 'content': 'Summarize these inputs'}, + {'type': 'uri', 'uri': 'https://example.com/chat-image.png', 'modality': 'image'}, + {'type': 'uri', 'uri': 'openai://file/file_chat_image_123', 'modality': 'image'}, + {'type': 'uri', 'uri': 'https://example.com/image.png', 'modality': 'image'}, + {'type': 'uri', 'uri': 'openai://file/file_image_123', 'modality': 'image'}, + {'type': 'uri', 'uri': 'https://example.com/report.pdf', 'modality': 'document'}, + {'type': 'uri', 'uri': 'openai://file/file_doc_123', 'modality': 'document'}, + {'type': 'blob', 'content': 'JVBERi0x', 'modality': 'document'}, + {'type': 'input_file', 'filename': 'missing-data.pdf'}, + ], + } + ], + [], + ) + ) + + def test_convert_responses_outputs_no_content() -> None: """Test convert_responses_outputs_to_semconv with non-message output.""" from unittest.mock import MagicMock