From 0ffd6350f0b7c06abfe71d0cc7fde8169ebafe5d Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 7 Apr 2026 11:13:01 -0400 Subject: [PATCH] fix(anthropic): convert multimodal base64 inputs to attachments Process Anthropic image and document input blocks before tracing so spans store Braintrust Attachments instead of raw base64 payloads. This keeps multimodal input logs consistent with other provider integrations and avoids bloating spans with binary data. Add a focused regression test for input normalization plus VCR-backed image and PDF tests that exercise the real Anthropic API. Closes #208 --- ...create_with_document_attachment_input.yaml | 111 ++++++++++++++ ...es_create_with_image_attachment_input.yaml | 108 +++++++++++++ .../integrations/anthropic/test_anthropic.py | 142 +++++++++++++++++- .../integrations/anthropic/tracing.py | 63 +++++++- 4 files changed, 417 insertions(+), 7 deletions(-) create mode 100644 py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_document_attachment_input.yaml create mode 100644 py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_image_attachment_input.yaml diff --git a/py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_document_attachment_input.yaml b/py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_document_attachment_input.yaml new file mode 100644 index 00000000..a4d9357a --- /dev/null +++ b/py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_document_attachment_input.yaml @@ -0,0 +1,111 @@ +interactions: +- request: + body: '{"max_tokens":100,"messages":[{"role":"user","content":[{"type":"text","text":"What + kind of file is this? Keep the answer short."},{"type":"document","source":{"type":"base64","media_type":"application/pdf","data":"JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg=="}}]}],"model":"claude-haiku-4-5-20251001"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '650' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.89.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.89.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_015cmxXq7gq4maczpwuGirpC","type":"message","role":"assistant","content":[{"type":"text","text":"This + is a PDF file (Portable Document Format). The page appears to be blank."}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1592,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":21,"service_tier":"standard","inference_geo":"not_available"}}' + headers: + CF-RAY: + - 9e89eed099110c69-YYZ + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 07 Apr 2026 15:05:52 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '4000000' + anthropic-ratelimit-input-tokens-remaining: + - '3999000' + anthropic-ratelimit-input-tokens-reset: + - '2026-04-07T15:05:52Z' + anthropic-ratelimit-output-tokens-limit: + - '800000' + anthropic-ratelimit-output-tokens-remaining: + - '800000' + anthropic-ratelimit-output-tokens-reset: + - '2026-04-07T15:05:52Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-04-07T15:05:51Z' + anthropic-ratelimit-tokens-limit: + - '4800000' + anthropic-ratelimit-tokens-remaining: + - '4799000' + anthropic-ratelimit-tokens-reset: + - '2026-04-07T15:05:52Z' + cf-cache-status: + - DYNAMIC + content-length: + - '535' + request-id: + - req_011CZpcnXpTrWv6YtAEAxi9V + server-timing: + - x-originResponse;dur=1194 + set-cookie: + - _cfuvid=OIa8v6K5xzgZU5vxp6apG5OTG.ycj_DwuURu3lEhtVo-1775574351.4530668-1.0.1.1-pTxxzXia1UzBV0kg2RwIM1XIlwK260mEXZ39yjst2Vo; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '1191' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_image_attachment_input.yaml b/py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_image_attachment_input.yaml new file mode 100644 index 00000000..7b327858 --- /dev/null +++ b/py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_image_attachment_input.yaml @@ -0,0 +1,108 @@ +interactions: +- request: + body: '{"max_tokens":100,"messages":[{"role":"user","content":[{"type":"text","text":"Respond + with one word: what color is this image?"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}}]}],"model":"claude-haiku-4-5-20251001"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '344' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.89.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.89.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_01SSWMsitHe4sz9FkCendotc","type":"message","role":"assistant","content":[{"type":"text","text":"Red"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":23,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":4,"service_tier":"standard","inference_geo":"not_available"}}' + headers: + CF-RAY: + - 9e89eec9dba5c8b2-YYZ + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 07 Apr 2026 15:05:51 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '4000000' + anthropic-ratelimit-input-tokens-remaining: + - '4000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-04-07T15:05:51Z' + anthropic-ratelimit-output-tokens-limit: + - '800000' + anthropic-ratelimit-output-tokens-remaining: + - '800000' + anthropic-ratelimit-output-tokens-reset: + - '2026-04-07T15:05:51Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-04-07T15:05:50Z' + anthropic-ratelimit-tokens-limit: + - '4800000' + anthropic-ratelimit-tokens-remaining: + - '4800000' + anthropic-ratelimit-tokens-reset: + - '2026-04-07T15:05:51Z' + cf-cache-status: + - DYNAMIC + content-length: + - '459' + request-id: + - req_011CZpcnTEdYbCJrdqVsmUTk + server-timing: + - x-originResponse;dur=807 + set-cookie: + - _cfuvid=aM3IKwdh7J1pTRjhcFYcFw44WIUE55Z6.rwHtG.B_Eo-1775574350.372165-1.0.1.1-HS8MHjk_TcIDF7OtwW8yVpwl.ZHq7ahuKjYVZcaqYzc; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '801' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 7d54a533..97ae08df 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -9,15 +9,18 @@ import anthropic import pytest -from braintrust import logger +from braintrust import Attachment, logger from braintrust.integrations.anthropic import AnthropicIntegration, wrap_anthropic from braintrust.integrations.anthropic._utils import extract_anthropic_usage -from braintrust.integrations.anthropic.tracing import _log_message_to_span +from braintrust.integrations.anthropic.tracing import _get_input_from_kwargs, _log_message_to_span from braintrust.test_helpers import init_test_logger PROJECT_NAME = "test-anthropic-app" MODEL = "claude-3-haiku-20240307" # use the cheapest model since answers dont matter +MULTIMODAL_MODEL = "claude-haiku-4-5-20251001" +PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" +PDF_BASE64 = "JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg==" @pytest.fixture(scope="module") @@ -40,6 +43,57 @@ def memory_logger(): yield bgl +def test_get_input_from_kwargs_converts_multimodal_base64_blocks_to_attachments(): + kwargs = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe these files."}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": PNG_BASE64, + }, + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": PDF_BASE64, + }, + }, + ], + } + ] + } + + processed_input = _get_input_from_kwargs(kwargs) + + content = processed_input[0]["content"] + image_block = content[1] + document_block = content[2] + + assert image_block["type"] == "image" + assert image_block["source"] == {"type": "base64", "media_type": "image/png"} + assert isinstance(image_block["image_url"]["url"], Attachment) + assert image_block["image_url"]["url"].reference["content_type"] == "image/png" + assert image_block["image_url"]["url"].reference["filename"] == "image.png" + + assert document_block["type"] == "document" + assert document_block["source"] == {"type": "base64", "media_type": "application/pdf"} + assert isinstance(document_block["image_url"]["url"], Attachment) + assert document_block["image_url"]["url"].reference["content_type"] == "application/pdf" + assert document_block["image_url"]["url"].reference["filename"] == "document.pdf" + + serialized = str(processed_input) + assert PNG_BASE64 not in serialized + assert PDF_BASE64 not in serialized + + def test_log_message_to_span_includes_stop_reason_and_stop_sequence(): span = unittest.mock.MagicMock() message = SimpleNamespace( @@ -112,6 +166,90 @@ def test_extract_anthropic_usage_includes_server_tool_use_metrics_from_objects() assert metadata == {} +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path"]) +def test_anthropic_messages_create_with_image_attachment_input(memory_logger): + assert not memory_logger.pop() + + client = wrap_anthropic(_get_client()) + response = client.messages.create( + model=MULTIMODAL_MODEL, + max_tokens=100, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Respond with one word: what color is this image?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": PNG_BASE64, + }, + }, + ], + } + ], + ) + + assert response.content[0].text + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + content = span["input"][0]["content"] + image_block = content[1] + + assert image_block["type"] == "image" + assert image_block["source"] == {"type": "base64", "media_type": "image/png"} + assert isinstance(image_block["image_url"]["url"], Attachment) + assert image_block["image_url"]["url"].reference["content_type"] == "image/png" + assert image_block["image_url"]["url"].reference["filename"] == "image.png" + assert PNG_BASE64 not in str(span["input"]) + + +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path"]) +def test_anthropic_messages_create_with_document_attachment_input(memory_logger): + assert not memory_logger.pop() + + client = wrap_anthropic(_get_client()) + response = client.messages.create( + model=MULTIMODAL_MODEL, + max_tokens=100, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What kind of file is this? Keep the answer short."}, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": PDF_BASE64, + }, + }, + ], + } + ], + ) + + assert response.content[0].text + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + content = span["input"][0]["content"] + document_block = content[1] + + assert document_block["type"] == "document" + assert document_block["source"] == {"type": "base64", "media_type": "application/pdf"} + assert isinstance(document_block["image_url"]["url"], Attachment) + assert document_block["image_url"]["url"].reference["content_type"] == "application/pdf" + assert document_block["image_url"]["url"].reference["filename"] == "document.pdf" + assert PDF_BASE64 not in str(span["input"]) + + @pytest.mark.vcr def test_anthropic_messages_create_stream_true(memory_logger): assert not memory_logger.pop() diff --git a/py/src/braintrust/integrations/anthropic/tracing.py b/py/src/braintrust/integrations/anthropic/tracing.py index 3e07b6ba..ac1e40c1 100644 --- a/py/src/braintrust/integrations/anthropic/tracing.py +++ b/py/src/braintrust/integrations/anthropic/tracing.py @@ -1,10 +1,12 @@ +import base64 import logging import time import warnings from contextlib import contextmanager +from braintrust.bt_json import bt_safe_deep_copy from braintrust.integrations.anthropic._utils import Wrapper, extract_anthropic_usage -from braintrust.logger import NOOP_SPAN, log_exc_info_to_span, start_span +from braintrust.logger import NOOP_SPAN, Attachment, log_exc_info_to_span, start_span log = logging.getLogger(__name__) @@ -414,13 +416,64 @@ def _start_batch_results_span(args, kwargs): return NOOP_SPAN +def _attachment_filename_for_media_type(media_type: str, block_type: str) -> str: + extension = media_type.split("/", 1)[1] if "/" in media_type else "bin" + extension = extension.split("+", 1)[0] + prefix = "image" if block_type == "image" else "document" + return f"{prefix}.{extension}" + + +def _convert_base64_source_to_attachment(block_type, source): + if not isinstance(source, dict): + return None + if source.get("type") != "base64": + return None + + media_type = source.get("media_type") + data = source.get("data") + if not isinstance(media_type, str) or not isinstance(data, str): + return None + + try: + binary_data = base64.b64decode(data, validate=True) + except Exception: + return None + + return Attachment( + data=binary_data, + filename=_attachment_filename_for_media_type(media_type, block_type), + content_type=media_type, + ) + + +def _process_input_attachments(value): + if isinstance(value, list): + return [_process_input_attachments(item) for item in value] + + if isinstance(value, dict): + block_type = value.get("type") + source = value.get("source") + + if block_type in {"image", "document"} and isinstance(source, dict): + attachment = _convert_base64_source_to_attachment(block_type, source) + if attachment is not None: + processed = {k: _process_input_attachments(v) for k, v in value.items() if k != "source"} + processed["source"] = {k: _process_input_attachments(v) for k, v in source.items() if k != "data"} + processed["image_url"] = {"url": attachment} + return processed + + return {k: _process_input_attachments(v) for k, v in value.items()} + + return value + + def _get_input_from_kwargs(kwargs): - msgs = list(kwargs.get("messages", [])) - kwargs["messages"] = msgs.copy() + msgs = bt_safe_deep_copy(list(kwargs.get("messages", []))) + msgs = _process_input_attachments(msgs) - system = kwargs.get("system", None) + system = bt_safe_deep_copy(kwargs.get("system", None)) if system: - msgs.append({"role": "system", "content": system}) + msgs.append({"role": "system", "content": _process_input_attachments(system)}) return msgs