diff --git a/py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_document_attachment_input.yaml b/py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_document_attachment_input.yaml new file mode 100644 index 00000000..a4d9357a --- /dev/null +++ b/py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_document_attachment_input.yaml @@ -0,0 +1,111 @@ +interactions: +- request: + body: '{"max_tokens":100,"messages":[{"role":"user","content":[{"type":"text","text":"What + kind of file is this? Keep the answer short."},{"type":"document","source":{"type":"base64","media_type":"application/pdf","data":"JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg=="}}]}],"model":"claude-haiku-4-5-20251001"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '650' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.89.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.89.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_015cmxXq7gq4maczpwuGirpC","type":"message","role":"assistant","content":[{"type":"text","text":"This + is a PDF file (Portable Document Format). The page appears to be blank."}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1592,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":21,"service_tier":"standard","inference_geo":"not_available"}}' + headers: + CF-RAY: + - 9e89eed099110c69-YYZ + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 07 Apr 2026 15:05:52 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '4000000' + anthropic-ratelimit-input-tokens-remaining: + - '3999000' + anthropic-ratelimit-input-tokens-reset: + - '2026-04-07T15:05:52Z' + anthropic-ratelimit-output-tokens-limit: + - '800000' + anthropic-ratelimit-output-tokens-remaining: + - '800000' + anthropic-ratelimit-output-tokens-reset: + - '2026-04-07T15:05:52Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-04-07T15:05:51Z' + anthropic-ratelimit-tokens-limit: + - '4800000' + anthropic-ratelimit-tokens-remaining: + - '4799000' + anthropic-ratelimit-tokens-reset: + - '2026-04-07T15:05:52Z' + cf-cache-status: + - DYNAMIC + content-length: + - '535' + request-id: + - req_011CZpcnXpTrWv6YtAEAxi9V + server-timing: + - x-originResponse;dur=1194 + set-cookie: + - _cfuvid=OIa8v6K5xzgZU5vxp6apG5OTG.ycj_DwuURu3lEhtVo-1775574351.4530668-1.0.1.1-pTxxzXia1UzBV0kg2RwIM1XIlwK260mEXZ39yjst2Vo; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '1191' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_image_attachment_input.yaml b/py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_image_attachment_input.yaml new file mode 100644 index 00000000..7b327858 --- /dev/null +++ b/py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_image_attachment_input.yaml @@ -0,0 +1,108 @@ +interactions: +- request: + body: '{"max_tokens":100,"messages":[{"role":"user","content":[{"type":"text","text":"Respond + with one word: what color is this image?"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}}]}],"model":"claude-haiku-4-5-20251001"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '344' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.89.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.89.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_01SSWMsitHe4sz9FkCendotc","type":"message","role":"assistant","content":[{"type":"text","text":"Red"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":23,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":4,"service_tier":"standard","inference_geo":"not_available"}}' + headers: + CF-RAY: + - 9e89eec9dba5c8b2-YYZ + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 07 Apr 2026 15:05:51 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '4000000' + anthropic-ratelimit-input-tokens-remaining: + - '4000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-04-07T15:05:51Z' + anthropic-ratelimit-output-tokens-limit: + - '800000' + anthropic-ratelimit-output-tokens-remaining: + - '800000' + anthropic-ratelimit-output-tokens-reset: + - '2026-04-07T15:05:51Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-04-07T15:05:50Z' + anthropic-ratelimit-tokens-limit: + - '4800000' + anthropic-ratelimit-tokens-remaining: + - '4800000' + anthropic-ratelimit-tokens-reset: + - '2026-04-07T15:05:51Z' + cf-cache-status: + - DYNAMIC + content-length: + - '459' + request-id: + - req_011CZpcnTEdYbCJrdqVsmUTk + server-timing: + - x-originResponse;dur=807 + set-cookie: + - _cfuvid=aM3IKwdh7J1pTRjhcFYcFw44WIUE55Z6.rwHtG.B_Eo-1775574350.372165-1.0.1.1-HS8MHjk_TcIDF7OtwW8yVpwl.ZHq7ahuKjYVZcaqYzc; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '801' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 7d54a533..97ae08df 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -9,15 +9,18 @@ import anthropic import pytest -from braintrust import logger +from braintrust import Attachment, logger from braintrust.integrations.anthropic import AnthropicIntegration, wrap_anthropic from braintrust.integrations.anthropic._utils import extract_anthropic_usage -from braintrust.integrations.anthropic.tracing import _log_message_to_span +from braintrust.integrations.anthropic.tracing import _get_input_from_kwargs, _log_message_to_span from braintrust.test_helpers import init_test_logger PROJECT_NAME = "test-anthropic-app" MODEL = "claude-3-haiku-20240307" # use the cheapest model since answers dont matter +MULTIMODAL_MODEL = "claude-haiku-4-5-20251001" +PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" +PDF_BASE64 = "JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg==" @pytest.fixture(scope="module") @@ -40,6 +43,57 @@ def memory_logger(): yield bgl +def test_get_input_from_kwargs_converts_multimodal_base64_blocks_to_attachments(): + kwargs = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe these files."}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": PNG_BASE64, + }, + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": PDF_BASE64, + }, + }, + ], + } + ] + } + + processed_input = _get_input_from_kwargs(kwargs) + + content = processed_input[0]["content"] + image_block = content[1] + document_block = content[2] + + assert image_block["type"] == "image" + assert image_block["source"] == {"type": "base64", "media_type": "image/png"} + assert isinstance(image_block["image_url"]["url"], Attachment) + assert image_block["image_url"]["url"].reference["content_type"] == "image/png" + assert image_block["image_url"]["url"].reference["filename"] == "image.png" + + assert document_block["type"] == "document" + assert document_block["source"] == {"type": "base64", "media_type": "application/pdf"} + assert isinstance(document_block["image_url"]["url"], Attachment) + assert document_block["image_url"]["url"].reference["content_type"] == "application/pdf" + assert document_block["image_url"]["url"].reference["filename"] == "document.pdf" + + serialized = str(processed_input) + assert PNG_BASE64 not in serialized + assert PDF_BASE64 not in serialized + + def test_log_message_to_span_includes_stop_reason_and_stop_sequence(): span = unittest.mock.MagicMock() message = SimpleNamespace( @@ -112,6 +166,90 @@ def test_extract_anthropic_usage_includes_server_tool_use_metrics_from_objects() assert metadata == {} +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path"]) +def test_anthropic_messages_create_with_image_attachment_input(memory_logger): + assert not memory_logger.pop() + + client = wrap_anthropic(_get_client()) + response = client.messages.create( + model=MULTIMODAL_MODEL, + max_tokens=100, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Respond with one word: what color is this image?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": PNG_BASE64, + }, + }, + ], + } + ], + ) + + assert response.content[0].text + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + content = span["input"][0]["content"] + image_block = content[1] + + assert image_block["type"] == "image" + assert image_block["source"] == {"type": "base64", "media_type": "image/png"} + assert isinstance(image_block["image_url"]["url"], Attachment) + assert image_block["image_url"]["url"].reference["content_type"] == "image/png" + assert image_block["image_url"]["url"].reference["filename"] == "image.png" + assert PNG_BASE64 not in str(span["input"]) + + +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path"]) +def test_anthropic_messages_create_with_document_attachment_input(memory_logger): + assert not memory_logger.pop() + + client = wrap_anthropic(_get_client()) + response = client.messages.create( + model=MULTIMODAL_MODEL, + max_tokens=100, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What kind of file is this? Keep the answer short."}, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": PDF_BASE64, + }, + }, + ], + } + ], + ) + + assert response.content[0].text + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + content = span["input"][0]["content"] + document_block = content[1] + + assert document_block["type"] == "document" + assert document_block["source"] == {"type": "base64", "media_type": "application/pdf"} + assert isinstance(document_block["image_url"]["url"], Attachment) + assert document_block["image_url"]["url"].reference["content_type"] == "application/pdf" + assert document_block["image_url"]["url"].reference["filename"] == "document.pdf" + assert PDF_BASE64 not in str(span["input"]) + + @pytest.mark.vcr def test_anthropic_messages_create_stream_true(memory_logger): assert not memory_logger.pop() diff --git a/py/src/braintrust/integrations/anthropic/tracing.py b/py/src/braintrust/integrations/anthropic/tracing.py index 3e07b6ba..ac1e40c1 100644 --- a/py/src/braintrust/integrations/anthropic/tracing.py +++ b/py/src/braintrust/integrations/anthropic/tracing.py @@ -1,10 +1,12 @@ +import base64 import logging import time import warnings from contextlib import contextmanager +from braintrust.bt_json import bt_safe_deep_copy from braintrust.integrations.anthropic._utils import Wrapper, extract_anthropic_usage -from braintrust.logger import NOOP_SPAN, log_exc_info_to_span, start_span +from braintrust.logger import NOOP_SPAN, Attachment, log_exc_info_to_span, start_span log = logging.getLogger(__name__) @@ -414,13 +416,64 @@ def _start_batch_results_span(args, kwargs): return NOOP_SPAN +def _attachment_filename_for_media_type(media_type: str, block_type: str) -> str: + extension = media_type.split("/", 1)[1] if "/" in media_type else "bin" + extension = extension.split("+", 1)[0] + prefix = "image" if block_type == "image" else "document" + return f"{prefix}.{extension}" + + +def _convert_base64_source_to_attachment(block_type, source): + if not isinstance(source, dict): + return None + if source.get("type") != "base64": + return None + + media_type = source.get("media_type") + data = source.get("data") + if not isinstance(media_type, str) or not isinstance(data, str): + return None + + try: + binary_data = base64.b64decode(data, validate=True) + except Exception: + return None + + return Attachment( + data=binary_data, + filename=_attachment_filename_for_media_type(media_type, block_type), + content_type=media_type, + ) + + +def _process_input_attachments(value): + if isinstance(value, list): + return [_process_input_attachments(item) for item in value] + + if isinstance(value, dict): + block_type = value.get("type") + source = value.get("source") + + if block_type in {"image", "document"} and isinstance(source, dict): + attachment = _convert_base64_source_to_attachment(block_type, source) + if attachment is not None: + processed = {k: _process_input_attachments(v) for k, v in value.items() if k != "source"} + processed["source"] = {k: _process_input_attachments(v) for k, v in source.items() if k != "data"} + processed["image_url"] = {"url": attachment} + return processed + + return {k: _process_input_attachments(v) for k, v in value.items()} + + return value + + def _get_input_from_kwargs(kwargs): - msgs = list(kwargs.get("messages", [])) - kwargs["messages"] = msgs.copy() + msgs = bt_safe_deep_copy(list(kwargs.get("messages", []))) + msgs = _process_input_attachments(msgs) - system = kwargs.get("system", None) + system = bt_safe_deep_copy(kwargs.get("system", None)) if system: - msgs.append({"role": "system", "content": system}) + msgs.append({"role": "system", "content": _process_input_attachments(system)}) return msgs