fix(anthropic): convert multimodal base64 inputs to attachments

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_document_attachment_input.yaml

@@ -0,0 +1,111 @@
+interactions:
+- request:
+    body: '{"max_tokens":100,"messages":[{"role":"user","content":[{"type":"text","text":"What
+      kind of file is this? Keep the answer short."},{"type":"document","source":{"type":"base64","media_type":"application/pdf","data":"JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg=="}}]}],"model":"claude-haiku-4-5-20251001"}'
+    headers:
+      Accept:
+      - application/json
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '650'
+      Content-Type:
+      - application/json
+      Host:
+      - api.anthropic.com
+      User-Agent:
+      - Anthropic/Python 0.89.0
+      X-Stainless-Arch:
+      - arm64
+      X-Stainless-Async:
+      - 'false'
+      X-Stainless-Lang:
+      - python
+      X-Stainless-OS:
+      - MacOS
+      X-Stainless-Package-Version:
+      - 0.89.0
+      X-Stainless-Runtime:
+      - CPython
+      X-Stainless-Runtime-Version:
+      - 3.13.3
+      anthropic-version:
+      - '2023-06-01'
+      x-stainless-read-timeout:
+      - '600'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-timeout:
+      - '600'
+    method: POST
+    uri: https://api.anthropic.com/v1/messages
+  response:
+    body:
+      string: '{"model":"claude-haiku-4-5-20251001","id":"msg_015cmxXq7gq4maczpwuGirpC","type":"message","role":"assistant","content":[{"type":"text","text":"This
+        is a PDF file (Portable Document Format). The page appears to be blank."}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1592,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":21,"service_tier":"standard","inference_geo":"not_available"}}'
+    headers:
+      CF-RAY:
+      - 9e89eed099110c69-YYZ
+      Connection:
+      - keep-alive
+      Content-Security-Policy:
+      - default-src 'none'; frame-ancestors 'none'
+      Content-Type:
+      - application/json
+      Date:
+      - Tue, 07 Apr 2026 15:05:52 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      X-Robots-Tag:
+      - none
+      anthropic-organization-id:
+      - 27796668-7351-40ac-acc4-024aee8995a5
+      anthropic-ratelimit-input-tokens-limit:
+      - '4000000'
+      anthropic-ratelimit-input-tokens-remaining:
+      - '3999000'
+      anthropic-ratelimit-input-tokens-reset:
+      - '2026-04-07T15:05:52Z'
+      anthropic-ratelimit-output-tokens-limit:
+      - '800000'
+      anthropic-ratelimit-output-tokens-remaining:
+      - '800000'
+      anthropic-ratelimit-output-tokens-reset:
+      - '2026-04-07T15:05:52Z'
+      anthropic-ratelimit-requests-limit:
+      - '20000'
+      anthropic-ratelimit-requests-remaining:
+      - '19999'
+      anthropic-ratelimit-requests-reset:
+      - '2026-04-07T15:05:51Z'
+      anthropic-ratelimit-tokens-limit:
+      - '4800000'
+      anthropic-ratelimit-tokens-remaining:
+      - '4799000'
+      anthropic-ratelimit-tokens-reset:
+      - '2026-04-07T15:05:52Z'
+      cf-cache-status:
+      - DYNAMIC
+      content-length:
+      - '535'
+      request-id:
+      - req_011CZpcnXpTrWv6YtAEAxi9V
+      server-timing:
+      - x-originResponse;dur=1194
+      set-cookie:
+      - _cfuvid=OIa8v6K5xzgZU5vxp6apG5OTG.ycj_DwuURu3lEhtVo-1775574351.4530668-1.0.1.1-pTxxzXia1UzBV0kg2RwIM1XIlwK260mEXZ39yjst2Vo;
+        HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      vary:
+      - Accept-Encoding
+      x-envoy-upstream-service-time:
+      - '1191'
+    status:
+      code: 200
+      message: OK
+version: 1

py/src/braintrust/integrations/anthropic/cassettes/test_anthropic_messages_create_with_image_attachment_input.yaml

@@ -0,0 +1,108 @@
+interactions:
+- request:
+    body: '{"max_tokens":100,"messages":[{"role":"user","content":[{"type":"text","text":"Respond
+      with one word: what color is this image?"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}}]}],"model":"claude-haiku-4-5-20251001"}'
+    headers:
+      Accept:
+      - application/json
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '344'
+      Content-Type:
+      - application/json
+      Host:
+      - api.anthropic.com
+      User-Agent:
+      - Anthropic/Python 0.89.0
+      X-Stainless-Arch:
+      - arm64
+      X-Stainless-Async:
+      - 'false'
+      X-Stainless-Lang:
+      - python
+      X-Stainless-OS:
+      - MacOS
+      X-Stainless-Package-Version:
+      - 0.89.0
+      X-Stainless-Runtime:
+      - CPython
+      X-Stainless-Runtime-Version:
+      - 3.13.3
+      anthropic-version:
+      - '2023-06-01'
+      x-stainless-read-timeout:
+      - '600'
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-timeout:
+      - '600'
+    method: POST
+    uri: https://api.anthropic.com/v1/messages
+  response:
+    body:
+      string: '{"model":"claude-haiku-4-5-20251001","id":"msg_01SSWMsitHe4sz9FkCendotc","type":"message","role":"assistant","content":[{"type":"text","text":"Red"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":23,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":4,"service_tier":"standard","inference_geo":"not_available"}}'
+    headers:
+      CF-RAY:
+      - 9e89eec9dba5c8b2-YYZ
+      Connection:
+      - keep-alive
+      Content-Security-Policy:
+      - default-src 'none'; frame-ancestors 'none'
+      Content-Type:
+      - application/json
+      Date:
+      - Tue, 07 Apr 2026 15:05:51 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      X-Robots-Tag:
+      - none
+      anthropic-organization-id:
+      - 27796668-7351-40ac-acc4-024aee8995a5
+      anthropic-ratelimit-input-tokens-limit:
+      - '4000000'
+      anthropic-ratelimit-input-tokens-remaining:
+      - '4000000'
+      anthropic-ratelimit-input-tokens-reset:
+      - '2026-04-07T15:05:51Z'
+      anthropic-ratelimit-output-tokens-limit:
+      - '800000'
+      anthropic-ratelimit-output-tokens-remaining:
+      - '800000'
+      anthropic-ratelimit-output-tokens-reset:
+      - '2026-04-07T15:05:51Z'
+      anthropic-ratelimit-requests-limit:
+      - '20000'
+      anthropic-ratelimit-requests-remaining:
+      - '19999'
+      anthropic-ratelimit-requests-reset:
+      - '2026-04-07T15:05:50Z'
+      anthropic-ratelimit-tokens-limit:
+      - '4800000'
+      anthropic-ratelimit-tokens-remaining:
+      - '4800000'
+      anthropic-ratelimit-tokens-reset:
+      - '2026-04-07T15:05:51Z'
+      cf-cache-status:
+      - DYNAMIC
+      content-length:
+      - '459'
+      request-id:
+      - req_011CZpcnTEdYbCJrdqVsmUTk
+      server-timing:
+      - x-originResponse;dur=807
+      set-cookie:
+      - _cfuvid=aM3IKwdh7J1pTRjhcFYcFw44WIUE55Z6.rwHtG.B_Eo-1775574350.372165-1.0.1.1-HS8MHjk_TcIDF7OtwW8yVpwl.ZHq7ahuKjYVZcaqYzc;
+        HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-envoy-upstream-service-time:
+      - '801'
+    status:
+      code: 200
+      message: OK
+version: 1

py/src/braintrust/integrations/anthropic/test_anthropic.py

@@ -9,15 +9,18 @@
 
 import anthropic
 import pytest
-from braintrust import logger
+from braintrust import Attachment, logger
 from braintrust.integrations.anthropic import AnthropicIntegration, wrap_anthropic
 from braintrust.integrations.anthropic._utils import extract_anthropic_usage
-from braintrust.integrations.anthropic.tracing import _log_message_to_span
+from braintrust.integrations.anthropic.tracing import _get_input_from_kwargs, _log_message_to_span
 from braintrust.test_helpers import init_test_logger
 
 
 PROJECT_NAME = "test-anthropic-app"
 MODEL = "claude-3-haiku-20240307"  # use the cheapest model since answers dont matter
+MULTIMODAL_MODEL = "claude-haiku-4-5-20251001"
+PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="
+PDF_BASE64 = "JVBERi0xLjAKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PmVuZG9iagoyIDAgb2JqCjw8L1R5cGUvUGFnZXMvS2lkc1szIDAgUl0vQ291bnQgMT4+ZW5kb2JqCjMgMCBvYmoKPDwvVHlwZS9QYWdlL01lZGlhQm94WzAgMCA2MTIgNzkyXT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZg0KMDAwMDAwMDAxMCAwMDAwMCBuDQowMDAwMDAwMDUzIDAwMDAwIG4NCjAwMDAwMDAxMDIgMDAwMDAgbg0KdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoxNDkKJUVPRg=="
 
 
 @pytest.fixture(scope="module")
@@ -40,6 +43,57 @@ def memory_logger():
         yield bgl
 
 
+def test_get_input_from_kwargs_converts_multimodal_base64_blocks_to_attachments():
+    kwargs = {
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "Describe these files."},
+                    {
+                        "type": "image",
+                        "source": {
+                            "type": "base64",
+                            "media_type": "image/png",
+                            "data": PNG_BASE64,
+                        },
+                    },
+                    {
+                        "type": "document",
+                        "source": {
+                            "type": "base64",
+                            "media_type": "application/pdf",
+                            "data": PDF_BASE64,
+                        },
+                    },
+                ],
+            }
+        ]
+    }
+
+    processed_input = _get_input_from_kwargs(kwargs)
+
+    content = processed_input[0]["content"]
+    image_block = content[1]
+    document_block = content[2]
+
+    assert image_block["type"] == "image"
+    assert image_block["source"] == {"type": "base64", "media_type": "image/png"}
+    assert isinstance(image_block["image_url"]["url"], Attachment)
+    assert image_block["image_url"]["url"].reference["content_type"] == "image/png"
+    assert image_block["image_url"]["url"].reference["filename"] == "image.png"
+
+    assert document_block["type"] == "document"
+    assert document_block["source"] == {"type": "base64", "media_type": "application/pdf"}
+    assert isinstance(document_block["image_url"]["url"], Attachment)
+    assert document_block["image_url"]["url"].reference["content_type"] == "application/pdf"
+    assert document_block["image_url"]["url"].reference["filename"] == "document.pdf"
+
+    serialized = str(processed_input)
+    assert PNG_BASE64 not in serialized
+    assert PDF_BASE64 not in serialized
+
+
 def test_log_message_to_span_includes_stop_reason_and_stop_sequence():
     span = unittest.mock.MagicMock()
     message = SimpleNamespace(
@@ -112,6 +166,90 @@ def test_extract_anthropic_usage_includes_server_tool_use_metrics_from_objects()
     assert metadata == {}
 
 
+@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path"])
+def test_anthropic_messages_create_with_image_attachment_input(memory_logger):
+    assert not memory_logger.pop()
+
+    client = wrap_anthropic(_get_client())
+    response = client.messages.create(
+        model=MULTIMODAL_MODEL,
+        max_tokens=100,
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "Respond with one word: what color is this image?"},
+                    {
+                        "type": "image",
+                        "source": {
+                            "type": "base64",
+                            "media_type": "image/png",
+                            "data": PNG_BASE64,
+                        },
+                    },
+                ],
+            }
+        ],
+    )
+
+    assert response.content[0].text
+
+    spans = memory_logger.pop()
+    assert len(spans) == 1
+    span = spans[0]
+    content = span["input"][0]["content"]
+    image_block = content[1]
+
+    assert image_block["type"] == "image"
+    assert image_block["source"] == {"type": "base64", "media_type": "image/png"}
+    assert isinstance(image_block["image_url"]["url"], Attachment)
+    assert image_block["image_url"]["url"].reference["content_type"] == "image/png"
+    assert image_block["image_url"]["url"].reference["filename"] == "image.png"
+    assert PNG_BASE64 not in str(span["input"])
+
+
+@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path"])
+def test_anthropic_messages_create_with_document_attachment_input(memory_logger):
+    assert not memory_logger.pop()
+
+    client = wrap_anthropic(_get_client())
+    response = client.messages.create(
+        model=MULTIMODAL_MODEL,
+        max_tokens=100,
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "What kind of file is this? Keep the answer short."},
+                    {
+                        "type": "document",
+                        "source": {
+                            "type": "base64",
+                            "media_type": "application/pdf",
+                            "data": PDF_BASE64,
+                        },
+                    },
+                ],
+            }
+        ],
+    )
+
+    assert response.content[0].text
+
+    spans = memory_logger.pop()
+    assert len(spans) == 1
+    span = spans[0]
+    content = span["input"][0]["content"]
+    document_block = content[1]
+
+    assert document_block["type"] == "document"
+    assert document_block["source"] == {"type": "base64", "media_type": "application/pdf"}
+    assert isinstance(document_block["image_url"]["url"], Attachment)
+    assert document_block["image_url"]["url"].reference["content_type"] == "application/pdf"
+    assert document_block["image_url"]["url"].reference["filename"] == "document.pdf"
+    assert PDF_BASE64 not in str(span["input"])
+
+
 @pytest.mark.vcr
 def test_anthropic_messages_create_stream_true(memory_logger):
     assert not memory_logger.pop()