diff --git a/py/src/braintrust/integrations/openai/cassettes/TestOpenAIIntegrationSetupSpans.test_setup_stream_helper_creates_spans.yaml b/py/src/braintrust/integrations/openai/cassettes/TestOpenAIIntegrationSetupSpans.test_setup_stream_helper_creates_spans.yaml new file mode 100644 index 00000000..e666c5db --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/TestOpenAIIntegrationSetupSpans.test_setup_stream_helper_creates_spans.yaml @@ -0,0 +1,271 @@ +interactions: + - request: + body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "134" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.82.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - "false" + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.82.0 + x-stainless-read-timeout: + - "600" + x-stainless-retry-count: + - "0" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.3 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: + 'data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + +"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + equals"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} + + + data: [DONE] + + + ' + headers: + CF-RAY: + - 9472cadbaa3a6109-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Thu, 29 May 2025 03:09:21 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=TaY3Tidv7pN1rwXifsEbRpgiJBSYyZmtrqFVhID5fxU-1748488161-1.0.1.1-yge4Qd_h2Czeue_4iHBseCabVMKErLW_dzyoNMs9A2ATm_3je1fFN03F_YfpeLqu1yq_W7BFbR0dqQpI_OxzTmh2tTKXdQDjQLoJP5ivPHI; + path=/; expires=Thu, 29-May-25 03:39:21 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=hV6VthxryE_9X2dzgxNwp6TyFZV.fQvAIpOzE5TrGpU-1748488161396-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - "681" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - "686" + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999993" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_301ee2fbed0944b5203d0a79fc80645b + status: + code: 200 + message: OK + - request: + body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "134" + content-type: + - application/json + cookie: + - __cf_bm=TaY3Tidv7pN1rwXifsEbRpgiJBSYyZmtrqFVhID5fxU-1748488161-1.0.1.1-yge4Qd_h2Czeue_4iHBseCabVMKErLW_dzyoNMs9A2ATm_3je1fFN03F_YfpeLqu1yq_W7BFbR0dqQpI_OxzTmh2tTKXdQDjQLoJP5ivPHI; + _cfuvid=hV6VthxryE_9X2dzgxNwp6TyFZV.fQvAIpOzE5TrGpU-1748488161396-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.82.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - "false" + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.82.0 + x-stainless-raw-response: + - "true" + x-stainless-read-timeout: + - "600" + x-stainless-retry-count: + - "0" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.3 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: + 'data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + +"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + equals"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} + + + data: [DONE] + + + ' + headers: + CF-RAY: + - 9472cae2792a6109-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Thu, 29 May 2025 03:09:21 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - "192" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - "195" + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999993" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_2d19c07b9dca45eb4d8ba0070cf1d930 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/test_openai_chat_stream_helper_async.yaml b/py/src/braintrust/integrations/openai/cassettes/test_openai_chat_stream_helper_async.yaml new file mode 100644 index 00000000..c51d1e70 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/test_openai_chat_stream_helper_async.yaml @@ -0,0 +1,271 @@ +interactions: + - request: + body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "134" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.82.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.82.0 + x-stainless-read-timeout: + - "600" + x-stainless-retry-count: + - "0" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.3 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: + 'data: {"id":"chatcmpl-BcNw1TcpT45K1JBmXqRdGP8bWXEsg","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_54eb4bd693","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1TcpT45K1JBmXqRdGP8bWXEsg","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_54eb4bd693","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1TcpT45K1JBmXqRdGP8bWXEsg","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_54eb4bd693","choices":[{"index":0,"delta":{"content":" + +"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1TcpT45K1JBmXqRdGP8bWXEsg","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_54eb4bd693","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1TcpT45K1JBmXqRdGP8bWXEsg","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_54eb4bd693","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1TcpT45K1JBmXqRdGP8bWXEsg","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_54eb4bd693","choices":[{"index":0,"delta":{"content":" + equals"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1TcpT45K1JBmXqRdGP8bWXEsg","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_54eb4bd693","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1TcpT45K1JBmXqRdGP8bWXEsg","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_54eb4bd693","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1TcpT45K1JBmXqRdGP8bWXEsg","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_54eb4bd693","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1TcpT45K1JBmXqRdGP8bWXEsg","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_54eb4bd693","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1TcpT45K1JBmXqRdGP8bWXEsg","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_54eb4bd693","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} + + + data: [DONE] + + + ' + headers: + CF-RAY: + - 9472cb0fba02b295-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Thu, 29 May 2025 03:09:29 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=UOMiv.S868Q9si.6iaHrxb9QmtUMkB6hpbJdsfYbS2I-1748488169-1.0.1.1-fRHbvwvzOTQqhRsu4RPT5ZyfaUX_DwxyHENDo7.X0zOBcWu_niWJBD5cmvDBWjXfEB3YZRU9cxHmPtmjyhG3rSIJPTUAkTnO_UdVjq_ocow; + path=/; expires=Thu, 29-May-25 03:39:29 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=VP44b.RP9HAvddSD71w9WOG9EnO.dwBkkNkzRK76STg-1748488169490-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - "467" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - "471" + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999993" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_456232b7d84706bbb08d9f97e42c8aaf + status: + code: 200 + message: OK + - request: + body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "134" + content-type: + - application/json + cookie: + - __cf_bm=UOMiv.S868Q9si.6iaHrxb9QmtUMkB6hpbJdsfYbS2I-1748488169-1.0.1.1-fRHbvwvzOTQqhRsu4RPT5ZyfaUX_DwxyHENDo7.X0zOBcWu_niWJBD5cmvDBWjXfEB3YZRU9cxHmPtmjyhG3rSIJPTUAkTnO_UdVjq_ocow; + _cfuvid=VP44b.RP9HAvddSD71w9WOG9EnO.dwBkkNkzRK76STg-1748488169490-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.82.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.82.0 + x-stainless-raw-response: + - "true" + x-stainless-read-timeout: + - "600" + x-stainless-retry-count: + - "0" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.3 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: + 'data: {"id":"chatcmpl-BcNw1HPqJBzP7RFEwiJU8Vr0fOlA7","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1HPqJBzP7RFEwiJU8Vr0fOlA7","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1HPqJBzP7RFEwiJU8Vr0fOlA7","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + +"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1HPqJBzP7RFEwiJU8Vr0fOlA7","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1HPqJBzP7RFEwiJU8Vr0fOlA7","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1HPqJBzP7RFEwiJU8Vr0fOlA7","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + equals"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1HPqJBzP7RFEwiJU8Vr0fOlA7","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1HPqJBzP7RFEwiJU8Vr0fOlA7","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1HPqJBzP7RFEwiJU8Vr0fOlA7","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1HPqJBzP7RFEwiJU8Vr0fOlA7","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + + + data: {"id":"chatcmpl-BcNw1HPqJBzP7RFEwiJU8Vr0fOlA7","object":"chat.completion.chunk","created":1748488169,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} + + + data: [DONE] + + + ' + headers: + CF-RAY: + - 9472cb14bcb3b295-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Thu, 29 May 2025 03:09:30 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - "237" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - "240" + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999993" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_136a017bdaa43b7493f0c1484fc1e487 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/test_openai_chat_stream_helper_sync.yaml b/py/src/braintrust/integrations/openai/cassettes/test_openai_chat_stream_helper_sync.yaml new file mode 100644 index 00000000..e666c5db --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/test_openai_chat_stream_helper_sync.yaml @@ -0,0 +1,271 @@ +interactions: + - request: + body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "134" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.82.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - "false" + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.82.0 + x-stainless-read-timeout: + - "600" + x-stainless-retry-count: + - "0" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.3 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: + 'data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + +"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + equals"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + + + data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} + + + data: [DONE] + + + ' + headers: + CF-RAY: + - 9472cadbaa3a6109-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Thu, 29 May 2025 03:09:21 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=TaY3Tidv7pN1rwXifsEbRpgiJBSYyZmtrqFVhID5fxU-1748488161-1.0.1.1-yge4Qd_h2Czeue_4iHBseCabVMKErLW_dzyoNMs9A2ATm_3je1fFN03F_YfpeLqu1yq_W7BFbR0dqQpI_OxzTmh2tTKXdQDjQLoJP5ivPHI; + path=/; expires=Thu, 29-May-25 03:39:21 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=hV6VthxryE_9X2dzgxNwp6TyFZV.fQvAIpOzE5TrGpU-1748488161396-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - "681" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - "686" + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999993" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_301ee2fbed0944b5203d0a79fc80645b + status: + code: 200 + message: OK + - request: + body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "134" + content-type: + - application/json + cookie: + - __cf_bm=TaY3Tidv7pN1rwXifsEbRpgiJBSYyZmtrqFVhID5fxU-1748488161-1.0.1.1-yge4Qd_h2Czeue_4iHBseCabVMKErLW_dzyoNMs9A2ATm_3je1fFN03F_YfpeLqu1yq_W7BFbR0dqQpI_OxzTmh2tTKXdQDjQLoJP5ivPHI; + _cfuvid=hV6VthxryE_9X2dzgxNwp6TyFZV.fQvAIpOzE5TrGpU-1748488161396-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.82.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - "false" + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.82.0 + x-stainless-raw-response: + - "true" + x-stainless-read-timeout: + - "600" + x-stainless-retry-count: + - "0" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.3 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: + 'data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + +"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + equals"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + + + data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} + + + data: [DONE] + + + ' + headers: + CF-RAY: + - 9472cae2792a6109-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Thu, 29 May 2025 03:09:21 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - "192" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - "195" + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999993" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_2d19c07b9dca45eb4d8ba0070cf1d930 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_stream_helper.yaml b/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_stream_helper.yaml new file mode 100644 index 00000000..37d4add2 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_stream_helper.yaml @@ -0,0 +1,254 @@ +interactions: +- request: + body: '{"input":"What''s 12 + 12?","instructions":"Just the number please","model":"gpt-4o-mini","stream":true}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '103' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.30.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.30.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_011c0be38b89f3cb0069d53005175881a0b0ed231ad335369c","object":"response","created_at":1775579141,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"Just + the number please","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_011c0be38b89f3cb0069d53005175881a0b0ed231ad335369c","object":"response","created_at":1775579141,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"Just + the number please","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_011c0be38b89f3cb0069d53005c18881a0936fc7ffabda2029","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_011c0be38b89f3cb0069d53005c18881a0936fc7ffabda2029","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"24","item_id":"msg_011c0be38b89f3cb0069d53005c18881a0936fc7ffabda2029","logprobs":[],"obfuscation":"3c2Yt64JUVBCsV","output_index":0,"sequence_number":4} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_011c0be38b89f3cb0069d53005c18881a0936fc7ffabda2029","logprobs":[],"output_index":0,"sequence_number":5,"text":"24"} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_011c0be38b89f3cb0069d53005c18881a0936fc7ffabda2029","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"24"},"sequence_number":6} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_011c0be38b89f3cb0069d53005c18881a0936fc7ffabda2029","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"24"}],"role":"assistant"},"output_index":0,"sequence_number":7} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_011c0be38b89f3cb0069d53005175881a0b0ed231ad335369c","object":"response","created_at":1775579141,"status":"completed","background":false,"completed_at":1775579141,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"Just + the number please","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[{"id":"msg_011c0be38b89f3cb0069d53005c18881a0936fc7ffabda2029","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"24"}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":22,"input_tokens_details":{"cached_tokens":0},"output_tokens":2,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":24},"user":null,"metadata":{}},"sequence_number":8} + + + ' + headers: + CF-RAY: + - 9e8a63bc8c56e910-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 07 Apr 2026 16:25:41 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '64' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=sEfCtHLbNdlFrx35NFPJwconpFw89.Ci7QssQT2yitE-1775579140.5661745-1.0.1.1-Ry18SrEBHpAwTsBD72td35g4e5WK03DK0Ti6yUnTVi9Jbs6zu7fqiZxsh384eIXZeOiIY4x87rKcIoSM91MWSdUDUjJNHrQLBdnF1FmhYPX_zFY2X5BbiWajkppAey9Y; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 07 Apr 2026 + 16:55:41 GMT + x-request-id: + - req_68f94c283afc4dbbacc9cb266a52aab9 + status: + code: 200 + message: OK +- request: + body: '{"input":"What''s 12 + 12?","instructions":"Just the number please","model":"gpt-4o-mini","stream":true}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '103' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.30.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.30.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_045ccb72342d21c30069d53006c24081908a670340cecf969f","object":"response","created_at":1775579142,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"Just + the number please","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_045ccb72342d21c30069d53006c24081908a670340cecf969f","object":"response","created_at":1775579142,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"Just + the number please","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_045ccb72342d21c30069d53007b7848190b91d230f55a1dcb3","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_045ccb72342d21c30069d53007b7848190b91d230f55a1dcb3","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"24","item_id":"msg_045ccb72342d21c30069d53007b7848190b91d230f55a1dcb3","logprobs":[],"obfuscation":"zeeYcSD7gzu9zj","output_index":0,"sequence_number":4} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_045ccb72342d21c30069d53007b7848190b91d230f55a1dcb3","logprobs":[],"output_index":0,"sequence_number":5,"text":"24"} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_045ccb72342d21c30069d53007b7848190b91d230f55a1dcb3","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"24"},"sequence_number":6} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_045ccb72342d21c30069d53007b7848190b91d230f55a1dcb3","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"24"}],"role":"assistant"},"output_index":0,"sequence_number":7} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_045ccb72342d21c30069d53006c24081908a670340cecf969f","object":"response","created_at":1775579142,"status":"completed","background":false,"completed_at":1775579143,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"Just + the number please","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[{"id":"msg_045ccb72342d21c30069d53007b7848190b91d230f55a1dcb3","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"24"}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":22,"input_tokens_details":{"cached_tokens":0},"output_tokens":2,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":24},"user":null,"metadata":{}},"sequence_number":8} + + + ' + headers: + CF-RAY: + - 9e8a63c658cff8ac-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 07 Apr 2026 16:25:42 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '151' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=vleTy.4X6LEq.lLNgSzJ9xG4EcJWNoKLZ9rY5V.Wu4o-1775579142.1390355-1.0.1.1-sojO4qLL5psLTwLrXhiLNFljhu_3ipagQwsjloOpdsOSy4MybZIsa.Vs2LYWbE3zYNA8ckfbpbbzStea1QpfEanYaKYF2.5UprWvkjGczljDOIVpyaT_SyY4JoWR0lAm; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 07 Apr 2026 + 16:55:42 GMT + x-request-id: + - req_14e55bd8a1a14212b75e0d0bf12e3e6d + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_stream_helper_async.yaml b/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_stream_helper_async.yaml new file mode 100644 index 00000000..52310262 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_stream_helper_async.yaml @@ -0,0 +1,254 @@ +interactions: +- request: + body: '{"input":"What''s 12 + 12?","instructions":"Just the number please","model":"gpt-4o-mini","stream":true}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '103' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - AsyncOpenAI/Python 2.30.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - async:asyncio + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.30.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_04ed07edb754d72e0069d53008ac7c8197ba2f75887b51fa95","object":"response","created_at":1775579144,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"Just + the number please","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_04ed07edb754d72e0069d53008ac7c8197ba2f75887b51fa95","object":"response","created_at":1775579144,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"Just + the number please","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_04ed07edb754d72e0069d5300989dc819799566cafbd4238d9","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_04ed07edb754d72e0069d5300989dc819799566cafbd4238d9","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"24","item_id":"msg_04ed07edb754d72e0069d5300989dc819799566cafbd4238d9","logprobs":[],"obfuscation":"cYqMAPJV0e6OXI","output_index":0,"sequence_number":4} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_04ed07edb754d72e0069d5300989dc819799566cafbd4238d9","logprobs":[],"output_index":0,"sequence_number":5,"text":"24"} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_04ed07edb754d72e0069d5300989dc819799566cafbd4238d9","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"24"},"sequence_number":6} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_04ed07edb754d72e0069d5300989dc819799566cafbd4238d9","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"24"}],"role":"assistant"},"output_index":0,"sequence_number":7} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_04ed07edb754d72e0069d53008ac7c8197ba2f75887b51fa95","object":"response","created_at":1775579144,"status":"completed","background":false,"completed_at":1775579145,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"Just + the number please","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[{"id":"msg_04ed07edb754d72e0069d5300989dc819799566cafbd4238d9","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"24"}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":22,"input_tokens_details":{"cached_tokens":0},"output_tokens":2,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":24},"user":null,"metadata":{}},"sequence_number":8} + + + ' + headers: + CF-RAY: + - 9e8a63d309abab78-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 07 Apr 2026 16:25:44 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '71' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=wCAkcAahQrEDYWNZI5iqFHEAcJ_Y6TWbsxwgMo0wyxw-1775579144.1650572-1.0.1.1-pasw9NvSZvTQhP42AHeCB2kz8Ud49F4nsoIR.IyKR5B32tmxrw9i7xkhHk43tvlREEzlTq2gjd09muO56bN9BDxjOM0qJYQrSTziYKNR1r15kIfPXUBpzjVvs_xvQzxa; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 07 Apr 2026 + 16:55:44 GMT + x-request-id: + - req_0480db5349d74b7b9df8c36cde4c1357 + status: + code: 200 + message: OK +- request: + body: '{"input":"What''s 12 + 12?","instructions":"Just the number please","model":"gpt-4o-mini","stream":true}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '103' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - AsyncOpenAI/Python 2.30.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - async:asyncio + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.30.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_0a066665c30f7dde0069d53009cba88190a7eb0f340ec92b71","object":"response","created_at":1775579145,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"Just + the number please","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_0a066665c30f7dde0069d53009cba88190a7eb0f340ec92b71","object":"response","created_at":1775579145,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"Just + the number please","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_0a066665c30f7dde0069d5300a3cc48190afe25a5b60c3c810","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0a066665c30f7dde0069d5300a3cc48190afe25a5b60c3c810","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"24","item_id":"msg_0a066665c30f7dde0069d5300a3cc48190afe25a5b60c3c810","logprobs":[],"obfuscation":"4xEtbnXMbggsND","output_index":0,"sequence_number":4} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0a066665c30f7dde0069d5300a3cc48190afe25a5b60c3c810","logprobs":[],"output_index":0,"sequence_number":5,"text":"24"} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0a066665c30f7dde0069d5300a3cc48190afe25a5b60c3c810","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"24"},"sequence_number":6} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_0a066665c30f7dde0069d5300a3cc48190afe25a5b60c3c810","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"24"}],"role":"assistant"},"output_index":0,"sequence_number":7} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_0a066665c30f7dde0069d53009cba88190a7eb0f340ec92b71","object":"response","created_at":1775579145,"status":"completed","background":false,"completed_at":1775579146,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"Just + the number please","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[{"id":"msg_0a066665c30f7dde0069d5300a3cc48190afe25a5b60c3c810","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"24"}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":22,"input_tokens_details":{"cached_tokens":0},"output_tokens":2,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":24},"user":null,"metadata":{}},"sequence_number":8} + + + ' + headers: + CF-RAY: + - 9e8a63dcfc8dab08-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 07 Apr 2026 16:25:45 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '77' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=4W0W71cvqjwed5_unrDEEKxoPSmrRMYS2wWJnsVqs_o-1775579145.75865-1.0.1.1-pw3la5bGMU9fYEKY5kT57Xh05gZMZBT1hJ08VEZ2AamDzFBMeJ0oACZqoxVrTQM3BMzD.Sb6Xvh0xlh867OXGjBuHDcLbLsXsN1eKeM_zuoahqWl0xDWHaLyoMPewGM7; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 07 Apr 2026 + 16:55:45 GMT + x-request-id: + - req_5ae474ba367b4152a8585b1dda628fa0 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index 14cca357..5f14335d 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -333,6 +333,56 @@ def test_openai_chat_streaming_sync(memory_logger): assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower() +@pytest.mark.vcr +def test_openai_chat_stream_helper_sync(memory_logger): + assert not memory_logger.pop() + + if not hasattr(openai.OpenAI().chat.completions, "stream"): + pytest.skip("openai.chat.completions.stream is not available in this SDK version") + + clients = [(openai.OpenAI(), False), (wrap_openai(openai.OpenAI()), True)] + + for client, is_wrapped in clients: + start = time.time() + + with client.chat.completions.stream( + model=TEST_MODEL, + messages=[{"role": "user", "content": TEST_PROMPT}], + stream_options={"include_usage": True}, + ) as stream: + event_types = [] + content_parts = [] + for event in stream: + event_types.append(event.type) + if event.type == "content.delta": + content_parts.append(event.delta) + final = stream.get_final_completion() + end = time.time() + + content = "".join(content_parts) + assert event_types + assert "content.delta" in event_types + assert final.choices[0].message.content + assert "24" in final.choices[0].message.content or "twenty-four" in final.choices[0].message.content.lower() + assert "24" in content or "twenty-four" in content.lower() + + if not is_wrapped: + assert not memory_logger.pop() + continue + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + metrics = span["metrics"] + assert_metrics_are_valid(metrics, start, end) + assert span["metadata"]["stream"] == True + assert span["metadata"]["extra_headers"]["X-Stainless-Helper-Method"] == "chat.completions.stream" + assert TEST_MODEL in span["metadata"]["model"] + assert span["metadata"]["provider"] == "openai" + assert TEST_PROMPT in str(span["input"]) + assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower() + + @pytest.mark.vcr def test_openai_chat_with_system_prompt(memory_logger): assert not memory_logger.pop() @@ -651,6 +701,57 @@ async def test_openai_chat_streaming_async(memory_logger): assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower() +@pytest.mark.asyncio +@pytest.mark.vcr +async def test_openai_chat_stream_helper_async(memory_logger): + assert not memory_logger.pop() + + if not hasattr(AsyncOpenAI().chat.completions, "stream"): + pytest.skip("openai.chat.completions.stream is not available in this SDK version") + + clients = [(AsyncOpenAI(), False), (wrap_openai(AsyncOpenAI()), True)] + + for client, is_wrapped in clients: + start = time.time() + + async with client.chat.completions.stream( + model=TEST_MODEL, + messages=[{"role": "user", "content": TEST_PROMPT}], + stream_options={"include_usage": True}, + ) as stream: + event_types = [] + content_parts = [] + async for event in stream: + event_types.append(event.type) + if event.type == "content.delta": + content_parts.append(event.delta) + final = await stream.get_final_completion() + end = time.time() + + content = "".join(content_parts) + assert event_types + assert "content.delta" in event_types + assert final.choices[0].message.content + assert "24" in final.choices[0].message.content or "twenty-four" in final.choices[0].message.content.lower() + assert "24" in content or "twenty-four" in content.lower() + + if not is_wrapped: + assert not memory_logger.pop() + continue + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + metrics = span["metrics"] + assert_metrics_are_valid(metrics, start, end) + assert span["metadata"]["stream"] == True + assert span["metadata"]["extra_headers"]["X-Stainless-Helper-Method"] == "chat.completions.stream" + assert TEST_MODEL in span["metadata"]["model"] + assert span["metadata"]["provider"] == "openai" + assert TEST_PROMPT in str(span["input"]) + assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower() + + @pytest.mark.asyncio @pytest.mark.vcr async def test_openai_chat_async_with_system_prompt(memory_logger): @@ -898,6 +999,133 @@ async def test_openai_response_streaming_async(memory_logger): assert "24" in str(span["output"]) +@pytest.mark.vcr +def test_openai_responses_stream_helper(memory_logger): + """responses.stream() should preserve the helper interface and emit a tracing span.""" + assert not memory_logger.pop() + + unwrapped_client = openai.OpenAI() + if not hasattr(unwrapped_client.responses, "stream"): + pytest.skip("openai.responses.stream is not available in this SDK version") + + with unwrapped_client.responses.stream( + model=TEST_MODEL, + input=TEST_PROMPT, + instructions="Just the number please", + ) as stream: + event_types = [] + chunks = [] + for event in stream: + event_types.append(event.type) + if event.type == "response.output_text.delta": + chunks.append(event.delta) + final_response = stream.get_final_response() + + output = "".join(chunks) + assert "response.output_text.delta" in event_types + assert final_response.output_text + assert "24" in output or "twenty-four" in output.lower() + assert "24" in final_response.output_text or "twenty-four" in final_response.output_text.lower() + assert not memory_logger.pop() + + client = wrap_openai(openai.OpenAI()) + start = time.time() + with client.responses.stream( + model=TEST_MODEL, + input=TEST_PROMPT, + instructions="Just the number please", + ) as stream: + event_types = [] + chunks = [] + for event in stream: + event_types.append(event.type) + if event.type == "response.output_text.delta": + chunks.append(event.delta) + final_response = stream.get_final_response() + end = time.time() + + output = "".join(chunks) + assert "response.output_text.delta" in event_types + assert final_response.output_text + assert "24" in output or "twenty-four" in output.lower() + assert "24" in final_response.output_text or "twenty-four" in final_response.output_text.lower() + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + metrics = span["metrics"] + assert_metrics_are_valid(metrics, start, end) + assert span["metadata"]["stream"] == True + assert TEST_MODEL in span["metadata"]["model"] + assert span["metadata"]["provider"] == "openai" + assert TEST_PROMPT in str(span["input"]) + assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower() + + +@pytest.mark.asyncio +@pytest.mark.vcr +async def test_openai_responses_stream_helper_async(memory_logger): + """Async responses.stream() should preserve the helper interface and emit a tracing span.""" + assert not memory_logger.pop() + + unwrapped_client = AsyncOpenAI() + if not hasattr(unwrapped_client.responses, "stream"): + pytest.skip("openai.responses.stream is not available in this SDK version") + + async with unwrapped_client.responses.stream( + model=TEST_MODEL, + input=TEST_PROMPT, + instructions="Just the number please", + ) as stream: + event_types = [] + chunks = [] + async for event in stream: + event_types.append(event.type) + if event.type == "response.output_text.delta": + chunks.append(event.delta) + final_response = await stream.get_final_response() + + output = "".join(chunks) + assert "response.output_text.delta" in event_types + assert final_response.output_text + assert "24" in output or "twenty-four" in output.lower() + assert "24" in final_response.output_text or "twenty-four" in final_response.output_text.lower() + assert not memory_logger.pop() + + client = wrap_openai(AsyncOpenAI()) + start = time.time() + async with client.responses.stream( + model=TEST_MODEL, + input=TEST_PROMPT, + instructions="Just the number please", + ) as stream: + event_types = [] + chunks = [] + async for event in stream: + event_types.append(event.type) + if event.type == "response.output_text.delta": + chunks.append(event.delta) + final_response = await stream.get_final_response() + end = time.time() + + output = "".join(chunks) + assert "response.output_text.delta" in event_types + assert final_response.output_text + assert "24" in output or "twenty-four" in output.lower() + assert "24" in final_response.output_text or "twenty-four" in final_response.output_text.lower() + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + metrics = span["metrics"] + assert_metrics_are_valid(metrics, start, end) + assert span["metadata"]["stream"] == True + assert TEST_MODEL in span["metadata"]["model"] + assert span["metadata"]["provider"] == "openai" + assert TEST_PROMPT in str(span["input"]) + assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower() + + @pytest.mark.asyncio @pytest.mark.vcr async def test_openai_async_parallel_requests(memory_logger): @@ -1465,6 +1693,42 @@ def test_setup_creates_spans(self, memory_logger): assert "gpt-4o-mini" in span["metadata"]["model"] assert span["input"] + @pytest.mark.vcr + def test_setup_stream_helper_creates_spans(self, memory_logger): + """OpenAIIntegration.setup() should trace chat.completions.stream().""" + assert not memory_logger.pop() + + if not hasattr(openai.OpenAI().chat.completions, "stream"): + pytest.skip("openai.chat.completions.stream is not available in this SDK version") + + OpenAIIntegration.setup() + client = openai.OpenAI() + + start = time.time() + with client.chat.completions.stream( + model=TEST_MODEL, + messages=[{"role": "user", "content": TEST_PROMPT}], + stream_options={"include_usage": True}, + ) as stream: + event_types = [event.type for event in stream] + final = stream.get_final_completion() + end = time.time() + + assert event_types + assert "content.delta" in event_types + assert final.choices[0].message.content + assert "24" in final.choices[0].message.content or "twenty-four" in final.choices[0].message.content.lower() + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert_metrics_are_valid(span["metrics"], start, end) + assert span["metadata"]["stream"] == True + assert span["metadata"]["extra_headers"]["X-Stainless-Helper-Method"] == "chat.completions.stream" + assert span["metadata"]["provider"] == "openai" + assert TEST_MODEL in span["metadata"]["model"] + assert TEST_PROMPT in str(span["input"]) + class TestOpenAIIntegrationSetupAsyncSpans: """VCR-based tests verifying that OpenAIIntegration.setup() produces spans for async clients."""