Skip to content

Commit b8eaf0e

Browse files
AbhiPrasadAbhiPrasad
authored
test(openai): add vcr regression coverage for stream helpers (#214)
Add explicit sync and async VCR tests for both chat.completions.stream() and responses.stream() and verify the helper paths produce Braintrust spans when the client is wrapped.\n\nAlso keep coverage for the OpenAIIntegration.setup() path so global instrumentation continues to trace the stream helpers and preserve helper-specific behavior.\n\nThese tests cover the regressions discussed in #114 and #170.
1 parent d2d8294 commit b8eaf0e

6 files changed

Lines changed: 1585 additions & 0 deletions

‎py/src/braintrust/integrations/openai/cassettes/TestOpenAIIntegrationSetupSpans.test_setup_stream_helper_creates_spans.yaml‎

Lines changed: 271 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,271 @@
1+
interactions:
2+
- request:
3+
body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}'
4+
headers:
5+
accept:
6+
- application/json
7+
accept-encoding:
8+
- gzip, deflate
9+
connection:
10+
- keep-alive
11+
content-length:
12+
- "134"
13+
content-type:
14+
- application/json
15+
host:
16+
- api.openai.com
17+
user-agent:
18+
- OpenAI/Python 1.82.0
19+
x-stainless-arch:
20+
- arm64
21+
x-stainless-async:
22+
- "false"
23+
x-stainless-lang:
24+
- python
25+
x-stainless-os:
26+
- MacOS
27+
x-stainless-package-version:
28+
- 1.82.0
29+
x-stainless-read-timeout:
30+
- "600"
31+
x-stainless-retry-count:
32+
- "0"
33+
x-stainless-runtime:
34+
- CPython
35+
x-stainless-runtime-version:
36+
- 3.13.3
37+
method: POST
38+
uri: https://api.openai.com/v1/chat/completions
39+
response:
40+
body:
41+
string:
42+
'data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null}
43+
44+
45+
data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null}
46+
47+
48+
data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"
49+
+"},"logprobs":null,"finish_reason":null}],"usage":null}
50+
51+
52+
data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"
53+
"},"logprobs":null,"finish_reason":null}],"usage":null}
54+
55+
56+
data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null}
57+
58+
59+
data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"
60+
equals"},"logprobs":null,"finish_reason":null}],"usage":null}
61+
62+
63+
data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"
64+
"},"logprobs":null,"finish_reason":null}],"usage":null}
65+
66+
67+
data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null}
68+
69+
70+
data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null}
71+
72+
73+
data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null}
74+
75+
76+
data: {"id":"chatcmpl-BcNvsUEkWF6Xk2kAAUSi21kgVYyER","object":"chat.completion.chunk","created":1748488160,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}
77+
78+
79+
data: [DONE]
80+
81+
82+
'
83+
headers:
84+
CF-RAY:
85+
- 9472cadbaa3a6109-EWR
86+
Connection:
87+
- keep-alive
88+
Content-Type:
89+
- text/event-stream; charset=utf-8
90+
Date:
91+
- Thu, 29 May 2025 03:09:21 GMT
92+
Server:
93+
- cloudflare
94+
Set-Cookie:
95+
- __cf_bm=TaY3Tidv7pN1rwXifsEbRpgiJBSYyZmtrqFVhID5fxU-1748488161-1.0.1.1-yge4Qd_h2Czeue_4iHBseCabVMKErLW_dzyoNMs9A2ATm_3je1fFN03F_YfpeLqu1yq_W7BFbR0dqQpI_OxzTmh2tTKXdQDjQLoJP5ivPHI;
96+
path=/; expires=Thu, 29-May-25 03:39:21 GMT; domain=.api.openai.com; HttpOnly;
97+
Secure; SameSite=None
98+
- _cfuvid=hV6VthxryE_9X2dzgxNwp6TyFZV.fQvAIpOzE5TrGpU-1748488161396-0.0.1.1-604800000;
99+
path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
100+
Transfer-Encoding:
101+
- chunked
102+
X-Content-Type-Options:
103+
- nosniff
104+
access-control-expose-headers:
105+
- X-Request-ID
106+
alt-svc:
107+
- h3=":443"; ma=86400
108+
cf-cache-status:
109+
- DYNAMIC
110+
openai-organization:
111+
- braintrust-data
112+
openai-processing-ms:
113+
- "681"
114+
openai-version:
115+
- "2020-10-01"
116+
strict-transport-security:
117+
- max-age=31536000; includeSubDomains; preload
118+
x-envoy-upstream-service-time:
119+
- "686"
120+
x-ratelimit-limit-requests:
121+
- "30000"
122+
x-ratelimit-limit-tokens:
123+
- "150000000"
124+
x-ratelimit-remaining-requests:
125+
- "29999"
126+
x-ratelimit-remaining-tokens:
127+
- "149999993"
128+
x-ratelimit-reset-requests:
129+
- 2ms
130+
x-ratelimit-reset-tokens:
131+
- 0s
132+
x-request-id:
133+
- req_301ee2fbed0944b5203d0a79fc80645b
134+
status:
135+
code: 200
136+
message: OK
137+
- request:
138+
body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}'
139+
headers:
140+
accept:
141+
- application/json
142+
accept-encoding:
143+
- gzip, deflate
144+
connection:
145+
- keep-alive
146+
content-length:
147+
- "134"
148+
content-type:
149+
- application/json
150+
cookie:
151+
- __cf_bm=TaY3Tidv7pN1rwXifsEbRpgiJBSYyZmtrqFVhID5fxU-1748488161-1.0.1.1-yge4Qd_h2Czeue_4iHBseCabVMKErLW_dzyoNMs9A2ATm_3je1fFN03F_YfpeLqu1yq_W7BFbR0dqQpI_OxzTmh2tTKXdQDjQLoJP5ivPHI;
152+
_cfuvid=hV6VthxryE_9X2dzgxNwp6TyFZV.fQvAIpOzE5TrGpU-1748488161396-0.0.1.1-604800000
153+
host:
154+
- api.openai.com
155+
user-agent:
156+
- OpenAI/Python 1.82.0
157+
x-stainless-arch:
158+
- arm64
159+
x-stainless-async:
160+
- "false"
161+
x-stainless-lang:
162+
- python
163+
x-stainless-os:
164+
- MacOS
165+
x-stainless-package-version:
166+
- 1.82.0
167+
x-stainless-raw-response:
168+
- "true"
169+
x-stainless-read-timeout:
170+
- "600"
171+
x-stainless-retry-count:
172+
- "0"
173+
x-stainless-runtime:
174+
- CPython
175+
x-stainless-runtime-version:
176+
- 3.13.3
177+
method: POST
178+
uri: https://api.openai.com/v1/chat/completions
179+
response:
180+
body:
181+
string:
182+
'data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null}
183+
184+
185+
data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null}
186+
187+
188+
data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"
189+
+"},"logprobs":null,"finish_reason":null}],"usage":null}
190+
191+
192+
data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"
193+
"},"logprobs":null,"finish_reason":null}],"usage":null}
194+
195+
196+
data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null}
197+
198+
199+
data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"
200+
equals"},"logprobs":null,"finish_reason":null}],"usage":null}
201+
202+
203+
data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"
204+
"},"logprobs":null,"finish_reason":null}],"usage":null}
205+
206+
207+
data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null}
208+
209+
210+
data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null}
211+
212+
213+
data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null}
214+
215+
216+
data: {"id":"chatcmpl-BcNvt62bH7Bdi0CHQVKMIhuQTFiFp","object":"chat.completion.chunk","created":1748488161,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}
217+
218+
219+
data: [DONE]
220+
221+
222+
'
223+
headers:
224+
CF-RAY:
225+
- 9472cae2792a6109-EWR
226+
Connection:
227+
- keep-alive
228+
Content-Type:
229+
- text/event-stream; charset=utf-8
230+
Date:
231+
- Thu, 29 May 2025 03:09:21 GMT
232+
Server:
233+
- cloudflare
234+
Transfer-Encoding:
235+
- chunked
236+
X-Content-Type-Options:
237+
- nosniff
238+
access-control-expose-headers:
239+
- X-Request-ID
240+
alt-svc:
241+
- h3=":443"; ma=86400
242+
cf-cache-status:
243+
- DYNAMIC
244+
openai-organization:
245+
- braintrust-data
246+
openai-processing-ms:
247+
- "192"
248+
openai-version:
249+
- "2020-10-01"
250+
strict-transport-security:
251+
- max-age=31536000; includeSubDomains; preload
252+
x-envoy-upstream-service-time:
253+
- "195"
254+
x-ratelimit-limit-requests:
255+
- "30000"
256+
x-ratelimit-limit-tokens:
257+
- "150000000"
258+
x-ratelimit-remaining-requests:
259+
- "29999"
260+
x-ratelimit-remaining-tokens:
261+
- "149999993"
262+
x-ratelimit-reset-requests:
263+
- 2ms
264+
x-ratelimit-reset-tokens:
265+
- 0s
266+
x-request-id:
267+
- req_2d19c07b9dca45eb4d8ba0070cf1d930
268+
status:
269+
code: 200
270+
message: OK
271+
version: 1

0 commit comments

Comments
 (0)