diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.log-payloads.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.log-payloads.json index f91eda777..dfce7a8c3 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.log-payloads.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.log-payloads.json @@ -527,5 +527,58 @@ } ], "type": "llm" + }, + { + "input": { + "kind": "undefined" + }, + "metadata": { + "operation": "responses-compact" + }, + "metrics": { + "has_time_to_first_token": false + }, + "name": "openai-responses-compact-operation", + "output": null, + "type": null + }, + { + "input": [ + { + "content_kind": "blocks", + "role": "user" + }, + { + "content_kind": "blocks", + "role": "assistant" + } + ], + "metadata": { + "model": "gpt-4o-mini", + "provider": "openai" + }, + "metrics": { + "has_time_to_first_token": true + }, + "name": "openai.responses.compact", + "output": [ + { + "content_types": [ + "input_text" + ], + "json_keys": [], + "role": "user", + "status": "completed", + "type": "message" + }, + { + "content_types": [], + "json_keys": [], + "role": null, + "status": null, + "type": "compaction" + } + ], + "type": "llm" } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.span-events.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.span-events.json index 4e97348fa..803e72f62 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.span-events.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.span-events.json @@ -361,5 +361,30 @@ }, "name": "openai.responses.parse", "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "responses-compact" + }, + "metrics": { + "has_time_to_first_token": false + }, + "name": "openai-responses-compact-operation", + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gpt-4o-mini", + "provider": "openai" + }, + "metrics": { + "has_time_to_first_token": true + }, + "name": "openai.responses.compact", + "type": "llm" } ] diff --git a/e2e/scenarios/openai-instrumentation/assertions.ts b/e2e/scenarios/openai-instrumentation/assertions.ts index a9cfc2896..8534428c3 100644 --- a/e2e/scenarios/openai-instrumentation/assertions.ts +++ b/e2e/scenarios/openai-instrumentation/assertions.ts @@ -37,6 +37,7 @@ type OperationSpec = { expectsTimeToFirstToken: boolean; name: string; operation: string; + requiresResponsesCompact?: boolean; testName: string; validate?: (span: CapturedLogEvent | undefined) => void; }; @@ -173,8 +174,36 @@ const OPERATION_SPECS: readonly OperationSpec[] = [ expect(output).toContain("value"); }, }, + { + childNames: ["openai.responses.compact"], + expectsOutput: true, + expectsTimeToFirstToken: false, + name: "openai-responses-compact-operation", + operation: "responses-compact", + requiresResponsesCompact: true, + testName: "captures trace for client.responses.compact()", + }, ] as const; +function supportsResponsesCompact(version: string): boolean { + const [majorRaw, minorRaw] = version.split("."); + const major = Number.parseInt(majorRaw ?? "", 10); + const minor = Number.parseInt(minorRaw ?? "", 10); + + return ( + Number.isFinite(major) && + Number.isFinite(minor) && + (major > 6 || (major === 6 && minor >= 10)) + ); +} + +function getOperationSpecs(version: string): OperationSpec[] { + const compactSupported = supportsResponsesCompact(version); + return OPERATION_SPECS.filter( + (spec) => !spec.requiresResponsesCompact || compactSupported, + ); +} + function pickMetadata( metadata: Record | undefined, keys: string[], @@ -335,7 +364,11 @@ function summarizeOutput(name: string, output: Json): Json { : null; } - if (name === "openai.responses.create" || name === "openai.responses.parse") { + if ( + name === "openai.responses.create" || + name === "openai.responses.parse" || + name === "openai.responses.compact" + ) { return summarizeResponsesOutput(output); } @@ -395,12 +428,15 @@ function findOpenAISpan( return undefined; } -function buildRelevantEvents(events: CapturedLogEvent[]) { +function buildRelevantEvents( + events: CapturedLogEvent[], + operationSpecs: OperationSpec[], +) { const relevantEvents: RelevantEvent[] = [ { event: findLatestSpan(events, ROOT_NAME)! }, ]; - for (const spec of OPERATION_SPECS) { + for (const spec of operationSpecs) { const operation = findLatestSpan(events, spec.name)!; relevantEvents.push({ event: operation }); relevantEvents.push({ @@ -412,17 +448,23 @@ function buildRelevantEvents(events: CapturedLogEvent[]) { return relevantEvents; } -function buildSpanSummary(events: CapturedLogEvent[]): Json { +function buildSpanSummary( + events: CapturedLogEvent[], + operationSpecs: OperationSpec[], +): Json { return normalizeForSnapshot( - buildRelevantEvents(events).map(({ event, summaryName }) => + buildRelevantEvents(events, operationSpecs).map(({ event, summaryName }) => summarizeOpenAISpan(event, summaryName), ) as Json, ); } -function buildPayloadSummary(events: CapturedLogEvent[]): Json { +function buildPayloadSummary( + events: CapturedLogEvent[], + operationSpecs: OperationSpec[], +): Json { return normalizeForSnapshot( - buildRelevantEvents(events).map(({ event, summaryName }) => + buildRelevantEvents(events, operationSpecs).map(({ event, summaryName }) => summarizeOpenAIPayload(event, summaryName), ) as Json, ); @@ -437,6 +479,7 @@ export function defineOpenAIInstrumentationAssertions(options: { timeoutMs: number; version: string; }): void { + const operationSpecs = getOperationSpecs(options.version); const spanSnapshotPath = resolveFileSnapshotPath( options.testFileUrl, `${options.snapshotName}.span-events.json`, @@ -489,7 +532,7 @@ export function defineOpenAIInstrumentationAssertions(options: { ); } - for (const spec of OPERATION_SPECS) { + for (const spec of operationSpecs) { test(spec.testName, testConfig, () => { const root = findLatestSpan(events, ROOT_NAME); const operation = findLatestSpan(events, spec.name); @@ -530,13 +573,13 @@ export function defineOpenAIInstrumentationAssertions(options: { test("matches the shared span snapshot", testConfig, async () => { await expect( - formatJsonFileSnapshot(buildSpanSummary(events)), + formatJsonFileSnapshot(buildSpanSummary(events, operationSpecs)), ).toMatchFileSnapshot(spanSnapshotPath); }); test("matches the shared payload snapshot", testConfig, async () => { await expect( - formatJsonFileSnapshot(buildPayloadSummary(events)), + formatJsonFileSnapshot(buildPayloadSummary(events, operationSpecs)), ).toMatchFileSnapshot(payloadSnapshotPath); }); }); diff --git a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs index 99771e5b0..90558bc80 100644 --- a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs @@ -337,6 +337,39 @@ export async function runOpenAIInstrumentationScenario(options) { } }, ); + + if (typeof client.responses?.compact === "function") { + await runOperation( + "openai-responses-compact-operation", + "responses-compact", + async () => { + await client.responses.compact({ + model: OPENAI_MODEL, + input: [ + { + role: "user", + content: [ + { + type: "input_text", + text: "I live in Paris and prefer concise answers.", + }, + ], + }, + { + role: "assistant", + content: [ + { + type: "output_text", + text: "Understood. I will keep answers concise.", + }, + ], + }, + ], + instructions: "Preserve only durable user preferences.", + }); + }, + ); + } }, metadata: { openaiSdkVersion: options.openaiSdkVersion, diff --git a/js/src/auto-instrumentations/configs/openai.ts b/js/src/auto-instrumentations/configs/openai.ts index ca145c887..7db437de4 100644 --- a/js/src/auto-instrumentations/configs/openai.ts +++ b/js/src/auto-instrumentations/configs/openai.ts @@ -186,4 +186,18 @@ export const openaiConfigs: InstrumentationConfig[] = [ kind: "Async", }, }, + + { + channelName: openAIChannels.responsesCompact.channelName, + module: { + name: "openai", + versionRange: ">=6.10.0", + filePath: "resources/responses/responses.mjs", + }, + functionQuery: { + className: "Responses", + methodName: "compact", + kind: "Async", + }, + }, ]; diff --git a/js/src/instrumentation/plugins/openai-channels.ts b/js/src/instrumentation/plugins/openai-channels.ts index 3bd8b2d5d..4347e55b3 100644 --- a/js/src/instrumentation/plugins/openai-channels.ts +++ b/js/src/instrumentation/plugins/openai-channels.ts @@ -12,6 +12,7 @@ import type { OpenAIModerationCreateParams, OpenAIModerationResponse, OpenAIResponse, + OpenAIResponseCompactParams, OpenAIResponseCreateParams, OpenAIResponseStreamEvent, } from "../../vendor-sdk-types/openai"; @@ -104,6 +105,15 @@ export const openAIChannels = defineChannels("openai", { channelName: "responses.parse", kind: "async", }), + + responsesCompact: channel< + [OpenAIResponseCompactParams], + OpenAIResponse, + OpenAIResponsesChannelExtras + >({ + channelName: "responses.compact", + kind: "async", + }), }); export type OpenAIChannel = diff --git a/js/src/instrumentation/plugins/openai-plugin.ts b/js/src/instrumentation/plugins/openai-plugin.ts index 62a48e62f..4adc7b068 100644 --- a/js/src/instrumentation/plugins/openai-plugin.ts +++ b/js/src/instrumentation/plugins/openai-plugin.ts @@ -29,7 +29,7 @@ import type { * - Embeddings * - Moderations * - Beta API (parse, stream) - * - Responses API (create, stream, parse) + * - Responses API (create, stream, parse, compact) */ export class OpenAIPlugin extends BasePlugin { constructor() { @@ -273,6 +273,42 @@ export class OpenAIPlugin extends BasePlugin { aggregateChunks: aggregateResponseStreamEvents, }), ); + + // Responses API - compact + this.unsubscribers.push( + traceAsyncChannel(openAIChannels.responsesCompact, { + name: "openai.responses.compact", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input: processInputAttachments(input), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return processImagesInOutput(result?.output); + }, + extractMetadata: (result) => { + if (!result) { + return undefined; + } + const { output: _output, usage: _usage, ...metadata } = result; + return Object.keys(metadata).length > 0 ? metadata : undefined; + }, + extractMetrics: (result, startTime, endEvent) => { + const metrics = withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + }), + ); } protected onDisable(): void { diff --git a/js/src/vendor-sdk-types/openai-common.ts b/js/src/vendor-sdk-types/openai-common.ts index 2c3eb4894..8f8d5727e 100644 --- a/js/src/vendor-sdk-types/openai-common.ts +++ b/js/src/vendor-sdk-types/openai-common.ts @@ -39,6 +39,11 @@ export interface OpenAIResponseCreateParams { [key: string]: unknown; } +export interface OpenAIResponseCompactParams { + input: unknown; + [key: string]: unknown; +} + // Responses export interface OpenAIUsage { @@ -208,6 +213,10 @@ export interface OpenAIResponses { params: OpenAIResponseCreateParams, options?: unknown, ) => OpenAIAPIPromise; + compact?: ( + params: OpenAIResponseCompactParams, + options?: unknown, + ) => OpenAIAPIPromise; parse?: ( params: OpenAIResponseCreateParams, options?: unknown, diff --git a/js/src/vendor-sdk-types/openai.ts b/js/src/vendor-sdk-types/openai.ts index 12f735a4b..c00dc261a 100644 --- a/js/src/vendor-sdk-types/openai.ts +++ b/js/src/vendor-sdk-types/openai.ts @@ -9,6 +9,7 @@ import type { OpenAIModerationCreateParams, OpenAIModerationResponse, OpenAIResponse, + OpenAIResponseCompactParams, OpenAIResponseCompletedEvent, OpenAIResponseCreateParams, OpenAIResponseStreamEvent, @@ -35,6 +36,7 @@ export type { OpenAIModerationCreateParams, OpenAIModerationResponse, OpenAIResponse, + OpenAIResponseCompactParams, OpenAIResponseCompletedEvent, OpenAIResponseCreateParams, OpenAIResponseStreamEvent, diff --git a/js/src/wrappers/oai.test.ts b/js/src/wrappers/oai.test.ts index cde312a47..ea2261fc9 100644 --- a/js/src/wrappers/oai.test.ts +++ b/js/src/wrappers/oai.test.ts @@ -734,6 +734,74 @@ describe("openai client unit tests", TEST_SUITE_OPTIONS, () => { assert.isTrue(m.completion_reasoning_tokens >= 0); }); + test("openai.responses.compact", async (context) => { + if (!oai.responses || typeof oai.responses.compact !== "function") { + context.skip(); + } + const wrappedCompact = client.responses?.compact; + if (typeof wrappedCompact !== "function") { + context.skip(); + } + + assert.lengthOf(await backgroundLogger.drain(), 0); + + const compactInput = [ + { + role: "user", + content: [ + { + type: "input_text", + text: "My name is Ada and I like concise responses.", + }, + ], + }, + { + role: "assistant", + content: [ + { + type: "output_text", + text: "Nice to meet you, Ada. I will keep responses concise.", + }, + ], + }, + ]; + + const compactArgs = { + model: TEST_MODEL, + input: compactInput, + instructions: "Keep only durable user preferences.", + }; + + const unwrappedResponse = await oai.responses.compact(compactArgs); + assert.ok(unwrappedResponse); + assert.lengthOf(await backgroundLogger.drain(), 0); + + const start = getCurrentUnixTimestamp(); + const response = await wrappedCompact(compactArgs); + const end = getCurrentUnixTimestamp(); + + assert.ok(response); + + const spans = await backgroundLogger.drain(); + assert.lengthOf(spans, 1); + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions, @typescript-eslint/no-explicit-any + const span = spans[0] as any; + assert.equal(span.span_attributes.name, "openai.responses.compact"); + assert.equal(span.span_attributes.type, "llm"); + assert.deepEqual(span.input, compactInput); + assert.equal(span.metadata.provider, "openai"); + assert.equal(span.metadata.instructions, compactArgs.instructions); + assert.ok(span.metadata.model.startsWith(TEST_MODEL)); + assert.isDefined(span.output); + + const m = span.metrics; + assert.isTrue(start <= m.start && m.start < m.end && m.end <= end); + if (m.tokens !== undefined) { + assert.isTrue(m.tokens > 0); + assert.isTrue(m.prompt_tokens > 0); + } + }); + test("openai.chat.completions.parse (v5 GA method)", async () => { // Test that the parse method is properly wrapped in the GA namespace (v5) if (!oai.chat?.completions?.parse) { diff --git a/js/src/wrappers/oai_responses.ts b/js/src/wrappers/oai_responses.ts index c108fedb5..8e936ebbc 100644 --- a/js/src/wrappers/oai_responses.ts +++ b/js/src/wrappers/oai_responses.ts @@ -26,21 +26,26 @@ export function responsesProxy(openai: any) { return new Proxy(openai.responses, { get(target, name, receiver) { - if (name === "create") { + if (name === "create" && typeof target.create === "function") { return wrapResponsesAsync( target.create.bind(target), openAIChannels.responsesCreate, ); - } else if (name === "stream") { + } else if (name === "stream" && typeof target.stream === "function") { return wrapResponsesSyncStream( target.stream.bind(target), openAIChannels.responsesStream, ); - } else if (name === "parse") { + } else if (name === "parse" && typeof target.parse === "function") { return wrapResponsesAsync( target.parse.bind(target), openAIChannels.responsesParse, ); + } else if (name === "compact" && typeof target.compact === "function") { + return wrapResponsesAsync( + target.compact.bind(target), + openAIChannels.responsesCompact, + ); } return Reflect.get(target, name, receiver); }, @@ -50,7 +55,8 @@ export function responsesProxy(openai: any) { function wrapResponsesAsync< TChannel extends | typeof openAIChannels.responsesCreate - | typeof openAIChannels.responsesParse, + | typeof openAIChannels.responsesParse + | typeof openAIChannels.responsesCompact, >( target: ( params: ArgsOf[0],