diff --git a/src/__tests__/schema-drift.test.ts b/src/__tests__/schema-drift.test.ts new file mode 100644 index 0000000..ca2a9bf --- /dev/null +++ b/src/__tests__/schema-drift.test.ts @@ -0,0 +1,671 @@ +/** + * Schema Drift Detection Tests + * + * Covers the response-envelope schema validator, Anthropic provider drift + * detection at the parser boundary, factory fallback routing on drift, + * and onSchemaDrift observability hook firing. + * + * Slice 1 of #39 — Anthropic is the template provider. Follow-up tests for + * openai/groq/cerebras/cloudflare land with the respective wire-ups. + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { validateSchema, type SchemaField } from '../utils/schema-validator'; +import { SchemaDriftError } from '../errors'; +import { AnthropicProvider } from '../providers/anthropic'; +import { LLMProviderFactory } from '../factory'; +import { defaultCircuitBreakerManager } from '../utils/circuit-breaker'; +import { defaultExhaustionRegistry } from '../utils/exhaustion'; +import { defaultCostTracker } from '../utils/cost-tracker'; +import { defaultLatencyHistogram } from '../utils/latency-histogram'; +import type { ObservabilityHooks, SchemaDriftEvent } from '../utils/hooks'; + +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +// ── validateSchema primitives ──────────────────────────────────────────── + +describe('validateSchema', () => { + it('accepts a valid envelope with all required fields', () => { + const schema: SchemaField[] = [ + { path: 'id', type: 'string' }, + { path: 'usage.input_tokens', type: 'number' }, + ]; + expect(() => validateSchema('test', { id: 'x', usage: { input_tokens: 5 } }, schema)) + .not.toThrow(); + }); + + it('throws SchemaDriftError when a required path is missing', () => { + const schema: SchemaField[] = [{ path: 'usage.input_tokens', type: 'number' }]; + try { + validateSchema('test', { usage: {} }, schema); + expect.fail('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(SchemaDriftError); + expect((err as SchemaDriftError).path).toBe('usage.input_tokens'); + expect((err as SchemaDriftError).expected).toBe('number'); + expect((err as SchemaDriftError).actual).toBe('undefined'); + } + }); + + it('throws with correct actual type when field has wrong type', () => { + const schema: SchemaField[] = [{ path: 'content', type: 'array' }]; + try { + validateSchema('test', { content: 'oops-its-a-string-now' }, schema); + expect.fail('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(SchemaDriftError); + expect((err as SchemaDriftError).actual).toBe('string'); + } + }); + + it('distinguishes null from undefined from object', () => { + const schema: SchemaField[] = [{ path: 'usage', type: 'object' }]; + + try { + validateSchema('test', { usage: null }, schema); + expect.fail('should have thrown'); + } catch (err) { + expect((err as SchemaDriftError).actual).toBe('null'); + } + + try { + validateSchema('test', {}, schema); + expect.fail('should have thrown'); + } catch (err) { + expect((err as SchemaDriftError).actual).toBe('undefined'); + } + }); + + it('allows optional fields to be missing', () => { + const schema: SchemaField[] = [ + { path: 'id', type: 'string' }, + { path: 'stop_sequence', type: 'string', optional: true }, + ]; + expect(() => validateSchema('test', { id: 'x' }, schema)).not.toThrow(); + }); + + it('rejects NaN for number fields (Number.isFinite guard)', () => { + const schema: SchemaField[] = [{ path: 'tokens', type: 'number' }]; + expect(() => validateSchema('test', { tokens: NaN }, schema)) + .toThrow(SchemaDriftError); + }); + + it('rejects arrays when type is "object"', () => { + const schema: SchemaField[] = [{ path: 'usage', type: 'object' }]; + try { + validateSchema('test', { usage: [] }, schema); + expect.fail('should have thrown'); + } catch (err) { + expect((err as SchemaDriftError).actual).toBe('array'); + } + }); + + it('throws when root is not an object', () => { + expect(() => validateSchema('test', null, [])) + .toThrow(SchemaDriftError); + expect(() => validateSchema('test', 'a string', [])) + .toThrow(SchemaDriftError); + }); + + it('supports string-or-null type for nullable content fields', () => { + const schema: SchemaField[] = [{ path: 'content', type: 'string-or-null' }]; + expect(() => validateSchema('test', { content: null }, schema)).not.toThrow(); + expect(() => validateSchema('test', { content: 'hi' }, schema)).not.toThrow(); + expect(() => validateSchema('test', { content: 42 }, schema)) + .toThrow(SchemaDriftError); + }); + + it('fails fast on first drift (does not keep walking)', () => { + const schema: SchemaField[] = [ + { path: 'first', type: 'string' }, + { path: 'second', type: 'string' }, + ]; + try { + validateSchema('test', { first: 42, second: 99 }, schema); + expect.fail('should have thrown'); + } catch (err) { + // Should report the first failure, not the second + expect((err as SchemaDriftError).path).toBe('first'); + } + }); +}); + +// ── Anthropic provider drift detection ─────────────────────────────────── + +describe('AnthropicProvider response schema validation', () => { + let provider: AnthropicProvider; + + beforeEach(() => { + vi.clearAllMocks(); + defaultCircuitBreakerManager.resetAll(); + provider = new AnthropicProvider({ apiKey: 'test-key', maxRetries: 0 }); + }); + + const validAnthropicResponse = { + id: 'msg_1', + type: 'message', + role: 'assistant', + content: [{ type: 'text', text: 'hello' }], + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 5 }, + }; + + it('passes through a well-formed Anthropic response', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => validAnthropicResponse, + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const res = await provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + }); + + expect(res.content).toBe('hello'); + expect(res.usage.inputTokens).toBe(10); + }); + + it('throws SchemaDriftError when usage.input_tokens is renamed', async () => { + // Simulated drift: Anthropic silently renames input_tokens → prompt_tokens + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + ...validAnthropicResponse, + usage: { prompt_tokens: 10, output_tokens: 5 }, + }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + name: 'LLMProviderError', + code: 'SCHEMA_DRIFT', + provider: 'anthropic', + path: 'usage.input_tokens', + }); + }); + + it('throws SchemaDriftError when content field is removed', async () => { + const { content: _content, ...rest } = validAnthropicResponse; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => rest, + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ code: 'SCHEMA_DRIFT', path: 'content' }); + }); + + it('throws SchemaDriftError when content type changes from array to string', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ ...validAnthropicResponse, content: 'flat text instead of blocks' }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + path: 'content', + expected: 'array', + actual: 'string', + }); + }); + + it('is non-retryable so retries do not burn budget on drift', async () => { + // maxRetries on the provider is 0, but even with retries enabled the + // retry manager should skip non-retryable errors. Assert via a single + // failing response: if SchemaDriftError were retryable, fetch would be + // called more than once. + const retryProvider = new AnthropicProvider({ apiKey: 'test-key', maxRetries: 3 }); + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ ...validAnthropicResponse, usage: {} }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(retryProvider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ code: 'SCHEMA_DRIFT' }); + + expect(mockFetch).toHaveBeenCalledTimes(1); + }); +}); + +// ── Factory fallback on SchemaDriftError ───────────────────────────────── + +describe('LLMProviderFactory schema drift fallback', () => { + beforeEach(() => { + vi.clearAllMocks(); + defaultCircuitBreakerManager.resetAll(); + defaultExhaustionRegistry.reset(); + defaultCostTracker.reset(); + defaultLatencyHistogram.reset(); + }); + + const validOpenAIResponse = { + id: 'chatcmpl-1', + model: 'gpt-4o', + choices: [{ + index: 0, + message: { role: 'assistant', content: 'from openai' }, + finish_reason: 'stop', + }], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }; + + it('falls over to another provider when the primary drifts', async () => { + // First call = anthropic with missing content field (drift) + // Second call = openai with valid shape (success) + mockFetch + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: 'msg_1', + type: 'message', + role: 'assistant', + // content intentionally missing — drift + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 5 }, + }), + headers: new Headers({ 'content-type': 'application/json' }), + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => validOpenAIResponse, + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const factory = new LLMProviderFactory({ + anthropic: { apiKey: 'test-anthropic', maxRetries: 0 }, + openai: { apiKey: 'test-openai', maxRetries: 0 }, + preferredProvider: 'anthropic', + }); + + const response = await factory.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + }); + + expect(response.provider).toBe('openai'); + expect(response.content).toBe('from openai'); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('fires onSchemaDrift hook with path, expected, actual', async () => { + const driftEvents: SchemaDriftEvent[] = []; + const hooks: ObservabilityHooks = { + onSchemaDrift: (e) => driftEvents.push(e), + }; + + mockFetch + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: 'msg_1', + type: 'message', + role: 'assistant', + content: [], + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { prompt_tokens: 10 }, // wrong field name — drift on usage.input_tokens + }), + headers: new Headers({ 'content-type': 'application/json' }), + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => validOpenAIResponse, + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const factory = new LLMProviderFactory({ + anthropic: { apiKey: 'test-anthropic', maxRetries: 0 }, + openai: { apiKey: 'test-openai', maxRetries: 0 }, + preferredProvider: 'anthropic', + hooks, + }); + + await factory.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + requestId: 'req-drift-1', + }); + + expect(driftEvents).toHaveLength(1); + expect(driftEvents[0]).toMatchObject({ + provider: 'anthropic', + requestId: 'req-drift-1', + path: 'usage.input_tokens', + expected: 'number', + actual: 'undefined', + }); + expect(driftEvents[0].timestamp).toBeGreaterThan(0); + }); + + it('propagates SchemaDriftError at end of chain when no fallback is available (M-2)', async () => { + // Single-provider chain: if anthropic drifts, there is nowhere to fall + // over to. The factory must surface the SchemaDriftError to the caller + // without wrapping it as generic ALL_PROVIDERS_FAILED, and the hook must + // still fire so ops sees the drift. + // + // (Multi-provider end-of-chain coverage lands with the follow-up PR + // that wires schema validation into the other providers — until then, + // the other providers' parsers throw TypeError on drift, which isn't + // this test's concern.) + const driftEvents: SchemaDriftEvent[] = []; + const hooks: ObservabilityHooks = { + onSchemaDrift: (e) => driftEvents.push(e), + }; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: 'msg_1', type: 'message', role: 'assistant', + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 5 }, + // content intentionally missing — drift + }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const factory = new LLMProviderFactory({ + anthropic: { apiKey: 'test-anthropic', maxRetries: 0 }, + preferredProvider: 'anthropic', + hooks, + }); + + await expect(factory.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + requestId: 'req-eoc-1', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + provider: 'anthropic', + path: 'content', + }); + + expect(driftEvents).toHaveLength(1); + expect(driftEvents[0]).toMatchObject({ + provider: 'anthropic', + requestId: 'req-eoc-1', + }); + }); +}); + +// ── Circuit breaker + drift interaction ────────────────────────────────── + +describe('SchemaDriftError and circuit breaker', () => { + beforeEach(() => { + vi.clearAllMocks(); + defaultCircuitBreakerManager.resetAll(); + }); + + it('repeated drift trips the circuit breaker open (M-1)', async () => { + // The default breaker uses a graduated degradation curve with a + // probabilistic rejection gate at DEGRADED levels. To get deterministic + // progression all the way to OPEN, stub Math.random so the gate always + // admits the request through to fn() - where the drift throws and the + // breaker advances one failure. Without this stub, the test is racy: + // locally it would pass by luck, CI would fail when the random sequence + // bounced attempts off the degradation gate without advancing. + const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0); + + try { + const provider = new AnthropicProvider({ apiKey: 'test-key', maxRetries: 0 }); + + // Return drifted response every call - missing usage.input_tokens + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + id: 'msg_1', type: 'message', role: 'assistant', + content: [], + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { output_tokens: 5 }, + }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + // Default failureThreshold = 5. 10 attempts is more than enough with + // the probabilistic gate pinned open. + for (let i = 0; i < 10; i++) { + try { + await provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + }); + } catch { + // Expected - drift or circuit-open + } + } + + const breaker = defaultCircuitBreakerManager.getBreaker('anthropic'); + expect(breaker.getState().state).toBe('OPEN'); + } finally { + randomSpy.mockRestore(); + } + }); +}); + +// ── Security: no value leakage in error surface ────────────────────────── + +describe('SchemaDriftError message surface (security)', () => { + it('error message and hook payload contain only path + type names, never values', async () => { + vi.clearAllMocks(); + defaultCircuitBreakerManager.resetAll(); + const provider = new AnthropicProvider({ apiKey: 'test-key', maxRetries: 0 }); + + // Drifted response contains a sensitive-looking value. If it leaks into + // the error message we have a telemetry PII problem. + const SECRET_VALUE = 'sk-verysecret-api-key-shouldnotleak'; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: SECRET_VALUE, // intentionally put a "secret" where we expect a string + // content missing — triggers drift on content path instead of id + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 5 }, + }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + try { + await provider.generateResponse({ + messages: [{ role: 'user', content: SECRET_VALUE }], + model: 'claude-3-haiku-20240307', + }); + expect.fail('should have thrown'); + } catch (err) { + const message = (err as Error).message; + const serialized = JSON.stringify(err, Object.getOwnPropertyNames(err)); + + // The path, expected type, actual type should all be present + expect(message).toContain('content'); + expect(message).toContain('array'); + expect(message).toContain('undefined'); + + // The secret value should NEVER be in the error surface + expect(message).not.toContain(SECRET_VALUE); + expect(serialized).not.toContain(SECRET_VALUE); + } + }); +}); + +// ── H-2: nested content block validation ───────────────────────────────── + +describe('Anthropic nested content-block validation (H-2 / #42)', () => { + let provider: AnthropicProvider; + + beforeEach(() => { + vi.clearAllMocks(); + defaultCircuitBreakerManager.resetAll(); + provider = new AnthropicProvider({ apiKey: 'test-key', maxRetries: 0 }); + }); + + const envelope = (content: unknown[]) => ({ + id: 'msg_1', + type: 'message', + role: 'assistant', + content, + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 5 }, + }); + + it('accepts a valid tool_use block', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([ + { type: 'tool_use', id: 'toolu_1', name: 'search', input: { q: 'test' } }, + ]), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const res = await provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + }); + expect(res.toolCalls).toHaveLength(1); + }); + + it('detects tool_use block with missing id', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([ + { type: 'tool_use', name: 'search', input: { q: 'test' } }, + ]), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + path: 'content[0].id', + expected: 'string', + actual: 'undefined', + }); + }); + + it('detects tool_use block with wrong-typed input', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([ + { type: 'tool_use', id: 'toolu_1', name: 'search', input: 'should-be-object' }, + ]), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + path: 'content[0].input', + expected: 'object', + actual: 'string', + }); + }); + + it('detects text block with missing .text field', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([ + { type: 'text' }, // .text renamed/removed + ]), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + path: 'content[0].text', + }); + }); + + it('detects element that is not an object', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope(['a bare string, not a block']), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + path: 'content[0]', + expected: 'object', + actual: 'string', + }); + }); + + it('detects missing discriminator (type field)', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([ + { text: 'no type field' }, + ]), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + path: 'content[0].type', + expected: 'string', + }); + }); + + it('accepts unknown block types (forward-compat)', async () => { + // Anthropic adds a new block type 'reasoning' — we should pass without + // error so the next deploy doesn't break on additive changes. + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([ + { type: 'text', text: 'hello' }, + { type: 'reasoning', thoughts: 'internal thinking trace', confidence: 0.9 }, + ]), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const res = await provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + }); + // Text content extracted; unknown block silently ignored by filter + expect(res.content).toBe('hello'); + }); + + it('accepts response without stop_sequence (optional field)', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([{ type: 'text', text: 'hi' }]), + // Note: no stop_sequence key — it's optional + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const res = await provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + }); + expect(res.content).toBe('hi'); + }); +}); diff --git a/src/errors.ts b/src/errors.ts index 85640df..5e72ac2 100755 --- a/src/errors.ts +++ b/src/errors.ts @@ -125,6 +125,33 @@ export class ToolLoopAbortedError extends LLMProviderError { super(message, 'TOOL_LOOP_ABORTED', provider, false, 400); } } + +/** + * Thrown when a provider's response envelope fails runtime schema validation. + * + * Indicates the upstream API silently changed shape - a field was renamed, + * removed, or had its type changed. Non-retryable: retrying hits the same + * broken shape. The factory treats this as fallback-eligible so traffic + * routes to a healthy provider while the drift is investigated. + */ +export class SchemaDriftError extends LLMProviderError { + path: string; + expected: string; + actual: string; + + constructor(provider: string, path: string, expected: string, actual: string) { + super( + `Response schema drift at ${path}: expected ${expected}, got ${actual}`, + 'SCHEMA_DRIFT', + provider, + false, + 422 + ); + this.path = path; + this.expected = expected; + this.actual = actual; + } +} /** * Error factory for creating provider-specific errors from HTTP responses diff --git a/src/factory.ts b/src/factory.ts index 48498d9..714eba6 100755 --- a/src/factory.ts +++ b/src/factory.ts @@ -51,6 +51,7 @@ import { AuthenticationError, RateLimitError, QuotaExceededError, + SchemaDriftError, ToolLoopAbortedError, ToolLoopLimitError, } from './errors'; @@ -274,6 +275,20 @@ export class LLMProviderFactory { }); } + // Schema drift — the upstream API silently changed shape. Surface + // structured telemetry so oncall sees the drift before it cascades. + if (err instanceof SchemaDriftError) { + this.hooks.onSchemaDrift?.({ + provider: providerName, + model: request.model, + requestId: request.requestId, + path: err.path, + expected: err.expected, + actual: err.actual, + timestamp: Date.now(), + }); + } + const fallbackDecision = this.getFallbackDecision(error as Error); if (!fallbackDecision.shouldFallback) { throw error; @@ -709,10 +724,17 @@ export class LLMProviderFactory { return { shouldFallback: true }; } + // Schema drift: provider's response shape changed. Retry won't help; + // only another provider can. Defense against silent API deprecations. + if (error instanceof SchemaDriftError) { + return { shouldFallback: true }; + } + if (error instanceof LLMProviderError) { if (error.code === 'SERVER_ERROR' || error.code === 'NETWORK_ERROR' || - error.code === 'TIMEOUT') { + error.code === 'TIMEOUT' || + error.code === 'SCHEMA_DRIFT') { return { shouldFallback: true }; } } diff --git a/src/index.ts b/src/index.ts index fd06fe8..a2268ed 100755 --- a/src/index.ts +++ b/src/index.ts @@ -99,10 +99,15 @@ export { TokenLimitError, ConfigurationError, CircuitBreakerOpenError, + SchemaDriftError, ToolLoopLimitError, ToolLoopAbortedError, LLMErrorFactory } from './errors'; + +// Schema validator (for custom provider authors) +export { validateSchema } from './utils/schema-validator'; +export type { SchemaField, SchemaFieldType } from './utils/schema-validator'; // Image generation export { ImageProvider, normalizeAiResponse } from './image/index'; @@ -128,6 +133,7 @@ export type { QuotaCheckEvent, QuotaDeniedEvent, ProviderBalanceEvent, + SchemaDriftEvent, } from './utils/hooks'; // Exhaustion registry diff --git a/src/providers/anthropic.ts b/src/providers/anthropic.ts index e5308a3..2565b5c 100755 --- a/src/providers/anthropic.ts +++ b/src/providers/anthropic.ts @@ -19,6 +19,42 @@ import { ModelNotFoundError, RateLimitError } from '../errors'; +import { validateSchema, type SchemaField } from '../utils/schema-validator'; + +/** + * Minimum envelope fields the Anthropic response parser reads. Changing this + * list is a contract change — keep it in sync with formatResponse below. + * + * Content blocks are validated as a discriminated union on `type`. Unknown + * block types are forward-compatible (skipped) so Anthropic adding a new + * block variant doesn't instantly break the fallback routing on every deploy. + */ +const ANTHROPIC_RESPONSE_SCHEMA: SchemaField[] = [ + { path: 'id', type: 'string' }, + { + path: 'content', + type: 'array', + items: { + discriminator: 'type', + variants: { + text: [ + { path: 'text', type: 'string' }, + ], + tool_use: [ + { path: 'id', type: 'string' }, + { path: 'name', type: 'string' }, + { path: 'input', type: 'object' }, + ], + }, + }, + }, + { path: 'model', type: 'string' }, + { path: 'stop_reason', type: 'string' }, + { path: 'stop_sequence', type: 'string', optional: true }, + { path: 'usage', type: 'object' }, + { path: 'usage.input_tokens', type: 'number' }, + { path: 'usage.output_tokens', type: 'number' }, +]; interface AnthropicContentBlock { type: 'text' | 'tool_use' | 'tool_result' | 'image'; @@ -132,8 +168,9 @@ export class AnthropicProvider extends BaseProvider { throw await LLMErrorFactory.fromFetchResponse('anthropic', httpResponse); } - const data: AnthropicResponse = await httpResponse.json(); - const formatted = this.formatResponse(data, Date.now() - startTime); + const data = await httpResponse.json() as unknown; + validateSchema('anthropic', data, ANTHROPIC_RESPONSE_SCHEMA); + const formatted = this.formatResponse(data as AnthropicResponse, Date.now() - startTime); // Restore the prefilled '{' consumed by the assistant turn, // but only if the response doesn't already start with one @@ -497,14 +534,17 @@ export class AnthropicProvider extends BaseProvider { } }; - // Extract tool calls if present (validated at provider boundary) + // Extract tool calls if present. id/name/input are guaranteed non-null + // for tool_use blocks by ANTHROPIC_RESPONSE_SCHEMA's discriminated-union + // validation — if they were missing, validateSchema would have thrown + // SchemaDriftError before we got here. const toolUses = data.content.filter(block => block.type === 'tool_use'); if (toolUses.length > 0) { const raw: ToolCall[] = toolUses.map(tool => ({ - id: tool.id!, + id: tool.id as string, type: 'function' as const, function: { - name: tool.name!, + name: tool.name as string, arguments: JSON.stringify(tool.input) } })); diff --git a/src/utils/hooks.ts b/src/utils/hooks.ts index 3eb96ac..d6f82da 100644 --- a/src/utils/hooks.ts +++ b/src/utils/hooks.ts @@ -112,6 +112,16 @@ export interface ProviderBalanceEvent { timestamp: number; } +export interface SchemaDriftEvent { + provider: string; + model?: string; + requestId?: string; + path: string; + expected: string; + actual: string; + timestamp: number; +} + // ── Hooks interface ────────────────────────────────────────────────────── export interface ObservabilityHooks { @@ -126,6 +136,7 @@ export interface ObservabilityHooks { onQuotaCheck?(event: QuotaCheckEvent): void; onQuotaDenied?(event: QuotaDeniedEvent): void; onProviderBalance?(event: ProviderBalanceEvent): void; + onSchemaDrift?(event: SchemaDriftEvent): void; } /** Silent hooks — default. */ @@ -140,7 +151,7 @@ export function composeHooks(...implementations: ObservabilityHooks[]): Observab 'onRequestStart', 'onRequestEnd', 'onRequestError', 'onRetry', 'onFallback', 'onCircuitStateChange', 'onQuotaExhausted', 'onBudgetThreshold', 'onQuotaCheck', - 'onQuotaDenied', 'onProviderBalance', + 'onQuotaDenied', 'onProviderBalance', 'onSchemaDrift', ] as const; const composed: ObservabilityHooks = {}; diff --git a/src/utils/schema-validator.ts b/src/utils/schema-validator.ts new file mode 100644 index 0000000..90d9203 --- /dev/null +++ b/src/utils/schema-validator.ts @@ -0,0 +1,213 @@ +/** + * Response Envelope Schema Validator + * + * Zero-dependency runtime validator for provider response shapes. Used at the + * provider boundary to detect when an upstream API silently changes its + * response envelope — the classic "field renamed at 2am, parser throws at + * 3am" failure mode. + * + * Philosophy: validate the *minimum* fields each provider's parser actually + * reads. Don't re-type the full upstream schema — that creates brittle + * over-specification that churns every time the provider adds an optional + * field. Only the fields we touch matter. + * + * Usage: + * validateSchema('anthropic', data, [ + * { path: 'content', type: 'array' }, + * { path: 'usage.input_tokens', type: 'number' }, + * { path: 'usage.output_tokens', type: 'number' }, + * { path: 'model', type: 'string' }, + * ]); + * + * On first failure, throws SchemaDriftError. The caller (typically the + * factory) catches it, fires the onSchemaDrift hook, and falls over to + * another provider. + */ + +import { SchemaDriftError } from '../errors'; + +export type SchemaFieldType = + | 'string' + | 'number' + | 'boolean' + | 'array' + | 'object' + | 'string-or-null'; // anthropic/openai sometimes null content fields + +export interface SchemaField { + /** + * Dot-separated path into the response object. Array indices not supported + * in the path itself — use `items` to validate array element shapes. + */ + path: string; + type: SchemaFieldType; + /** + * If true, missing paths are allowed and skipped. Useful for fields that + * are genuinely optional (e.g. stop_sequence on Anthropic). + */ + optional?: boolean; + /** + * For type: 'array' — validate each element against a nested schema. + * + * - `shape`: a flat SchemaField[] applied to every element (all elements + * the same shape). + * - `variants` + `discriminator`: discriminated union. Each element is + * routed by the value of `discriminator` (a field name on the element) + * to the matching variant schema. **Unknown discriminator values are + * allowed and skipped** — we want forward-compat for additive API + * changes (e.g. Anthropic adds a new content block type). Only *missing* + * discriminators or *wrong-typed* known variants trigger drift. + */ + items?: { + shape?: SchemaField[]; + discriminator?: string; + variants?: Record; + }; +} + +/** + * Walk a dot-path into an object. Returns undefined if any segment is missing. + * Does NOT distinguish "path missing" from "path present but undefined" — + * callers should treat both as missing. + */ +function getPath(obj: unknown, path: string): unknown { + const segments = path.split('.'); + let current: unknown = obj; + + for (const segment of segments) { + if (current == null || typeof current !== 'object') { + return undefined; + } + current = (current as Record)[segment]; + } + + return current; +} + +/** + * Describe the actual runtime type of a value for error messages. + * Separates null / undefined / array from generic object. + */ +function describeType(value: unknown): string { + if (value === null) return 'null'; + if (value === undefined) return 'undefined'; + if (Array.isArray(value)) return 'array'; + return typeof value; +} + +function matchesType(value: unknown, type: SchemaFieldType): boolean { + switch (type) { + case 'string': + return typeof value === 'string'; + case 'number': + return typeof value === 'number' && Number.isFinite(value); + case 'boolean': + return typeof value === 'boolean'; + case 'array': + return Array.isArray(value); + case 'object': + return value !== null && typeof value === 'object' && !Array.isArray(value); + case 'string-or-null': + return value === null || typeof value === 'string'; + } +} + +/** + * Validate a single object against a flat SchemaField list, prefixing any + * error path with `pathPrefix` so nested element errors read like + * `content[2].id` instead of bare `id`. + */ +function validateFields( + provider: string, + data: unknown, + fields: SchemaField[], + pathPrefix: string +): void { + if (data == null || typeof data !== 'object') { + throw new SchemaDriftError( + provider, + pathPrefix || '$root', + 'object', + describeType(data) + ); + } + + for (const field of fields) { + const fullPath = pathPrefix ? `${pathPrefix}.${field.path}` : field.path; + const value = getPath(data, field.path); + + if (value === undefined) { + if (field.optional) continue; + throw new SchemaDriftError(provider, fullPath, field.type, 'undefined'); + } + + if (!matchesType(value, field.type)) { + throw new SchemaDriftError(provider, fullPath, field.type, describeType(value)); + } + + if (field.type === 'array' && field.items) { + validateArrayItems(provider, value as unknown[], field.items, fullPath); + } + } +} + +/** + * Validate the elements of an array against either a flat `shape` or a + * discriminated `variants` map. Unknown discriminator values are + * forward-compatible and skipped. + */ +function validateArrayItems( + provider: string, + items: unknown[], + spec: NonNullable, + arrayPath: string +): void { + for (let i = 0; i < items.length; i++) { + const element = items[i]; + const elementPath = `${arrayPath}[${i}]`; + + if (element == null || typeof element !== 'object') { + throw new SchemaDriftError(provider, elementPath, 'object', describeType(element)); + } + + if (spec.discriminator && spec.variants) { + const disc = (element as Record)[spec.discriminator]; + if (typeof disc !== 'string') { + throw new SchemaDriftError( + provider, + `${elementPath}.${spec.discriminator}`, + 'string', + describeType(disc) + ); + } + const variantFields = spec.variants[disc]; + // Unknown discriminator value — forward-compat. Skip validation + // rather than reject, so adding a new block type upstream doesn't + // break every consumer on the next deploy. + if (!variantFields) continue; + validateFields(provider, element, variantFields, elementPath); + continue; + } + + if (spec.shape) { + validateFields(provider, element, spec.shape, elementPath); + } + } +} + +/** + * Validate a response envelope against a minimal field schema. + * Throws SchemaDriftError on the first mismatch, with provider + path + + * expected/actual types surfaced for observability. + * + * We fail fast rather than collecting all errors: the first drift is enough + * to trigger fallback, and walking the whole schema when we're already + * broken wastes budget. + */ +export function validateSchema( + provider: string, + data: unknown, + fields: SchemaField[] +): void { + validateFields(provider, data, fields, ''); +}