From 0095c158f9e66f025f196a4ae7f31cc9fad6ebe2 Mon Sep 17 00:00:00 2001 From: Codebeast Date: Sun, 12 Apr 2026 14:02:31 -0500 Subject: [PATCH 1/3] feat(schema-drift): add response envelope validation for Anthropic + factory fallback Defense-in-depth against silent upstream API deprecations. Adds SchemaDriftError, a zero-dep path-based validator, and wires the Anthropic provider as the template. Drift is non-retryable but fallback-eligible - factory routes to the next healthy provider and fires onSchemaDrift hook for observability. Slice 1 of #39. Remaining providers and schema canary land in follow-ups. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/__tests__/schema-drift.test.ts | 354 +++++++++++++++++++++++++++++ src/errors.ts | 27 +++ src/factory.ts | 24 +- src/index.ts | 6 + src/providers/anthropic.ts | 21 +- src/utils/hooks.ts | 13 +- src/utils/schema-validator.ts | 145 ++++++++++++ 7 files changed, 586 insertions(+), 4 deletions(-) create mode 100644 src/__tests__/schema-drift.test.ts create mode 100644 src/utils/schema-validator.ts diff --git a/src/__tests__/schema-drift.test.ts b/src/__tests__/schema-drift.test.ts new file mode 100644 index 0000000..a4184ef --- /dev/null +++ b/src/__tests__/schema-drift.test.ts @@ -0,0 +1,354 @@ +/** + * Schema Drift Detection Tests + * + * Covers the response-envelope schema validator, Anthropic provider drift + * detection at the parser boundary, factory fallback routing on drift, + * and onSchemaDrift observability hook firing. + * + * Slice 1 of #39 — Anthropic is the template provider. Follow-up tests for + * openai/groq/cerebras/cloudflare land with the respective wire-ups. + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { validateSchema, type SchemaField } from '../utils/schema-validator'; +import { SchemaDriftError } from '../errors'; +import { AnthropicProvider } from '../providers/anthropic'; +import { LLMProviderFactory } from '../factory'; +import { defaultCircuitBreakerManager } from '../utils/circuit-breaker'; +import { defaultExhaustionRegistry } from '../utils/exhaustion'; +import { defaultCostTracker } from '../utils/cost-tracker'; +import { defaultLatencyHistogram } from '../utils/latency-histogram'; +import type { ObservabilityHooks, SchemaDriftEvent } from '../utils/hooks'; + +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +// ── validateSchema primitives ──────────────────────────────────────────── + +describe('validateSchema', () => { + it('accepts a valid envelope with all required fields', () => { + const schema: SchemaField[] = [ + { path: 'id', type: 'string' }, + { path: 'usage.input_tokens', type: 'number' }, + ]; + expect(() => validateSchema('test', { id: 'x', usage: { input_tokens: 5 } }, schema)) + .not.toThrow(); + }); + + it('throws SchemaDriftError when a required path is missing', () => { + const schema: SchemaField[] = [{ path: 'usage.input_tokens', type: 'number' }]; + try { + validateSchema('test', { usage: {} }, schema); + expect.fail('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(SchemaDriftError); + expect((err as SchemaDriftError).path).toBe('usage.input_tokens'); + expect((err as SchemaDriftError).expected).toBe('number'); + expect((err as SchemaDriftError).actual).toBe('undefined'); + } + }); + + it('throws with correct actual type when field has wrong type', () => { + const schema: SchemaField[] = [{ path: 'content', type: 'array' }]; + try { + validateSchema('test', { content: 'oops-its-a-string-now' }, schema); + expect.fail('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(SchemaDriftError); + expect((err as SchemaDriftError).actual).toBe('string'); + } + }); + + it('distinguishes null from undefined from object', () => { + const schema: SchemaField[] = [{ path: 'usage', type: 'object' }]; + + try { + validateSchema('test', { usage: null }, schema); + expect.fail('should have thrown'); + } catch (err) { + expect((err as SchemaDriftError).actual).toBe('null'); + } + + try { + validateSchema('test', {}, schema); + expect.fail('should have thrown'); + } catch (err) { + expect((err as SchemaDriftError).actual).toBe('undefined'); + } + }); + + it('allows optional fields to be missing', () => { + const schema: SchemaField[] = [ + { path: 'id', type: 'string' }, + { path: 'stop_sequence', type: 'string', optional: true }, + ]; + expect(() => validateSchema('test', { id: 'x' }, schema)).not.toThrow(); + }); + + it('rejects NaN for number fields (Number.isFinite guard)', () => { + const schema: SchemaField[] = [{ path: 'tokens', type: 'number' }]; + expect(() => validateSchema('test', { tokens: NaN }, schema)) + .toThrow(SchemaDriftError); + }); + + it('rejects arrays when type is "object"', () => { + const schema: SchemaField[] = [{ path: 'usage', type: 'object' }]; + try { + validateSchema('test', { usage: [] }, schema); + expect.fail('should have thrown'); + } catch (err) { + expect((err as SchemaDriftError).actual).toBe('array'); + } + }); + + it('throws when root is not an object', () => { + expect(() => validateSchema('test', null, [])) + .toThrow(SchemaDriftError); + expect(() => validateSchema('test', 'a string', [])) + .toThrow(SchemaDriftError); + }); + + it('supports string-or-null type for nullable content fields', () => { + const schema: SchemaField[] = [{ path: 'content', type: 'string-or-null' }]; + expect(() => validateSchema('test', { content: null }, schema)).not.toThrow(); + expect(() => validateSchema('test', { content: 'hi' }, schema)).not.toThrow(); + expect(() => validateSchema('test', { content: 42 }, schema)) + .toThrow(SchemaDriftError); + }); + + it('fails fast on first drift (does not keep walking)', () => { + const schema: SchemaField[] = [ + { path: 'first', type: 'string' }, + { path: 'second', type: 'string' }, + ]; + try { + validateSchema('test', { first: 42, second: 99 }, schema); + expect.fail('should have thrown'); + } catch (err) { + // Should report the first failure, not the second + expect((err as SchemaDriftError).path).toBe('first'); + } + }); +}); + +// ── Anthropic provider drift detection ─────────────────────────────────── + +describe('AnthropicProvider response schema validation', () => { + let provider: AnthropicProvider; + + beforeEach(() => { + vi.clearAllMocks(); + defaultCircuitBreakerManager.resetAll(); + provider = new AnthropicProvider({ apiKey: 'test-key', maxRetries: 0 }); + }); + + const validAnthropicResponse = { + id: 'msg_1', + type: 'message', + role: 'assistant', + content: [{ type: 'text', text: 'hello' }], + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 5 }, + }; + + it('passes through a well-formed Anthropic response', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => validAnthropicResponse, + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const res = await provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + }); + + expect(res.content).toBe('hello'); + expect(res.usage.inputTokens).toBe(10); + }); + + it('throws SchemaDriftError when usage.input_tokens is renamed', async () => { + // Simulated drift: Anthropic silently renames input_tokens → prompt_tokens + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + ...validAnthropicResponse, + usage: { prompt_tokens: 10, output_tokens: 5 }, + }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + name: 'LLMProviderError', + code: 'SCHEMA_DRIFT', + provider: 'anthropic', + path: 'usage.input_tokens', + }); + }); + + it('throws SchemaDriftError when content field is removed', async () => { + const { content: _content, ...rest } = validAnthropicResponse; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => rest, + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ code: 'SCHEMA_DRIFT', path: 'content' }); + }); + + it('throws SchemaDriftError when content type changes from array to string', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ ...validAnthropicResponse, content: 'flat text instead of blocks' }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + path: 'content', + expected: 'array', + actual: 'string', + }); + }); + + it('is non-retryable so retries do not burn budget on drift', async () => { + // maxRetries on the provider is 0, but even with retries enabled the + // retry manager should skip non-retryable errors. Assert via a single + // failing response: if SchemaDriftError were retryable, fetch would be + // called more than once. + const retryProvider = new AnthropicProvider({ apiKey: 'test-key', maxRetries: 3 }); + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ ...validAnthropicResponse, usage: {} }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(retryProvider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ code: 'SCHEMA_DRIFT' }); + + expect(mockFetch).toHaveBeenCalledTimes(1); + }); +}); + +// ── Factory fallback on SchemaDriftError ───────────────────────────────── + +describe('LLMProviderFactory schema drift fallback', () => { + beforeEach(() => { + vi.clearAllMocks(); + defaultCircuitBreakerManager.resetAll(); + defaultExhaustionRegistry.reset(); + defaultCostTracker.reset(); + defaultLatencyHistogram.reset(); + }); + + const validOpenAIResponse = { + id: 'chatcmpl-1', + model: 'gpt-4o', + choices: [{ + index: 0, + message: { role: 'assistant', content: 'from openai' }, + finish_reason: 'stop', + }], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }; + + it('falls over to another provider when the primary drifts', async () => { + // First call = anthropic with missing content field (drift) + // Second call = openai with valid shape (success) + mockFetch + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: 'msg_1', + type: 'message', + role: 'assistant', + // content intentionally missing — drift + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 5 }, + }), + headers: new Headers({ 'content-type': 'application/json' }), + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => validOpenAIResponse, + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const factory = new LLMProviderFactory({ + anthropic: { apiKey: 'test-anthropic', maxRetries: 0 }, + openai: { apiKey: 'test-openai', maxRetries: 0 }, + preferredProvider: 'anthropic', + }); + + const response = await factory.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + }); + + expect(response.provider).toBe('openai'); + expect(response.content).toBe('from openai'); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('fires onSchemaDrift hook with path, expected, actual', async () => { + const driftEvents: SchemaDriftEvent[] = []; + const hooks: ObservabilityHooks = { + onSchemaDrift: (e) => driftEvents.push(e), + }; + + mockFetch + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: 'msg_1', + type: 'message', + role: 'assistant', + content: [], + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { prompt_tokens: 10 }, // wrong field name — drift on usage.input_tokens + }), + headers: new Headers({ 'content-type': 'application/json' }), + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => validOpenAIResponse, + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const factory = new LLMProviderFactory({ + anthropic: { apiKey: 'test-anthropic', maxRetries: 0 }, + openai: { apiKey: 'test-openai', maxRetries: 0 }, + preferredProvider: 'anthropic', + hooks, + }); + + await factory.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + requestId: 'req-drift-1', + }); + + expect(driftEvents).toHaveLength(1); + expect(driftEvents[0]).toMatchObject({ + provider: 'anthropic', + requestId: 'req-drift-1', + path: 'usage.input_tokens', + expected: 'number', + actual: 'undefined', + }); + expect(driftEvents[0].timestamp).toBeGreaterThan(0); + }); +}); diff --git a/src/errors.ts b/src/errors.ts index 85640df..32711c9 100755 --- a/src/errors.ts +++ b/src/errors.ts @@ -125,6 +125,33 @@ export class ToolLoopAbortedError extends LLMProviderError { super(message, 'TOOL_LOOP_ABORTED', provider, false, 400); } } + +/** + * Thrown when a provider's response envelope fails runtime schema validation. + * + * Indicates the upstream API silently changed shape - a field was renamed, + * removed, or had its type changed. Non-retryable: retrying hits the same + * broken shape. The factory treats this as fallback-eligible so traffic + * routes to a healthy provider while the drift is investigated. + */ +export class SchemaDriftError extends LLMProviderError { + path: string; + expected: string; + actual: string; + + constructor(provider: string, path: string, expected: string, actual: string) { + super( + `Response schema drift at ${path}: expected ${expected}, got ${actual}`, + 'SCHEMA_DRIFT', + provider, + false, + 502 + ); + this.path = path; + this.expected = expected; + this.actual = actual; + } +} /** * Error factory for creating provider-specific errors from HTTP responses diff --git a/src/factory.ts b/src/factory.ts index 48498d9..714eba6 100755 --- a/src/factory.ts +++ b/src/factory.ts @@ -51,6 +51,7 @@ import { AuthenticationError, RateLimitError, QuotaExceededError, + SchemaDriftError, ToolLoopAbortedError, ToolLoopLimitError, } from './errors'; @@ -274,6 +275,20 @@ export class LLMProviderFactory { }); } + // Schema drift — the upstream API silently changed shape. Surface + // structured telemetry so oncall sees the drift before it cascades. + if (err instanceof SchemaDriftError) { + this.hooks.onSchemaDrift?.({ + provider: providerName, + model: request.model, + requestId: request.requestId, + path: err.path, + expected: err.expected, + actual: err.actual, + timestamp: Date.now(), + }); + } + const fallbackDecision = this.getFallbackDecision(error as Error); if (!fallbackDecision.shouldFallback) { throw error; @@ -709,10 +724,17 @@ export class LLMProviderFactory { return { shouldFallback: true }; } + // Schema drift: provider's response shape changed. Retry won't help; + // only another provider can. Defense against silent API deprecations. + if (error instanceof SchemaDriftError) { + return { shouldFallback: true }; + } + if (error instanceof LLMProviderError) { if (error.code === 'SERVER_ERROR' || error.code === 'NETWORK_ERROR' || - error.code === 'TIMEOUT') { + error.code === 'TIMEOUT' || + error.code === 'SCHEMA_DRIFT') { return { shouldFallback: true }; } } diff --git a/src/index.ts b/src/index.ts index fd06fe8..a2268ed 100755 --- a/src/index.ts +++ b/src/index.ts @@ -99,10 +99,15 @@ export { TokenLimitError, ConfigurationError, CircuitBreakerOpenError, + SchemaDriftError, ToolLoopLimitError, ToolLoopAbortedError, LLMErrorFactory } from './errors'; + +// Schema validator (for custom provider authors) +export { validateSchema } from './utils/schema-validator'; +export type { SchemaField, SchemaFieldType } from './utils/schema-validator'; // Image generation export { ImageProvider, normalizeAiResponse } from './image/index'; @@ -128,6 +133,7 @@ export type { QuotaCheckEvent, QuotaDeniedEvent, ProviderBalanceEvent, + SchemaDriftEvent, } from './utils/hooks'; // Exhaustion registry diff --git a/src/providers/anthropic.ts b/src/providers/anthropic.ts index e5308a3..0876373 100755 --- a/src/providers/anthropic.ts +++ b/src/providers/anthropic.ts @@ -19,6 +19,22 @@ import { ModelNotFoundError, RateLimitError } from '../errors'; +import { validateSchema, type SchemaField } from '../utils/schema-validator'; + +/** + * Minimum envelope fields the Anthropic response parser reads. Changing this + * list is a contract change — keep it in sync with formatResponse below. + * Optional fields (e.g. stop_sequence) are not listed. + */ +const ANTHROPIC_RESPONSE_SCHEMA: SchemaField[] = [ + { path: 'id', type: 'string' }, + { path: 'content', type: 'array' }, + { path: 'model', type: 'string' }, + { path: 'stop_reason', type: 'string' }, + { path: 'usage', type: 'object' }, + { path: 'usage.input_tokens', type: 'number' }, + { path: 'usage.output_tokens', type: 'number' }, +]; interface AnthropicContentBlock { type: 'text' | 'tool_use' | 'tool_result' | 'image'; @@ -132,8 +148,9 @@ export class AnthropicProvider extends BaseProvider { throw await LLMErrorFactory.fromFetchResponse('anthropic', httpResponse); } - const data: AnthropicResponse = await httpResponse.json(); - const formatted = this.formatResponse(data, Date.now() - startTime); + const data = await httpResponse.json() as unknown; + validateSchema('anthropic', data, ANTHROPIC_RESPONSE_SCHEMA); + const formatted = this.formatResponse(data as AnthropicResponse, Date.now() - startTime); // Restore the prefilled '{' consumed by the assistant turn, // but only if the response doesn't already start with one diff --git a/src/utils/hooks.ts b/src/utils/hooks.ts index 3eb96ac..d6f82da 100644 --- a/src/utils/hooks.ts +++ b/src/utils/hooks.ts @@ -112,6 +112,16 @@ export interface ProviderBalanceEvent { timestamp: number; } +export interface SchemaDriftEvent { + provider: string; + model?: string; + requestId?: string; + path: string; + expected: string; + actual: string; + timestamp: number; +} + // ── Hooks interface ────────────────────────────────────────────────────── export interface ObservabilityHooks { @@ -126,6 +136,7 @@ export interface ObservabilityHooks { onQuotaCheck?(event: QuotaCheckEvent): void; onQuotaDenied?(event: QuotaDeniedEvent): void; onProviderBalance?(event: ProviderBalanceEvent): void; + onSchemaDrift?(event: SchemaDriftEvent): void; } /** Silent hooks — default. */ @@ -140,7 +151,7 @@ export function composeHooks(...implementations: ObservabilityHooks[]): Observab 'onRequestStart', 'onRequestEnd', 'onRequestError', 'onRetry', 'onFallback', 'onCircuitStateChange', 'onQuotaExhausted', 'onBudgetThreshold', 'onQuotaCheck', - 'onQuotaDenied', 'onProviderBalance', + 'onQuotaDenied', 'onProviderBalance', 'onSchemaDrift', ] as const; const composed: ObservabilityHooks = {}; diff --git a/src/utils/schema-validator.ts b/src/utils/schema-validator.ts new file mode 100644 index 0000000..1f4c596 --- /dev/null +++ b/src/utils/schema-validator.ts @@ -0,0 +1,145 @@ +/** + * Response Envelope Schema Validator + * + * Zero-dependency runtime validator for provider response shapes. Used at the + * provider boundary to detect when an upstream API silently changes its + * response envelope — the classic "field renamed at 2am, parser throws at + * 3am" failure mode. + * + * Philosophy: validate the *minimum* fields each provider's parser actually + * reads. Don't re-type the full upstream schema — that creates brittle + * over-specification that churns every time the provider adds an optional + * field. Only the fields we touch matter. + * + * Usage: + * validateSchema('anthropic', data, [ + * { path: 'content', type: 'array' }, + * { path: 'usage.input_tokens', type: 'number' }, + * { path: 'usage.output_tokens', type: 'number' }, + * { path: 'model', type: 'string' }, + * ]); + * + * On first failure, throws SchemaDriftError. The caller (typically the + * factory) catches it, fires the onSchemaDrift hook, and falls over to + * another provider. + */ + +import { SchemaDriftError } from '../errors'; + +export type SchemaFieldType = + | 'string' + | 'number' + | 'boolean' + | 'array' + | 'object' + | 'string-or-null'; // anthropic/openai sometimes null content fields + +export interface SchemaField { + /** + * Dot-separated path into the response object. Array indices not supported + * because we validate shape, not contents — if you care about element + * shape, validate the array type here and the elements at their own + * parse site. + */ + path: string; + type: SchemaFieldType; + /** + * If true, missing paths are allowed and skipped. Useful for fields that + * are genuinely optional (e.g. stop_sequence on Anthropic). + */ + optional?: boolean; +} + +/** + * Walk a dot-path into an object. Returns undefined if any segment is missing. + * Does NOT distinguish "path missing" from "path present but undefined" — + * callers should treat both as missing. + */ +function getPath(obj: unknown, path: string): unknown { + const segments = path.split('.'); + let current: unknown = obj; + + for (const segment of segments) { + if (current == null || typeof current !== 'object') { + return undefined; + } + current = (current as Record)[segment]; + } + + return current; +} + +/** + * Describe the actual runtime type of a value for error messages. + * Separates null / undefined / array from generic object. + */ +function describeType(value: unknown): string { + if (value === null) return 'null'; + if (value === undefined) return 'undefined'; + if (Array.isArray(value)) return 'array'; + return typeof value; +} + +function matchesType(value: unknown, type: SchemaFieldType): boolean { + switch (type) { + case 'string': + return typeof value === 'string'; + case 'number': + return typeof value === 'number' && Number.isFinite(value); + case 'boolean': + return typeof value === 'boolean'; + case 'array': + return Array.isArray(value); + case 'object': + return value !== null && typeof value === 'object' && !Array.isArray(value); + case 'string-or-null': + return value === null || typeof value === 'string'; + } +} + +/** + * Validate a response envelope against a minimal field schema. + * Throws SchemaDriftError on the first mismatch, with provider + path + + * expected/actual types surfaced for observability. + * + * We fail fast rather than collecting all errors: the first drift is enough + * to trigger fallback, and walking the whole schema when we're already + * broken wastes budget. + */ +export function validateSchema( + provider: string, + data: unknown, + fields: SchemaField[] +): void { + if (data == null || typeof data !== 'object') { + throw new SchemaDriftError( + provider, + '$root', + 'object', + describeType(data) + ); + } + + for (const field of fields) { + const value = getPath(data, field.path); + + if (value === undefined) { + if (field.optional) continue; + throw new SchemaDriftError( + provider, + field.path, + field.type, + 'undefined' + ); + } + + if (!matchesType(value, field.type)) { + throw new SchemaDriftError( + provider, + field.path, + field.type, + describeType(value) + ); + } + } +} From b6ef2af50a8ba4d22e0bf38d9240944242afca52 Mon Sep 17 00:00:00 2001 From: Codebeast Date: Sun, 12 Apr 2026 14:45:36 -0500 Subject: [PATCH 2/3] fix(schema-drift): address PR review findings - nested shapes, 422, tests Responds to CodeBeast review of #40. Closes H-2 (#42) in-PR, fixes M-3, adds missing tests for M-1 and M-2 and security message surface. - Extend validator with items/discriminator for discriminated-union array element validation. Unknown variants are forward-compatible (skipped, not rejected) so additive upstream changes don't break consumers on the next deploy. - Rewire ANTHROPIC_RESPONSE_SCHEMA to validate content[] elements: text blocks need .text (string), tool_use blocks need .id, .name, .input. stop_sequence now marked optional. - Change SchemaDriftError statusCode 502 -> 422. Upstream returned HTTP 200 with a wrong body; Unprocessable Entity is accurate, Bad Gateway was misleading (M-3). - Replace tool.id! / tool.name! non-null assertions with explicit casts and a comment documenting that the schema validator guarantees the invariant (L-4). - Add tests: circuit breaker opens after repeated drift (M-1), SchemaDriftError propagates correctly at end of single-provider chain with hook fired (M-2), error message/serialized form never contains actual response values only type names (security), and six new H-2 tests covering tool_use missing/wrong fields, text block missing .text, bare-string element, missing discriminator, forward-compat unknown block type, and optional stop_sequence. 212 tests passing (11 new). H-1 (streaming) tracked separately in #41. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/__tests__/schema-drift.test.ts | 304 +++++++++++++++++++++++++++++ src/errors.ts | 2 +- src/providers/anthropic.ts | 33 +++- src/utils/schema-validator.ts | 118 ++++++++--- 4 files changed, 426 insertions(+), 31 deletions(-) diff --git a/src/__tests__/schema-drift.test.ts b/src/__tests__/schema-drift.test.ts index a4184ef..7be563c 100644 --- a/src/__tests__/schema-drift.test.ts +++ b/src/__tests__/schema-drift.test.ts @@ -351,4 +351,308 @@ describe('LLMProviderFactory schema drift fallback', () => { }); expect(driftEvents[0].timestamp).toBeGreaterThan(0); }); + + it('propagates SchemaDriftError at end of chain when no fallback is available (M-2)', async () => { + // Single-provider chain: if anthropic drifts, there is nowhere to fall + // over to. The factory must surface the SchemaDriftError to the caller + // without wrapping it as generic ALL_PROVIDERS_FAILED, and the hook must + // still fire so ops sees the drift. + // + // (Multi-provider end-of-chain coverage lands with the follow-up PR + // that wires schema validation into the other providers — until then, + // the other providers' parsers throw TypeError on drift, which isn't + // this test's concern.) + const driftEvents: SchemaDriftEvent[] = []; + const hooks: ObservabilityHooks = { + onSchemaDrift: (e) => driftEvents.push(e), + }; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: 'msg_1', type: 'message', role: 'assistant', + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 5 }, + // content intentionally missing — drift + }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const factory = new LLMProviderFactory({ + anthropic: { apiKey: 'test-anthropic', maxRetries: 0 }, + preferredProvider: 'anthropic', + hooks, + }); + + await expect(factory.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + requestId: 'req-eoc-1', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + provider: 'anthropic', + path: 'content', + }); + + expect(driftEvents).toHaveLength(1); + expect(driftEvents[0]).toMatchObject({ + provider: 'anthropic', + requestId: 'req-eoc-1', + }); + }); +}); + +// ── Circuit breaker + drift interaction ────────────────────────────────── + +describe('SchemaDriftError and circuit breaker', () => { + beforeEach(() => { + vi.clearAllMocks(); + defaultCircuitBreakerManager.resetAll(); + }); + + it('repeated drift trips the circuit breaker open (M-1)', async () => { + const provider = new AnthropicProvider({ apiKey: 'test-key', maxRetries: 0 }); + + // Return drifted response every call — missing usage.input_tokens + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + id: 'msg_1', type: 'message', role: 'assistant', + content: [], + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { output_tokens: 5 }, + }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + // Hit the provider until something opens the breaker. Default threshold + // is low enough that this is bounded; 10 tries is plenty of headroom. + for (let i = 0; i < 10; i++) { + try { + await provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + }); + } catch { + // Expected — drift or circuit-open + } + } + + const breaker = defaultCircuitBreakerManager.getBreaker('anthropic'); + expect(breaker.getState().state).toBe('OPEN'); + }); +}); + +// ── Security: no value leakage in error surface ────────────────────────── + +describe('SchemaDriftError message surface (security)', () => { + it('error message and hook payload contain only path + type names, never values', async () => { + vi.clearAllMocks(); + defaultCircuitBreakerManager.resetAll(); + const provider = new AnthropicProvider({ apiKey: 'test-key', maxRetries: 0 }); + + // Drifted response contains a sensitive-looking value. If it leaks into + // the error message we have a telemetry PII problem. + const SECRET_VALUE = 'sk-verysecret-api-key-shouldnotleak'; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: SECRET_VALUE, // intentionally put a "secret" where we expect a string + // content missing — triggers drift on content path instead of id + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 5 }, + }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + try { + await provider.generateResponse({ + messages: [{ role: 'user', content: SECRET_VALUE }], + model: 'claude-3-haiku-20240307', + }); + expect.fail('should have thrown'); + } catch (err) { + const message = (err as Error).message; + const serialized = JSON.stringify(err, Object.getOwnPropertyNames(err)); + + // The path, expected type, actual type should all be present + expect(message).toContain('content'); + expect(message).toContain('array'); + expect(message).toContain('undefined'); + + // The secret value should NEVER be in the error surface + expect(message).not.toContain(SECRET_VALUE); + expect(serialized).not.toContain(SECRET_VALUE); + } + }); +}); + +// ── H-2: nested content block validation ───────────────────────────────── + +describe('Anthropic nested content-block validation (H-2 / #42)', () => { + let provider: AnthropicProvider; + + beforeEach(() => { + vi.clearAllMocks(); + defaultCircuitBreakerManager.resetAll(); + provider = new AnthropicProvider({ apiKey: 'test-key', maxRetries: 0 }); + }); + + const envelope = (content: unknown[]) => ({ + id: 'msg_1', + type: 'message', + role: 'assistant', + content, + model: 'claude-3-haiku-20240307', + stop_reason: 'end_turn', + usage: { input_tokens: 10, output_tokens: 5 }, + }); + + it('accepts a valid tool_use block', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([ + { type: 'tool_use', id: 'toolu_1', name: 'search', input: { q: 'test' } }, + ]), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const res = await provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + }); + expect(res.toolCalls).toHaveLength(1); + }); + + it('detects tool_use block with missing id', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([ + { type: 'tool_use', name: 'search', input: { q: 'test' } }, + ]), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + path: 'content[0].id', + expected: 'string', + actual: 'undefined', + }); + }); + + it('detects tool_use block with wrong-typed input', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([ + { type: 'tool_use', id: 'toolu_1', name: 'search', input: 'should-be-object' }, + ]), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + path: 'content[0].input', + expected: 'object', + actual: 'string', + }); + }); + + it('detects text block with missing .text field', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([ + { type: 'text' }, // .text renamed/removed + ]), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + path: 'content[0].text', + }); + }); + + it('detects element that is not an object', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope(['a bare string, not a block']), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + path: 'content[0]', + expected: 'object', + actual: 'string', + }); + }); + + it('detects missing discriminator (type field)', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([ + { text: 'no type field' }, + ]), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + await expect(provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + })).rejects.toMatchObject({ + code: 'SCHEMA_DRIFT', + path: 'content[0].type', + expected: 'string', + }); + }); + + it('accepts unknown block types (forward-compat)', async () => { + // Anthropic adds a new block type 'reasoning' — we should pass without + // error so the next deploy doesn't break on additive changes. + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([ + { type: 'text', text: 'hello' }, + { type: 'reasoning', thoughts: 'internal thinking trace', confidence: 0.9 }, + ]), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const res = await provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + }); + // Text content extracted; unknown block silently ignored by filter + expect(res.content).toBe('hello'); + }); + + it('accepts response without stop_sequence (optional field)', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => envelope([{ type: 'text', text: 'hi' }]), + // Note: no stop_sequence key — it's optional + headers: new Headers({ 'content-type': 'application/json' }), + }); + + const res = await provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + }); + expect(res.content).toBe('hi'); + }); }); diff --git a/src/errors.ts b/src/errors.ts index 32711c9..5e72ac2 100755 --- a/src/errors.ts +++ b/src/errors.ts @@ -145,7 +145,7 @@ export class SchemaDriftError extends LLMProviderError { 'SCHEMA_DRIFT', provider, false, - 502 + 422 ); this.path = path; this.expected = expected; diff --git a/src/providers/anthropic.ts b/src/providers/anthropic.ts index 0876373..2565b5c 100755 --- a/src/providers/anthropic.ts +++ b/src/providers/anthropic.ts @@ -24,13 +24,33 @@ import { validateSchema, type SchemaField } from '../utils/schema-validator'; /** * Minimum envelope fields the Anthropic response parser reads. Changing this * list is a contract change — keep it in sync with formatResponse below. - * Optional fields (e.g. stop_sequence) are not listed. + * + * Content blocks are validated as a discriminated union on `type`. Unknown + * block types are forward-compatible (skipped) so Anthropic adding a new + * block variant doesn't instantly break the fallback routing on every deploy. */ const ANTHROPIC_RESPONSE_SCHEMA: SchemaField[] = [ { path: 'id', type: 'string' }, - { path: 'content', type: 'array' }, + { + path: 'content', + type: 'array', + items: { + discriminator: 'type', + variants: { + text: [ + { path: 'text', type: 'string' }, + ], + tool_use: [ + { path: 'id', type: 'string' }, + { path: 'name', type: 'string' }, + { path: 'input', type: 'object' }, + ], + }, + }, + }, { path: 'model', type: 'string' }, { path: 'stop_reason', type: 'string' }, + { path: 'stop_sequence', type: 'string', optional: true }, { path: 'usage', type: 'object' }, { path: 'usage.input_tokens', type: 'number' }, { path: 'usage.output_tokens', type: 'number' }, @@ -514,14 +534,17 @@ export class AnthropicProvider extends BaseProvider { } }; - // Extract tool calls if present (validated at provider boundary) + // Extract tool calls if present. id/name/input are guaranteed non-null + // for tool_use blocks by ANTHROPIC_RESPONSE_SCHEMA's discriminated-union + // validation — if they were missing, validateSchema would have thrown + // SchemaDriftError before we got here. const toolUses = data.content.filter(block => block.type === 'tool_use'); if (toolUses.length > 0) { const raw: ToolCall[] = toolUses.map(tool => ({ - id: tool.id!, + id: tool.id as string, type: 'function' as const, function: { - name: tool.name!, + name: tool.name as string, arguments: JSON.stringify(tool.input) } })); diff --git a/src/utils/schema-validator.ts b/src/utils/schema-validator.ts index 1f4c596..90d9203 100644 --- a/src/utils/schema-validator.ts +++ b/src/utils/schema-validator.ts @@ -37,9 +37,7 @@ export type SchemaFieldType = export interface SchemaField { /** * Dot-separated path into the response object. Array indices not supported - * because we validate shape, not contents — if you care about element - * shape, validate the array type here and the elements at their own - * parse site. + * in the path itself — use `items` to validate array element shapes. */ path: string; type: SchemaFieldType; @@ -48,6 +46,23 @@ export interface SchemaField { * are genuinely optional (e.g. stop_sequence on Anthropic). */ optional?: boolean; + /** + * For type: 'array' — validate each element against a nested schema. + * + * - `shape`: a flat SchemaField[] applied to every element (all elements + * the same shape). + * - `variants` + `discriminator`: discriminated union. Each element is + * routed by the value of `discriminator` (a field name on the element) + * to the matching variant schema. **Unknown discriminator values are + * allowed and skipped** — we want forward-compat for additive API + * changes (e.g. Anthropic adds a new content block type). Only *missing* + * discriminators or *wrong-typed* known variants trigger drift. + */ + items?: { + shape?: SchemaField[]; + discriminator?: string; + variants?: Record; + }; } /** @@ -98,48 +113,101 @@ function matchesType(value: unknown, type: SchemaFieldType): boolean { } /** - * Validate a response envelope against a minimal field schema. - * Throws SchemaDriftError on the first mismatch, with provider + path + - * expected/actual types surfaced for observability. - * - * We fail fast rather than collecting all errors: the first drift is enough - * to trigger fallback, and walking the whole schema when we're already - * broken wastes budget. + * Validate a single object against a flat SchemaField list, prefixing any + * error path with `pathPrefix` so nested element errors read like + * `content[2].id` instead of bare `id`. */ -export function validateSchema( +function validateFields( provider: string, data: unknown, - fields: SchemaField[] + fields: SchemaField[], + pathPrefix: string ): void { if (data == null || typeof data !== 'object') { throw new SchemaDriftError( provider, - '$root', + pathPrefix || '$root', 'object', describeType(data) ); } for (const field of fields) { + const fullPath = pathPrefix ? `${pathPrefix}.${field.path}` : field.path; const value = getPath(data, field.path); if (value === undefined) { if (field.optional) continue; - throw new SchemaDriftError( - provider, - field.path, - field.type, - 'undefined' - ); + throw new SchemaDriftError(provider, fullPath, field.type, 'undefined'); } if (!matchesType(value, field.type)) { - throw new SchemaDriftError( - provider, - field.path, - field.type, - describeType(value) - ); + throw new SchemaDriftError(provider, fullPath, field.type, describeType(value)); + } + + if (field.type === 'array' && field.items) { + validateArrayItems(provider, value as unknown[], field.items, fullPath); + } + } +} + +/** + * Validate the elements of an array against either a flat `shape` or a + * discriminated `variants` map. Unknown discriminator values are + * forward-compatible and skipped. + */ +function validateArrayItems( + provider: string, + items: unknown[], + spec: NonNullable, + arrayPath: string +): void { + for (let i = 0; i < items.length; i++) { + const element = items[i]; + const elementPath = `${arrayPath}[${i}]`; + + if (element == null || typeof element !== 'object') { + throw new SchemaDriftError(provider, elementPath, 'object', describeType(element)); + } + + if (spec.discriminator && spec.variants) { + const disc = (element as Record)[spec.discriminator]; + if (typeof disc !== 'string') { + throw new SchemaDriftError( + provider, + `${elementPath}.${spec.discriminator}`, + 'string', + describeType(disc) + ); + } + const variantFields = spec.variants[disc]; + // Unknown discriminator value — forward-compat. Skip validation + // rather than reject, so adding a new block type upstream doesn't + // break every consumer on the next deploy. + if (!variantFields) continue; + validateFields(provider, element, variantFields, elementPath); + continue; + } + + if (spec.shape) { + validateFields(provider, element, spec.shape, elementPath); } } } + +/** + * Validate a response envelope against a minimal field schema. + * Throws SchemaDriftError on the first mismatch, with provider + path + + * expected/actual types surfaced for observability. + * + * We fail fast rather than collecting all errors: the first drift is enough + * to trigger fallback, and walking the whole schema when we're already + * broken wastes budget. + */ +export function validateSchema( + provider: string, + data: unknown, + fields: SchemaField[] +): void { + validateFields(provider, data, fields, ''); +} From dd15a9948369e6c11675fe9e97c298366ff11684 Mon Sep 17 00:00:00 2001 From: Codebeast Date: Sun, 12 Apr 2026 14:53:11 -0500 Subject: [PATCH 3/3] test(schema-drift): stub Math.random in M-1 breaker test The default circuit breaker uses a graduated degradation curve with a probabilistic rejection gate at DEGRADED levels. With 10 attempts and the default [1.0, 0.9, 0.7, 0.4, 0.1] curve, reaching OPEN requires getting lucky at each step - expected value ~16 attempts, not 10. Locally passed by luck, CI failed on all three Node versions when the random sequence bounced attempts off the gate without advancing. Stub Math.random to 0 inside the test so the gate always admits the request through to fn() where the drift advances a real failure, and restore the stub in finally{}. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/__tests__/schema-drift.test.ts | 63 ++++++++++++++++++------------ 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/src/__tests__/schema-drift.test.ts b/src/__tests__/schema-drift.test.ts index 7be563c..ca2a9bf 100644 --- a/src/__tests__/schema-drift.test.ts +++ b/src/__tests__/schema-drift.test.ts @@ -411,36 +411,49 @@ describe('SchemaDriftError and circuit breaker', () => { }); it('repeated drift trips the circuit breaker open (M-1)', async () => { - const provider = new AnthropicProvider({ apiKey: 'test-key', maxRetries: 0 }); + // The default breaker uses a graduated degradation curve with a + // probabilistic rejection gate at DEGRADED levels. To get deterministic + // progression all the way to OPEN, stub Math.random so the gate always + // admits the request through to fn() - where the drift throws and the + // breaker advances one failure. Without this stub, the test is racy: + // locally it would pass by luck, CI would fail when the random sequence + // bounced attempts off the degradation gate without advancing. + const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0); - // Return drifted response every call — missing usage.input_tokens - mockFetch.mockResolvedValue({ - ok: true, - json: async () => ({ - id: 'msg_1', type: 'message', role: 'assistant', - content: [], - model: 'claude-3-haiku-20240307', - stop_reason: 'end_turn', - usage: { output_tokens: 5 }, - }), - headers: new Headers({ 'content-type': 'application/json' }), - }); + try { + const provider = new AnthropicProvider({ apiKey: 'test-key', maxRetries: 0 }); - // Hit the provider until something opens the breaker. Default threshold - // is low enough that this is bounded; 10 tries is plenty of headroom. - for (let i = 0; i < 10; i++) { - try { - await provider.generateResponse({ - messages: [{ role: 'user', content: 'hi' }], + // Return drifted response every call - missing usage.input_tokens + mockFetch.mockResolvedValue({ + ok: true, + json: async () => ({ + id: 'msg_1', type: 'message', role: 'assistant', + content: [], model: 'claude-3-haiku-20240307', - }); - } catch { - // Expected — drift or circuit-open + stop_reason: 'end_turn', + usage: { output_tokens: 5 }, + }), + headers: new Headers({ 'content-type': 'application/json' }), + }); + + // Default failureThreshold = 5. 10 attempts is more than enough with + // the probabilistic gate pinned open. + for (let i = 0; i < 10; i++) { + try { + await provider.generateResponse({ + messages: [{ role: 'user', content: 'hi' }], + model: 'claude-3-haiku-20240307', + }); + } catch { + // Expected - drift or circuit-open + } } - } - const breaker = defaultCircuitBreakerManager.getBreaker('anthropic'); - expect(breaker.getState().state).toBe('OPEN'); + const breaker = defaultCircuitBreakerManager.getBreaker('anthropic'); + expect(breaker.getState().state).toBe('OPEN'); + } finally { + randomSpy.mockRestore(); + } }); });