From 054df0c5d2d1f9dfcdefd4e32adb93f44cc829b9 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 30 Apr 2026 10:53:08 -0700 Subject: [PATCH 01/11] feat: add source-metronome and destination-redis connectors Add two new connector packages: - **source-metronome**: Reads from the Metronome billing API (customers, contracts, products, rate cards, credit grants, invoices, entitlements). Supports cursor-based pagination, per-customer/per-contract fan-out, and webhook-driven live updates for credit balance and entitlement changes. - **destination-redis**: Writes synced data to Redis as individual SET keys (`{prefix}{stream}:{pk}`). Pipelined batch writes, per-stream failure tracking, SCAN-based teardown. Also: - Register both connectors in the engine (default-connectors.ts) - Add Redis service to compose.yml (port 56379) - Regenerate OpenAPI specs with new connector config schemas - Add e2e test script (scripts/e2e-metronome-redis.sh) Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- apps/engine/package.json | 2 + apps/engine/src/__generated__/openapi.d.ts | 46 +- apps/engine/src/__generated__/openapi.json | 113 ++++- apps/engine/src/lib/default-connectors.ts | 5 +- apps/service/src/__generated__/openapi.d.ts | 42 ++ apps/service/src/__generated__/openapi.json | 101 +++++ compose.yml | 10 + packages/destination-redis/package.json | 34 ++ packages/destination-redis/src/bin.ts | 6 + packages/destination-redis/src/index.test.ts | 87 ++++ packages/destination-redis/src/index.ts | 244 +++++++++++ .../destination-redis/src/integration.test.ts | 103 +++++ packages/destination-redis/src/logger.ts | 4 + packages/destination-redis/src/spec.ts | 24 ++ packages/destination-redis/tsconfig.json | 9 + packages/source-metronome/package.json | 33 ++ packages/source-metronome/src/bin.ts | 6 + packages/source-metronome/src/client.test.ts | 115 +++++ packages/source-metronome/src/client.ts | 128 ++++++ packages/source-metronome/src/index.test.ts | 243 +++++++++++ packages/source-metronome/src/index.ts | 406 ++++++++++++++++++ packages/source-metronome/src/logger.ts | 4 + packages/source-metronome/src/resources.ts | 194 +++++++++ packages/source-metronome/src/spec.ts | 48 +++ packages/source-metronome/src/webhook.ts | 97 +++++ packages/source-metronome/tsconfig.json | 9 + pnpm-lock.yaml | 111 +++++ scripts/e2e-metronome-redis.sh | 112 +++++ scripts/generate-openapi-specs.ts | 8 +- 29 files changed, 2339 insertions(+), 5 deletions(-) create mode 100644 packages/destination-redis/package.json create mode 100644 packages/destination-redis/src/bin.ts create mode 100644 packages/destination-redis/src/index.test.ts create mode 100644 packages/destination-redis/src/index.ts create mode 100644 packages/destination-redis/src/integration.test.ts create mode 100644 packages/destination-redis/src/logger.ts create mode 100644 packages/destination-redis/src/spec.ts create mode 100644 packages/destination-redis/tsconfig.json create mode 100644 packages/source-metronome/package.json create mode 100644 packages/source-metronome/src/bin.ts create mode 100644 packages/source-metronome/src/client.test.ts create mode 100644 packages/source-metronome/src/client.ts create mode 100644 packages/source-metronome/src/index.test.ts create mode 100644 packages/source-metronome/src/index.ts create mode 100644 packages/source-metronome/src/logger.ts create mode 100644 packages/source-metronome/src/resources.ts create mode 100644 packages/source-metronome/src/spec.ts create mode 100644 packages/source-metronome/src/webhook.ts create mode 100644 packages/source-metronome/tsconfig.json create mode 100755 scripts/e2e-metronome-redis.sh diff --git a/apps/engine/package.json b/apps/engine/package.json index 78ec30e3f..614dc9d2c 100644 --- a/apps/engine/package.json +++ b/apps/engine/package.json @@ -54,7 +54,9 @@ "@stripe/sync-hono-zod-openapi": "workspace:*", "@stripe/sync-logger": "workspace:*", "@stripe/sync-protocol": "workspace:*", + "@stripe/sync-source-metronome": "workspace:*", "@stripe/sync-source-stripe": "workspace:*", + "@stripe/sync-destination-redis": "workspace:*", "@stripe/sync-ts-cli": "workspace:*", "@stripe/sync-util-postgres": "workspace:*", "citty": "^0.1.6", diff --git a/apps/engine/src/__generated__/openapi.d.ts b/apps/engine/src/__generated__/openapi.d.ts index 1b37c41ec..e0dd0a446 100644 --- a/apps/engine/src/__generated__/openapi.d.ts +++ b/apps/engine/src/__generated__/openapi.d.ts @@ -291,6 +291,10 @@ export interface components { /** @constant */ type: "stripe"; stripe: components["schemas"]["SourceStripeConfig"]; + } | { + /** @constant */ + type: "metronome"; + metronome: components["schemas"]["SourceMetronomeConfig"]; }; SourceStripeConfig: { /** @description Stripe API key (sk_test_... or sk_live_...) */ @@ -328,6 +332,19 @@ export interface components { /** @description Override max requests per second (default: auto-derived from API key mode — 20 live, 10 test). */ rate_limit?: number; }; + SourceMetronomeConfig: { + /** @description Metronome API bearer token */ + api_key: string; + /** + * Format: uri + * @description Override the Metronome API base URL (default: https://api.metronome.com) + */ + base_url?: string; + /** @description Max requests per second (default: no limit) */ + rate_limit?: number; + /** @description Max records to fetch per stream (useful for testing) */ + backfill_limit?: number; + }; DestinationConfig: { /** @constant */ type: "postgres"; @@ -336,6 +353,10 @@ export interface components { /** @constant */ type: "google_sheets"; google_sheets: components["schemas"]["DestinationGoogleSheetsConfig"]; + } | { + /** @constant */ + type: "redis"; + redis: components["schemas"]["DestinationRedisConfig"]; }; DestinationPostgresConfig: { /** @description Postgres connection string */ @@ -396,6 +417,27 @@ export interface components { */ batch_size: number; }; + DestinationRedisConfig: { + /** @description Redis connection URL (redis://host:port) */ + url?: string; + /** @description Redis host (default: localhost) */ + host?: string; + /** @description Redis port (default: 6379) */ + port?: number; + /** @description Redis password */ + password?: string; + /** @description Redis database number (default: 0) */ + db?: number; + /** @description Enable TLS */ + tls?: boolean; + /** @description Prefix for all Redis keys (default: empty) */ + key_prefix?: string; + /** + * @description Records to buffer before flushing via pipeline + * @default 100 + */ + batch_size: number; + }; RecordMessage: { /** @description Who emitted this message: "source/{type}", "destination/{type}", or "engine". Set by the engine. */ _emitted_by?: string; @@ -649,11 +691,11 @@ export interface components { control: { /** @constant */ control_type: "source_config"; - source_config: components["schemas"]["SourceStripeConfig"]; + source_config: components["schemas"]["SourceStripeConfig"] | components["schemas"]["SourceMetronomeConfig"]; } | { /** @constant */ control_type: "destination_config"; - destination_config: components["schemas"]["DestinationPostgresConfig"] | components["schemas"]["DestinationGoogleSheetsConfig"]; + destination_config: components["schemas"]["DestinationPostgresConfig"] | components["schemas"]["DestinationGoogleSheetsConfig"] | components["schemas"]["DestinationRedisConfig"]; }; }; ProgressMessage: { diff --git a/apps/engine/src/__generated__/openapi.json b/apps/engine/src/__generated__/openapi.json index 805a6020e..5002c7531 100644 --- a/apps/engine/src/__generated__/openapi.json +++ b/apps/engine/src/__generated__/openapi.json @@ -969,6 +969,22 @@ "type", "stripe" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "metronome" + }, + "metronome": { + "$ref": "#/components/schemas/SourceMetronomeConfig" + } + }, + "required": [ + "type", + "metronome" + ] } ], "type": "object", @@ -1106,6 +1122,36 @@ ], "additionalProperties": false }, + "SourceMetronomeConfig": { + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Metronome API bearer token" + }, + "base_url": { + "type": "string", + "format": "uri", + "description": "Override the Metronome API base URL (default: https://api.metronome.com)" + }, + "rate_limit": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "description": "Max requests per second (default: no limit)" + }, + "backfill_limit": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "description": "Max records to fetch per stream (useful for testing)" + } + }, + "required": [ + "api_key" + ], + "additionalProperties": false + }, "DestinationConfig": { "oneOf": [ { @@ -1139,6 +1185,22 @@ "type", "google_sheets" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "redis" + }, + "redis": { + "$ref": "#/components/schemas/DestinationRedisConfig" + } + }, + "required": [ + "type", + "redis" + ] } ], "type": "object", @@ -1261,6 +1323,45 @@ ], "additionalProperties": false }, + "DestinationRedisConfig": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "Redis connection URL (redis://host:port)" + }, + "host": { + "type": "string", + "description": "Redis host (default: localhost)" + }, + "port": { + "type": "number", + "description": "Redis port (default: 6379)" + }, + "password": { + "type": "string", + "description": "Redis password" + }, + "db": { + "type": "number", + "description": "Redis database number (default: 0)" + }, + "tls": { + "type": "boolean", + "description": "Enable TLS" + }, + "key_prefix": { + "type": "string", + "description": "Prefix for all Redis keys (default: empty)" + }, + "batch_size": { + "default": 100, + "type": "number", + "description": "Records to buffer before flushing via pipeline" + } + }, + "additionalProperties": false + }, "RecordMessage": { "type": "object", "properties": { @@ -1812,7 +1913,14 @@ "const": "source_config" }, "source_config": { - "$ref": "#/components/schemas/SourceStripeConfig" + "oneOf": [ + { + "$ref": "#/components/schemas/SourceStripeConfig" + }, + { + "$ref": "#/components/schemas/SourceMetronomeConfig" + } + ] } }, "required": [ @@ -1834,6 +1942,9 @@ }, { "$ref": "#/components/schemas/DestinationGoogleSheetsConfig" + }, + { + "$ref": "#/components/schemas/DestinationRedisConfig" } ] } diff --git a/apps/engine/src/lib/default-connectors.ts b/apps/engine/src/lib/default-connectors.ts index a3414b114..c4fcd6134 100644 --- a/apps/engine/src/lib/default-connectors.ts +++ b/apps/engine/src/lib/default-connectors.ts @@ -1,13 +1,16 @@ import sourceStripe from '@stripe/sync-source-stripe' +import sourceMetronome from '@stripe/sync-source-metronome' import destinationPostgres from '@stripe/sync-destination-postgres' import destinationGoogleSheets from '@stripe/sync-destination-google-sheets' +import destinationRedis from '@stripe/sync-destination-redis' import type { RegisteredConnectors } from './resolver.js' /** Default in-process connectors bundled with the engine. */ export const defaultConnectors: RegisteredConnectors = { - sources: { stripe: sourceStripe }, + sources: { stripe: sourceStripe, metronome: sourceMetronome }, destinations: { postgres: destinationPostgres, google_sheets: destinationGoogleSheets, + redis: destinationRedis, }, } diff --git a/apps/service/src/__generated__/openapi.d.ts b/apps/service/src/__generated__/openapi.d.ts index d9a416097..f5c6b9022 100644 --- a/apps/service/src/__generated__/openapi.d.ts +++ b/apps/service/src/__generated__/openapi.d.ts @@ -226,6 +226,10 @@ export interface components { /** @constant */ type: "stripe"; stripe: components["schemas"]["SourceStripeConfig"]; + } | { + /** @constant */ + type: "metronome"; + metronome: components["schemas"]["SourceMetronomeConfig"]; }; SourceStripeConfig: { /** @description Stripe API key (sk_test_... or sk_live_...) */ @@ -263,6 +267,19 @@ export interface components { /** @description Override max requests per second (default: auto-derived from API key mode — 20 live, 10 test). */ rate_limit?: number; }; + SourceMetronomeConfig: { + /** @description Metronome API bearer token */ + api_key: string; + /** + * Format: uri + * @description Override the Metronome API base URL (default: https://api.metronome.com) + */ + base_url?: string; + /** @description Max requests per second (default: no limit) */ + rate_limit?: number; + /** @description Max records to fetch per stream (useful for testing) */ + backfill_limit?: number; + }; DestinationConfig: { /** @constant */ type: "postgres"; @@ -271,6 +288,10 @@ export interface components { /** @constant */ type: "google_sheets"; google_sheets: components["schemas"]["DestinationGoogleSheetsConfig"]; + } | { + /** @constant */ + type: "redis"; + redis: components["schemas"]["DestinationRedisConfig"]; }; DestinationPostgresConfig: { /** @description Postgres connection string */ @@ -331,6 +352,27 @@ export interface components { */ batch_size: number; }; + DestinationRedisConfig: { + /** @description Redis connection URL (redis://host:port) */ + url?: string; + /** @description Redis host (default: localhost) */ + host?: string; + /** @description Redis port (default: 6379) */ + port?: number; + /** @description Redis password */ + password?: string; + /** @description Redis database number (default: 0) */ + db?: number; + /** @description Enable TLS */ + tls?: boolean; + /** @description Prefix for all Redis keys (default: empty) */ + key_prefix?: string; + /** + * @description Records to buffer before flushing via pipeline + * @default 100 + */ + batch_size: number; + }; /** @description Full sync checkpoint with separate sections for source, destination, and sync run. Connectors only see their own section; the engine manages routing. */ SyncState: { source: components["schemas"]["SourceState"]; diff --git a/apps/service/src/__generated__/openapi.json b/apps/service/src/__generated__/openapi.json index 377ec6ed7..8ebb73b80 100644 --- a/apps/service/src/__generated__/openapi.json +++ b/apps/service/src/__generated__/openapi.json @@ -1180,6 +1180,22 @@ "type", "stripe" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "metronome" + }, + "metronome": { + "$ref": "#/components/schemas/SourceMetronomeConfig" + } + }, + "required": [ + "type", + "metronome" + ] } ], "type": "object", @@ -1317,6 +1333,36 @@ ], "additionalProperties": false }, + "SourceMetronomeConfig": { + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Metronome API bearer token" + }, + "base_url": { + "type": "string", + "format": "uri", + "description": "Override the Metronome API base URL (default: https://api.metronome.com)" + }, + "rate_limit": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "description": "Max requests per second (default: no limit)" + }, + "backfill_limit": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "description": "Max records to fetch per stream (useful for testing)" + } + }, + "required": [ + "api_key" + ], + "additionalProperties": false + }, "DestinationConfig": { "oneOf": [ { @@ -1350,6 +1396,22 @@ "type", "google_sheets" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "redis" + }, + "redis": { + "$ref": "#/components/schemas/DestinationRedisConfig" + } + }, + "required": [ + "type", + "redis" + ] } ], "type": "object", @@ -1472,6 +1534,45 @@ ], "additionalProperties": false }, + "DestinationRedisConfig": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "Redis connection URL (redis://host:port)" + }, + "host": { + "type": "string", + "description": "Redis host (default: localhost)" + }, + "port": { + "type": "number", + "description": "Redis port (default: 6379)" + }, + "password": { + "type": "string", + "description": "Redis password" + }, + "db": { + "type": "number", + "description": "Redis database number (default: 0)" + }, + "tls": { + "type": "boolean", + "description": "Enable TLS" + }, + "key_prefix": { + "type": "string", + "description": "Prefix for all Redis keys (default: empty)" + }, + "batch_size": { + "default": 100, + "type": "number", + "description": "Records to buffer before flushing via pipeline" + } + }, + "additionalProperties": false + }, "SyncState": { "type": "object", "properties": { diff --git a/compose.yml b/compose.yml index 3c3dcc6f3..5fbc9e903 100644 --- a/compose.yml +++ b/compose.yml @@ -33,6 +33,16 @@ services: timeout: 3s retries: 5 + redis: + image: redis:7-alpine + ports: + - '56379:6379' + healthcheck: + test: redis-cli ping + interval: 5s + timeout: 3s + retries: 5 + stripe-mock: image: stripe/stripe-mock:latest ports: diff --git a/packages/destination-redis/package.json b/packages/destination-redis/package.json new file mode 100644 index 000000000..dd8fd1309 --- /dev/null +++ b/packages/destination-redis/package.json @@ -0,0 +1,34 @@ +{ + "name": "@stripe/sync-destination-redis", + "version": "0.2.5", + "private": false, + "type": "module", + "exports": { + ".": { + "bun": "./src/index.ts", + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "bin": { + "destination-redis": "./dist/bin.js" + }, + "scripts": { + "build": "tsc", + "test": "vitest" + }, + "files": [ + "src", + "dist" + ], + "dependencies": { + "@stripe/sync-logger": "workspace:*", + "@stripe/sync-protocol": "workspace:*", + "ioredis": "^5.6.1", + "zod": "^4.3.6" + }, + "devDependencies": { + "@types/node": "^24.5.0", + "vitest": "^3.2.4" + } +} diff --git a/packages/destination-redis/src/bin.ts b/packages/destination-redis/src/bin.ts new file mode 100644 index 000000000..6ea2d562f --- /dev/null +++ b/packages/destination-redis/src/bin.ts @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import connector from './index.js' +import { configSchema } from './spec.js' +import { runConnectorCli } from '@stripe/sync-protocol/cli' + +runConnectorCli(connector, { name: 'destination-redis', configSchema }) diff --git a/packages/destination-redis/src/index.test.ts b/packages/destination-redis/src/index.test.ts new file mode 100644 index 000000000..8c4839549 --- /dev/null +++ b/packages/destination-redis/src/index.test.ts @@ -0,0 +1,87 @@ +import { describe, it, expect } from 'vitest' +import { configSchema } from './spec.js' +import { buildRecordKey } from './index.js' +import destination from './index.js' + +describe('destination-redis', () => { + describe('config validation', () => { + it('accepts url-only config', () => { + const result = configSchema.safeParse({ url: 'redis://localhost:6379' }) + expect(result.success).toBe(true) + }) + + it('accepts host/port config', () => { + const result = configSchema.safeParse({ host: 'localhost', port: 6379 }) + expect(result.success).toBe(true) + }) + + it('rejects both url and host', () => { + const result = configSchema.safeParse({ url: 'redis://localhost:6379', host: 'localhost' }) + expect(result.success).toBe(false) + }) + + it('rejects both url and port', () => { + const result = configSchema.safeParse({ url: 'redis://localhost:6379', port: 6379 }) + expect(result.success).toBe(false) + }) + + it('defaults batch_size to 100', () => { + const result = configSchema.safeParse({}) + expect(result.success).toBe(true) + if (result.success) { + expect(result.data.batch_size).toBe(100) + } + }) + + it('accepts full config', () => { + const result = configSchema.safeParse({ + host: 'redis.example.com', + port: 6380, + password: 'secret', + db: 1, + tls: true, + key_prefix: 'myapp:', + batch_size: 50, + }) + expect(result.success).toBe(true) + }) + }) + + describe('buildRecordKey', () => { + it('builds single-column key', () => { + expect(buildRecordKey('sync:', 'customers', ['id'], { id: 'cust_123', name: 'Alice' })).toBe( + 'sync:customers:cust_123' + ) + }) + + it('builds composite key', () => { + expect( + buildRecordKey('sync:', 'entitlements', ['account_id', 'created'], { + account_id: 'acct_1', + created: 1000, + }) + ).toBe('sync:entitlements:acct_1:1000') + }) + + it('handles missing key column with empty string', () => { + expect(buildRecordKey('', 'items', ['id'], { name: 'Alice' })).toBe('items:') + }) + + it('handles empty prefix', () => { + expect(buildRecordKey('', 'customers', ['id'], { id: 'cust_1' })).toBe('customers:cust_1') + }) + }) + + describe('spec()', () => { + it('yields a spec message with config schema', async () => { + const msgs: unknown[] = [] + for await (const msg of destination.spec()) { + msgs.push(msg) + } + expect(msgs).toHaveLength(1) + const msg = msgs[0] as { type: string; spec: { config: unknown } } + expect(msg.type).toBe('spec') + expect(msg.spec).toHaveProperty('config') + }) + }) +}) diff --git a/packages/destination-redis/src/index.ts b/packages/destination-redis/src/index.ts new file mode 100644 index 000000000..dd7c4661f --- /dev/null +++ b/packages/destination-redis/src/index.ts @@ -0,0 +1,244 @@ +import { Redis } from 'ioredis' +import type { Destination, DestinationInput, ConfiguredCatalog } from '@stripe/sync-protocol' +import defaultSpec from './spec.js' +import { log } from './logger.js' +import type { Config } from './spec.js' + +export { configSchema, type Config } from './spec.js' + +function createRedisClient(config: Config): Redis { + if (config.url) { + return new Redis(config.url, { + tls: config.tls ? {} : undefined, + }) + } + return new Redis({ + host: config.host ?? 'localhost', + port: config.port ?? 6379, + password: config.password, + db: config.db ?? 0, + tls: config.tls ? {} : undefined, + }) +} + +/** Build the Redis key from prefix, stream name, and primary key columns */ +export function buildRecordKey( + prefix: string, + stream: string, + primaryKeyColumns: string[], + data: Record +): string { + const pk = primaryKeyColumns.map((col) => String(data[col] ?? '')).join(':') + return `${prefix}${stream}:${pk}` +} + +function errorMessage(err: unknown): string { + if (!(err instanceof Error)) return String(err) + return err.message || (err as NodeJS.ErrnoException).code || err.constructor.name +} + +const destination = { + async *spec() { + yield { type: 'spec' as const, spec: defaultSpec } + }, + + async *check({ config }: { config: Config }) { + let redis: Redis | undefined + try { + redis = createRedisClient(config) + await redis.ping() + yield { + type: 'connection_status' as const, + connection_status: { status: 'succeeded' as const }, + } + } catch (err) { + yield { + type: 'connection_status' as const, + connection_status: { + status: 'failed' as const, + message: err instanceof Error ? err.message : String(err), + }, + } + } finally { + await redis?.quit() + } + }, + + async *setup({ config, catalog }: { config: Config; catalog: ConfiguredCatalog }) { + log.info( + { streams: catalog.streams.map((s) => s.stream.name), key_prefix: config.key_prefix }, + 'dest redis: setup (no-op for schemaless store)' + ) + }, + + async *teardown({ config }: { config: Config }) { + const prefix = config.key_prefix ?? '' + if (!prefix) { + throw new Error( + 'Refusing to teardown Redis without a key_prefix — would delete all keys in the database' + ) + } + const redis = createRedisClient(config) + try { + let cursor = '0' + do { + const [next, keys] = await redis.scan(cursor, 'MATCH', `${prefix}*`, 'COUNT', 100) + cursor = next + if (keys.length > 0) { + await redis.del(...keys) + } + } while (cursor !== '0') + log.info({ prefix }, 'dest redis: teardown complete') + } finally { + await redis.quit() + } + }, + + async *write( + { config, catalog }: { config: Config; catalog: ConfiguredCatalog }, + $stdin: AsyncIterable + ) { + const redis = createRedisClient(config) + const batchSize = config.batch_size + const keyPrefix = config.key_prefix ?? '' + + // Map stream name → primary key columns + const streamKeyColumns = new Map( + catalog.streams.map((cs) => [ + cs.stream.name, + cs.stream.primary_key?.map((pk) => pk[0]) ?? ['id'], + ]) + ) + + const failedStreams = new Set() + + // Per-stream buffers: array of { key, value } + const streamBuffers = new Map() + + /** Flush buffered records for a stream. Returns error message if failed. */ + const flushStream = async (streamName: string): Promise => { + if (failedStreams.has(streamName)) return undefined + const buffer = streamBuffers.get(streamName) + if (!buffer || buffer.length === 0) return undefined + + const startedAt = Date.now() + log.debug({ stream: streamName, batch_size: buffer.length }, 'dest redis: flush start') + + try { + const pipeline = redis.pipeline() + for (const { key, value } of buffer) { + pipeline.set(key, value) + } + await pipeline.exec() + log.debug( + { + stream: streamName, + batch_size: buffer.length, + duration_ms: Date.now() - startedAt, + }, + 'dest redis: flush complete' + ) + } catch (err) { + const errMsg = errorMessage(err) + log.error( + { stream: streamName, batch_size: buffer.length, error: errMsg }, + 'dest redis: flush failed' + ) + failedStreams.add(streamName) + streamBuffers.set(streamName, []) + return `${errMsg} (stream=${streamName})` + } + streamBuffers.set(streamName, []) + return undefined + } + + function streamError(stream: string, error: string) { + return { + type: 'stream_status' as const, + stream_status: { stream, status: 'error' as const, error }, + } + } + + try { + await redis.ping() // verify connection + + for await (const msg of $stdin) { + if (msg.type === 'record') { + const { stream, data } = msg.record + + if (failedStreams.has(stream)) { + log.debug({ stream }, 'dest redis: skipping record for failed stream') + continue + } + + if (!streamBuffers.has(stream)) { + streamBuffers.set(stream, []) + } + + const pk = streamKeyColumns.get(stream) ?? ['id'] + const key = buildRecordKey(keyPrefix, stream, pk, data as Record) + const buffer = streamBuffers.get(stream)! + buffer.push({ key, value: JSON.stringify(data) }) + + if (buffer.length >= batchSize) { + const err = await flushStream(stream) + if (err) { + log.error( + { stream, error: err }, + 'dest redis: yielding stream_status error (batch flush)' + ) + yield streamError(stream, err) + continue + } + } + yield msg + } else if (msg.type === 'source_state') { + if (msg.source_state.state_type !== 'global') { + const stream = msg.source_state.stream + if (failedStreams.has(stream)) { + log.debug({ stream }, 'dest redis: skipping source_state for failed stream') + continue + } + const err = await flushStream(stream) + if (err) { + log.error( + { stream, error: err }, + 'dest redis: yielding stream_status error (state flush)' + ) + yield streamError(stream, err) + continue + } + } + yield msg + } else { + yield msg + } + } + + // Final flush + for (const streamName of streamBuffers.keys()) { + const err = await flushStream(streamName) + if (err) { + log.error( + { stream: streamName, error: err }, + 'dest redis: yielding stream_status error (final flush)' + ) + yield streamError(streamName, err) + } + } + + if (failedStreams.size > 0) { + log.error( + { failed_streams: [...failedStreams] }, + `Redis destination: completed with ${failedStreams.size} failed stream(s)` + ) + } else { + log.debug('Redis destination: write complete') + } + } finally { + await redis.quit() + } + }, +} satisfies Destination + +export default destination diff --git a/packages/destination-redis/src/integration.test.ts b/packages/destination-redis/src/integration.test.ts new file mode 100644 index 000000000..a600c9bae --- /dev/null +++ b/packages/destination-redis/src/integration.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { Redis } from 'ioredis' +import destination from './index.js' + +const REDIS_URL = process.env.REDIS_URL ?? 'redis://localhost:56379' + +/** Collect all items from an async iterable */ +async function collectAll(iter: AsyncIterable): Promise { + const results: T[] = [] + for await (const item of iter) results.push(item) + return results +} + +// Detect Redis availability at module level (synchronous probe) +let available = false +try { + const r = new Redis(REDIS_URL, { lazyConnect: true, connectTimeout: 1000 }) + await r.connect() + await r.ping() + available = true + await r.quit() +} catch { + available = false +} + +describe.skipIf(!available)('destination-redis integration', () => { + const config = { url: REDIS_URL, key_prefix: 'test_sync:', batch_size: 2 } + const catalog = { + streams: [ + { + stream: { + name: 'customers', + primary_key: [['id']], + newer_than_field: '_synced_at', + json_schema: {}, + }, + sync_mode: 'full_refresh' as const, + destination_sync_mode: 'append_dedup' as const, + }, + ], + } + + afterAll(async () => { + const r = new Redis(REDIS_URL) + const keys = await r.keys('test_sync:*') + if (keys.length > 0) await r.del(...keys) + await r.quit() + }) + + it('check succeeds with valid connection', async () => { + const msgs = await collectAll(destination.check({ config })) + expect(msgs).toEqual([ + { type: 'connection_status', connection_status: { status: 'succeeded' } }, + ]) + }) + + it('writes records to Redis hash', async () => { + async function* input() { + yield { + type: 'record' as const, + record: { + stream: 'customers', + data: { id: 'cust_1', name: 'Alice', _synced_at: 1000 }, + emitted_at: '2024-01-01T00:00:00.000Z', + }, + } + yield { + type: 'record' as const, + record: { + stream: 'customers', + data: { id: 'cust_2', name: 'Bob', _synced_at: 1001 }, + emitted_at: '2024-01-01T00:00:00.000Z', + }, + } + yield { + type: 'source_state' as const, + source_state: { state_type: 'stream' as const, stream: 'customers', data: {} }, + } + } + + await collectAll(destination.write({ config, catalog }, input())) + + const r = new Redis(REDIS_URL) + const val1 = await r.get('test_sync:customers:cust_1') + const val2 = await r.get('test_sync:customers:cust_2') + expect(JSON.parse(val1!)).toMatchObject({ id: 'cust_1', name: 'Alice' }) + expect(JSON.parse(val2!)).toMatchObject({ id: 'cust_2', name: 'Bob' }) + await r.quit() + }) + + it('teardown deletes prefixed keys', async () => { + const r = new Redis(REDIS_URL) + await r.set('test_sync:teardown_test:key1', 'val1') + await r.quit() + + await collectAll(destination.teardown!({ config })) + + const r2 = new Redis(REDIS_URL) + const keys = await r2.keys('test_sync:*') + expect(keys).toHaveLength(0) + await r2.quit() + }) +}) diff --git a/packages/destination-redis/src/logger.ts b/packages/destination-redis/src/logger.ts new file mode 100644 index 000000000..b635d3100 --- /dev/null +++ b/packages/destination-redis/src/logger.ts @@ -0,0 +1,4 @@ +import { createLogger } from '@stripe/sync-logger' +import type { Logger } from '@stripe/sync-logger' + +export const log: Logger = createLogger({ name: 'destination-redis' }) diff --git a/packages/destination-redis/src/spec.ts b/packages/destination-redis/src/spec.ts new file mode 100644 index 000000000..8537bedf1 --- /dev/null +++ b/packages/destination-redis/src/spec.ts @@ -0,0 +1,24 @@ +import { z } from 'zod' +import type { ConnectorSpecification } from '@stripe/sync-protocol' + +export const configSchema = z + .object({ + url: z.string().optional().describe('Redis connection URL (redis://host:port)'), + host: z.string().optional().describe('Redis host (default: localhost)'), + port: z.number().optional().describe('Redis port (default: 6379)'), + password: z.string().optional().describe('Redis password'), + db: z.number().optional().describe('Redis database number (default: 0)'), + tls: z.boolean().optional().describe('Enable TLS'), + key_prefix: z.string().optional().describe('Prefix for all Redis keys (default: empty)'), + batch_size: z.number().default(100).describe('Records to buffer before flushing via pipeline'), + }) + .refine((c) => !(c.url && (c.host || c.port)), { + message: 'Specify either url or host/port, not both', + path: ['url'], + }) + +export type Config = z.infer + +export default { + config: z.toJSONSchema(configSchema), +} satisfies ConnectorSpecification diff --git a/packages/destination-redis/tsconfig.json b/packages/destination-redis/tsconfig.json new file mode 100644 index 000000000..2481fe545 --- /dev/null +++ b/packages/destination-redis/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts", "src/**/__tests__/**"] +} diff --git a/packages/source-metronome/package.json b/packages/source-metronome/package.json new file mode 100644 index 000000000..914c83e8d --- /dev/null +++ b/packages/source-metronome/package.json @@ -0,0 +1,33 @@ +{ + "name": "@stripe/sync-source-metronome", + "version": "0.2.5", + "private": false, + "type": "module", + "exports": { + ".": { + "bun": "./src/index.ts", + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "bin": { + "source-metronome": "./dist/bin.js" + }, + "scripts": { + "build": "tsc", + "test": "vitest" + }, + "files": [ + "src", + "dist" + ], + "dependencies": { + "@stripe/sync-logger": "workspace:*", + "@stripe/sync-protocol": "workspace:*", + "zod": "^4.3.6" + }, + "devDependencies": { + "@types/node": "^24.5.0", + "vitest": "^3.2.4" + } +} diff --git a/packages/source-metronome/src/bin.ts b/packages/source-metronome/src/bin.ts new file mode 100644 index 000000000..97cb7a3f1 --- /dev/null +++ b/packages/source-metronome/src/bin.ts @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import connector from './index.js' +import { configSchema } from './spec.js' +import { runConnectorCli } from '@stripe/sync-protocol/cli' + +runConnectorCli(connector, { name: 'source-metronome', configSchema }) diff --git a/packages/source-metronome/src/client.test.ts b/packages/source-metronome/src/client.test.ts new file mode 100644 index 000000000..db6453e96 --- /dev/null +++ b/packages/source-metronome/src/client.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it, vi } from 'vitest' +import { MetronomeClient } from './client.js' + +function makeResponse(body: unknown, status = 200, headers?: Record): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json', ...headers }, + }) +} + +describe('MetronomeClient', () => { + describe('paginate', () => { + it('stops after a single page when next_page is null', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + makeResponse({ data: [{ id: 'cus_1' }, { id: 'cus_2' }], next_page: null }) + ) + + const client = new MetronomeClient({ apiKey: 'test-key', fetch: fetchMock }) + const pages: unknown[][] = [] + for await (const page of client.paginate('GET', '/v1/customers')) { + pages.push(page.data) + } + + expect(pages).toHaveLength(1) + expect(pages[0]).toHaveLength(2) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('follows next_page cursor across multiple pages', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(makeResponse({ data: [{ id: 'cus_1' }], next_page: 'cursor_abc' })) + .mockResolvedValueOnce(makeResponse({ data: [{ id: 'cus_2' }], next_page: null })) + + const client = new MetronomeClient({ apiKey: 'test-key', fetch: fetchMock }) + const allRecords: unknown[] = [] + for await (const page of client.paginate('GET', '/v1/customers')) { + allRecords.push(...page.data) + } + + expect(allRecords).toHaveLength(2) + expect(fetchMock).toHaveBeenCalledTimes(2) + + // Second call should include next_page in query params + const secondCall = fetchMock.mock.calls[1] + expect(secondCall[0]).toContain('next_page=cursor_abc') + }) + + it('uses POST body for POST pagination', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(makeResponse({ data: [{ id: 'contract_1' }], next_page: null })) + + const client = new MetronomeClient({ apiKey: 'test-key', fetch: fetchMock }) + const pages: unknown[][] = [] + for await (const page of client.paginate('POST', '/v1/contracts/list', { + customer_id: 'cus_1', + })) { + pages.push(page.data) + } + + expect(pages).toHaveLength(1) + const [url, init] = fetchMock.mock.calls[0] + expect(url).toContain('/v1/contracts/list') + expect(init?.method).toBe('POST') + const body = JSON.parse(init?.body as string) + expect(body).toMatchObject({ customer_id: 'cus_1', limit: 100 }) + }) + }) + + describe('retry logic', () => { + it('retries on 429 and eventually succeeds', async () => { + vi.useFakeTimers() + const fetchMock = vi + .fn() + .mockResolvedValueOnce(makeResponse({ error: 'rate limited' }, 429, { 'retry-after': '0' })) + .mockResolvedValueOnce(makeResponse({ id: 'cus_1', name: 'Test' })) + + const client = new MetronomeClient({ apiKey: 'test-key', fetch: fetchMock }) + const pending = client.get('/v1/customers/cus_1') + await vi.runAllTimersAsync() + const result = await pending + + expect(result).toEqual({ id: 'cus_1', name: 'Test' }) + expect(fetchMock).toHaveBeenCalledTimes(2) + vi.useRealTimers() + }) + + it('throws immediately on non-retryable 400 error', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(makeResponse({ error: 'bad request' }, 400)) + + const client = new MetronomeClient({ apiKey: 'test-key', fetch: fetchMock }) + await expect(client.get('/v1/customers/bad')).rejects.toThrow('Metronome API 400') + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('includes Authorization header on all requests', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(makeResponse({ data: [], next_page: null })) + + const client = new MetronomeClient({ apiKey: 'my-secret-key', fetch: fetchMock }) + await client.get('/v1/customers') + + const [, init] = fetchMock.mock.calls[0] + expect((init?.headers as Record)['Authorization']).toBe( + 'Bearer my-secret-key' + ) + }) + }) +}) diff --git a/packages/source-metronome/src/client.ts b/packages/source-metronome/src/client.ts new file mode 100644 index 000000000..e12a9f9d5 --- /dev/null +++ b/packages/source-metronome/src/client.ts @@ -0,0 +1,128 @@ +import { log } from './logger.js' + +export interface MetronomeClientOptions { + apiKey: string + baseUrl?: string + rateLimitPerSecond?: number + fetch?: typeof globalThis.fetch +} + +export interface MetronomePageResponse> { + data: T[] + next_page: string | null +} + +const DEFAULT_BASE_URL = 'https://api.metronome.com' +const MAX_RETRIES = 3 +const PAGE_SIZE = 100 + +export class MetronomeClient { + private apiKey: string + private baseUrl: string + private rateLimitPerSecond?: number + private lastRequestAt = 0 + private fetchFn: typeof globalThis.fetch + + constructor(opts: MetronomeClientOptions) { + this.apiKey = opts.apiKey + this.baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, '') + this.rateLimitPerSecond = opts.rateLimitPerSecond + this.fetchFn = opts.fetch ?? globalThis.fetch + } + + private async rateLimit(): Promise { + if (!this.rateLimitPerSecond) return + const minInterval = 1000 / this.rateLimitPerSecond + const elapsed = Date.now() - this.lastRequestAt + if (elapsed < minInterval) { + await new Promise((r) => setTimeout(r, minInterval - elapsed)) + } + this.lastRequestAt = Date.now() + } + + private async request( + method: 'GET' | 'POST', + path: string, + body?: Record + ): Promise { + await this.rateLimit() + const url = `${this.baseUrl}${path}` + let lastError: Error | undefined + + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + if (attempt > 0) { + const delay = 1000 * 2 ** (attempt - 1) + log.debug({ attempt, delay, path }, 'metronome: retrying request') + await new Promise((r) => setTimeout(r, delay)) + } + + const res = await this.fetchFn(url, { + method, + headers: { + Authorization: `Bearer ${this.apiKey}`, + 'Content-Type': 'application/json', + }, + body: body ? JSON.stringify(body) : undefined, + }) + + if (res.status === 429 || res.status >= 500) { + const retryAfter = res.headers.get('retry-after') + if (retryAfter && attempt < MAX_RETRIES) { + const delaySec = Number(retryAfter) + if (Number.isFinite(delaySec)) { + await new Promise((r) => setTimeout(r, delaySec * 1000)) + } + } + lastError = new Error(`Metronome API ${res.status}: ${await res.text()}`) + continue + } + + if (!res.ok) { + throw new Error(`Metronome API ${res.status}: ${await res.text()}`) + } + + return await res.json() + } + + throw lastError ?? new Error('Metronome API: max retries exceeded') + } + + async get(path: string): Promise { + return (await this.request('GET', path)) as T + } + + async post(path: string, body?: Record): Promise { + return (await this.request('POST', path, body)) as T + } + + async *paginate>( + method: 'GET' | 'POST', + path: string, + body?: Record, + startCursor?: string | null + ): AsyncGenerator> { + let nextPage: string | undefined = startCursor ?? undefined + + while (true) { + let page: MetronomePageResponse + + if (method === 'GET') { + const params = new URLSearchParams({ limit: String(PAGE_SIZE) }) + if (nextPage) params.set('next_page', nextPage) + page = await this.get>(`${path}?${params.toString()}`) + } else { + const reqBody: Record = { + ...(body ?? {}), + limit: PAGE_SIZE, + } + if (nextPage) reqBody['next_page'] = nextPage + page = await this.post>(path, reqBody) + } + + yield page + + if (!page.next_page) break + nextPage = page.next_page + } + } +} diff --git a/packages/source-metronome/src/index.test.ts b/packages/source-metronome/src/index.test.ts new file mode 100644 index 000000000..f19ca2ee5 --- /dev/null +++ b/packages/source-metronome/src/index.test.ts @@ -0,0 +1,243 @@ +import { describe, expect, it, vi } from 'vitest' +import type { ConfiguredCatalog, Message } from '@stripe/sync-protocol' +import source from './index.js' +import { resources } from './resources.js' + +async function collectAll(iter: AsyncIterable): Promise { + const results: T[] = [] + for await (const item of iter) results.push(item) + return results +} + +function makeResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +const TEST_CONFIG = { api_key: 'test-bearer-token' } + +const CUSTOMERS_CATALOG: ConfiguredCatalog = { + streams: [ + { + stream: { + name: 'customers', + primary_key: [['id']], + newer_than_field: '_synced_at', + json_schema: {}, + }, + sync_mode: 'full_refresh', + destination_sync_mode: 'append_dedup', + }, + ], +} + +describe('source-metronome', () => { + describe('spec()', () => { + it('yields a spec message with config JSON schema', async () => { + const messages = await collectAll(source.spec()) + expect(messages).toHaveLength(1) + const [msg] = messages + expect(msg.type).toBe('spec') + if (msg.type !== 'spec') throw new Error('expected spec') + expect(msg.spec.config).toBeDefined() + expect(typeof msg.spec.config).toBe('object') + expect(msg.spec.source_state_stream).toBeDefined() + }) + }) + + describe('discover()', () => { + it('yields a catalog with all expected stream names', async () => { + const messages = await collectAll(source.discover({ config: TEST_CONFIG })) + expect(messages).toHaveLength(1) + const [msg] = messages + expect(msg.type).toBe('catalog') + if (msg.type !== 'catalog') throw new Error('expected catalog') + const streamNames = msg.catalog.streams.map((s) => s.name) + for (const resource of resources) { + expect(streamNames).toContain(resource.name) + } + }) + }) + + describe('check()', () => { + it('yields connection_status succeeded when API responds 200', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(makeResponse({ data: [], next_page: null })) + + vi.stubGlobal('fetch', fetchMock) + try { + const messages = await collectAll( + source.check({ config: { ...TEST_CONFIG, base_url: 'http://metronome.test' } }) + ) + expect(messages).toHaveLength(1) + const [msg] = messages + expect(msg.type).toBe('connection_status') + if (msg.type !== 'connection_status') throw new Error('expected connection_status') + expect(msg.connection_status.status).toBe('succeeded') + } finally { + vi.unstubAllGlobals() + } + }) + + it('yields connection_status failed when API returns an error', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(makeResponse({ error: 'Unauthorized' }, 401)) + + vi.stubGlobal('fetch', fetchMock) + try { + const messages = await collectAll( + source.check({ config: { ...TEST_CONFIG, base_url: 'http://metronome.test' } }) + ) + expect(messages).toHaveLength(1) + const [msg] = messages + expect(msg.type).toBe('connection_status') + if (msg.type !== 'connection_status') throw new Error('expected connection_status') + expect(msg.connection_status.status).toBe('failed') + expect(msg.connection_status.message).toContain('401') + } finally { + vi.unstubAllGlobals() + } + }) + }) + + describe('read()', () => { + it('yields the correct message sequence: started, records, source_state, complete', async () => { + const fetchMock = vi.fn().mockResolvedValueOnce( + makeResponse({ + data: [ + { id: 'cus_1', name: 'Acme Corp', external_id: 'ext_1' }, + { id: 'cus_2', name: 'Globex', external_id: 'ext_2' }, + ], + next_page: null, + }) + ) + + vi.stubGlobal('fetch', fetchMock) + try { + const messages = (await collectAll( + source.read({ config: TEST_CONFIG, catalog: CUSTOMERS_CATALOG }) + )) as Message[] + + // stream_status: start + const startMsg = messages.find( + (m) => m.type === 'stream_status' && m.stream_status.status === 'start' + ) + expect(startMsg).toBeDefined() + + // records + const records = messages.filter((m) => m.type === 'record') + expect(records).toHaveLength(2) + if (records[0].type !== 'record') throw new Error('expected record') + expect(records[0].record.stream).toBe('customers') + expect(records[0].record.data).toMatchObject({ id: 'cus_1', name: 'Acme Corp' }) + expect(typeof records[0].record.data['_synced_at']).toBe('number') + + // source_state checkpoint + const stateMessages = messages.filter((m) => m.type === 'source_state') + expect(stateMessages.length).toBeGreaterThanOrEqual(1) + + // stream_status: complete + const completeMsg = messages.find( + (m) => m.type === 'stream_status' && m.stream_status.status === 'complete' + ) + expect(completeMsg).toBeDefined() + + // order: start before complete + const startIdx = messages.indexOf(startMsg!) + const completeIdx = messages.indexOf(completeMsg!) + expect(startIdx).toBeLessThan(completeIdx) + } finally { + vi.unstubAllGlobals() + } + }) + + it('only syncs streams in the configured catalog', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(makeResponse({ data: [{ id: 'plan_1', name: 'Pro' }], next_page: null })) + + vi.stubGlobal('fetch', fetchMock) + try { + const plansCatalog: ConfiguredCatalog = { + streams: [ + { + stream: { + name: 'plans', + primary_key: [['id']], + newer_than_field: '_synced_at', + json_schema: {}, + }, + sync_mode: 'full_refresh', + destination_sync_mode: 'append_dedup', + }, + ], + } + + const messages = (await collectAll( + source.read({ config: TEST_CONFIG, catalog: plansCatalog }) + )) as Message[] + + const recordStreams = new Set( + messages + .filter((m): m is Extract => m.type === 'record') + .map((m) => m.record.stream) + ) + expect(recordStreams).toEqual(new Set(['plans'])) + } finally { + vi.unstubAllGlobals() + } + }) + + it('resumes from a stored cursor via startCursor', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + makeResponse({ data: [{ id: 'cus_3', name: 'NewCo' }], next_page: null }) + ) + + vi.stubGlobal('fetch', fetchMock) + try { + await collectAll( + source.read({ + config: TEST_CONFIG, + catalog: CUSTOMERS_CATALOG, + state: { + streams: { customers: { next_page: 'resume_cursor_xyz' } }, + global: {}, + }, + }) + ) + + // The fetch URL should include the resume cursor + const [url] = fetchMock.mock.calls[0] + expect(url as string).toContain('next_page=resume_cursor_xyz') + } finally { + vi.unstubAllGlobals() + } + }) + + it('emits stream_status error when API throws', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(makeResponse({ error: 'Internal Server Error' }, 500)) + + vi.stubGlobal('fetch', fetchMock) + try { + const messages = (await collectAll( + source.read({ config: TEST_CONFIG, catalog: CUSTOMERS_CATALOG }) + )) as Message[] + + const errorMsg = messages.find( + (m) => m.type === 'stream_status' && m.stream_status.status === 'error' + ) + expect(errorMsg).toBeDefined() + } finally { + vi.unstubAllGlobals() + } + }) + }) +}) diff --git a/packages/source-metronome/src/index.ts b/packages/source-metronome/src/index.ts new file mode 100644 index 000000000..5de8794a2 --- /dev/null +++ b/packages/source-metronome/src/index.ts @@ -0,0 +1,406 @@ +import type { + CatalogPayload, + Source, + SpecOutput, + CheckOutput, + DiscoverOutput, + Message, +} from '@stripe/sync-protocol' +import { createSourceMessageFactory } from '@stripe/sync-protocol' +import defaultSpec from './spec.js' +import type { Config, StreamState } from './spec.js' +import { MetronomeClient } from './client.js' +import { resources } from './resources.js' +import { log } from './logger.js' +import { startWebhookServer } from './webhook.js' +import type { MetronomeWebhookEvent } from './webhook.js' + +export { configSchema, type Config } from './spec.js' + +export const msg = createSourceMessageFactory< + StreamState, + Record, + Record +>() + +function buildCatalog(): CatalogPayload { + return { + streams: resources.map((r) => ({ + name: r.name, + primary_key: r.primaryKey, + newer_than_field: '_synced_at', + json_schema: r.jsonSchema, + })), + } +} + +/** Event types that affect credit balances or entitlements */ +const ENTITLEMENT_EVENT_TYPES = new Set([ + 'contract.create', + 'contract.start', + 'contract.edit', + 'contract.end', + 'contract.archive', + 'commit.create', + 'commit.edit', + 'commit.segment.start', + 'commit.segment.end', + 'credit.create', + 'credit.edit', + 'credit.segment.start', + 'credit.segment.end', +]) + +/** + * On a webhook event, re-fetch affected data from Metronome and yield updated records. + * For credit events: re-fetch credit grants for the customer. + * For contract events: re-fetch entitlements (rate schedule) for the customer's contracts. + */ +async function* processWebhookEvent( + event: MetronomeWebhookEvent, + client: MetronomeClient, + configuredStreamNames: Set +): AsyncGenerator { + const customerId = event.customer_id ?? (event.properties?.customer_id as string | undefined) + if (!customerId) { + log.warn({ eventType: event.type }, 'metronome: webhook event has no customer_id, skipping') + return + } + + log.info( + { eventType: event.type, customerId, eventId: event.id }, + 'metronome: processing webhook event' + ) + + const now = Math.floor(Date.now() / 1000) + + // Re-fetch credit grants for this customer + if (configuredStreamNames.has('credit_grants')) { + for await (const page of client.paginate('POST', '/v1/credits/listGrants', { + customer_ids: [customerId], + })) { + for (const grant of page.data) { + yield msg.record({ + stream: 'credit_grants', + data: { ...(grant as Record), _synced_at: now }, + emitted_at: new Date().toISOString(), + }) + } + } + } + + // Re-fetch entitlements (rate schedules) for this customer's contracts + if (configuredStreamNames.has('entitlements')) { + const contractId = event.contract_id ?? (event.properties?.contract_id as string | undefined) + const contractIds: string[] = [] + + if (contractId) { + contractIds.push(contractId) + } else { + // Fetch all contracts for this customer + for await (const page of client.paginate<{ id: string }>('POST', '/v2/contracts/list', { + customer_id: customerId, + })) { + for (const c of page.data) contractIds.push(c.id) + } + } + + for (const cid of contractIds) { + for await (const page of client.paginate('POST', '/v1/contracts/getContractRateSchedule', { + customer_id: customerId, + contract_id: cid, + at: new Date().toISOString(), + })) { + for (const record of page.data) { + yield msg.record({ + stream: 'entitlements', + data: { + ...(record as Record), + customer_id: customerId, + contract_id: cid, + _synced_at: now, + }, + emitted_at: new Date().toISOString(), + }) + } + } + } + } +} + +const source: Source = { + async *spec(): AsyncGenerator { + yield { type: 'spec' as const, spec: defaultSpec } + }, + + async *check({ config }: { config: Config }): AsyncGenerator { + const client = new MetronomeClient({ + apiKey: config.api_key, + baseUrl: config.base_url, + }) + try { + await client.get('/v1/customers?limit=1') + yield msg.connection_status({ status: 'succeeded' }) + } catch (err) { + yield msg.connection_status({ + status: 'failed', + message: err instanceof Error ? err.message : String(err), + }) + } + }, + + async *discover(): AsyncGenerator { + yield { type: 'catalog' as const, catalog: buildCatalog() } + }, + + async *read({ + config, + catalog, + state, + }: { + config: Config + catalog: import('@stripe/sync-protocol').ConfiguredCatalog + state?: { streams: Record; global: Record } + }) { + const client = new MetronomeClient({ + apiKey: config.api_key, + baseUrl: config.base_url, + rateLimitPerSecond: config.rate_limit, + }) + const streamStates = state?.streams ?? {} + const configuredStreamNames = new Set(catalog.streams.map((s) => s.stream.name)) + + // For per-customer and per-contract resources, we need parent IDs + let customerIds: string[] | undefined + let customerContracts: Map | undefined + + /** Lazy-load all customer IDs */ + async function ensureCustomerIds() { + if (customerIds) return customerIds + customerIds = [] + for await (const page of client.paginate<{ id: string }>('GET', '/v1/customers')) { + for (const c of page.data) { + customerIds.push(c.id) + } + } + log.info({ count: customerIds.length }, 'metronome: loaded customer IDs') + return customerIds + } + + /** Lazy-load all customer → contract ID mappings */ + async function ensureCustomerContracts() { + if (customerContracts) return customerContracts + const custIds = await ensureCustomerIds() + customerContracts = new Map() + for (const customerId of custIds) { + const contractIds: string[] = [] + for await (const page of client.paginate<{ id: string }>('POST', '/v2/contracts/list', { + customer_id: customerId, + })) { + for (const c of page.data) { + contractIds.push(c.id) + } + } + if (contractIds.length > 0) { + customerContracts.set(customerId, contractIds) + } + } + log.info( + { + customers: customerContracts.size, + contracts: [...customerContracts.values()].reduce((s, c) => s + c.length, 0), + }, + 'metronome: loaded customer→contract mappings' + ) + return customerContracts + } + + for (const resource of resources) { + if (!configuredStreamNames.has(resource.name)) continue + + const streamName = resource.name + yield msg.stream_status({ stream: streamName, status: 'start' }) + + try { + let recordCount = 0 + const existingState = streamStates[streamName] + const startCursor = existingState?.next_page + + if (resource.perContract) { + // Per-contract: iterate customers → contracts → rate schedule + const mapping = await ensureCustomerContracts() + + outer_contract: for (const [customerId, contractIds] of mapping) { + for (const contractId of contractIds) { + for await (const page of client.paginate(resource.method, resource.endpoint, { + customer_id: customerId, + contract_id: contractId, + at: new Date().toISOString(), + })) { + for (const record of page.data) { + const data = { + ...(record as Record), + customer_id: customerId, + contract_id: contractId, + _synced_at: Math.floor(Date.now() / 1000), + } + yield msg.record({ + stream: streamName, + data, + emitted_at: new Date().toISOString(), + }) + recordCount++ + if (config.backfill_limit && recordCount >= config.backfill_limit) + break outer_contract + } + if (config.backfill_limit && recordCount >= config.backfill_limit) break + } + } + } + } else if (resource.perCustomer) { + // Per-customer: iterate customers + const custIds = await ensureCustomerIds() + + outer: for (const customerId of custIds) { + for await (const page of client.paginate(resource.method, resource.endpoint, { + customer_id: customerId, + })) { + for (const record of page.data) { + const data = { + ...(record as Record), + _synced_at: Math.floor(Date.now() / 1000), + } + yield msg.record({ + stream: streamName, + data, + emitted_at: new Date().toISOString(), + }) + recordCount++ + if (config.backfill_limit && recordCount >= config.backfill_limit) break outer + } + if (config.backfill_limit && recordCount >= config.backfill_limit) break + } + } + } else { + for await (const page of client.paginate( + resource.method, + resource.endpoint, + undefined, + startCursor + )) { + for (const record of page.data) { + const data = { + ...(record as Record), + _synced_at: Math.floor(Date.now() / 1000), + } + yield msg.record({ + stream: streamName, + data, + emitted_at: new Date().toISOString(), + }) + recordCount++ + if (config.backfill_limit && recordCount >= config.backfill_limit) break + } + + // Checkpoint after each page + yield msg.source_state({ + state_type: 'stream', + stream: streamName, + data: { next_page: page.next_page }, + }) + + if (config.backfill_limit && recordCount >= config.backfill_limit) break + } + } + + // Final state: null cursor means complete + yield msg.source_state({ + state_type: 'stream', + stream: streamName, + data: { next_page: null }, + }) + + log.info({ stream: streamName, records: recordCount }, 'metronome: stream complete') + yield msg.stream_status({ stream: streamName, status: 'complete' }) + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + log.error({ stream: streamName, error: message }, 'metronome: stream error') + yield msg.stream_status({ stream: streamName, status: 'error', error: message }) + } + } + + // After backfill: start webhook server for live updates + if (config.webhook_port) { + log.info( + { port: config.webhook_port }, + 'metronome: starting webhook listener for live updates' + ) + + type QueueItem = { event: MetronomeWebhookEvent; resolve: () => void } + const queue: QueueItem[] = [] + let waiter: ((item: QueueItem) => void) | null = null + + const server = startWebhookServer(config.webhook_port, config.webhook_secret, (input) => { + if (!ENTITLEMENT_EVENT_TYPES.has(input.event.type)) { + log.debug({ eventType: input.event.type }, 'metronome: ignoring non-entitlement event') + return + } + const { promise, resolve } = Promise.withResolvers() + const item = { event: input.event, resolve } + if (waiter) { + const w = waiter + waiter = null + w(item) + } else { + queue.push(item) + } + // Block HTTP response until we've processed the event + return promise + }) + + try { + // Process webhook events forever (until abort) + while (true) { + const item: QueueItem = await new Promise((resolve) => { + if (queue.length > 0) { + resolve(queue.shift()!) + } else { + waiter = resolve + } + }) + + try { + yield* processWebhookEvent(item.event, client, configuredStreamNames) + // Emit state checkpoint so destination flushes immediately + if (configuredStreamNames.has('credit_grants')) { + yield msg.source_state({ + state_type: 'stream', + stream: 'credit_grants', + data: { next_page: null }, + }) + } + if (configuredStreamNames.has('entitlements')) { + yield msg.source_state({ + state_type: 'stream', + stream: 'entitlements', + data: { next_page: null }, + }) + } + item.resolve() + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + log.error( + { error: message, eventType: item.event.type }, + 'metronome: webhook event processing error' + ) + item.resolve() // still resolve to unblock HTTP response + } + } + } finally { + server.close() + } + } + }, +} + +export default source diff --git a/packages/source-metronome/src/logger.ts b/packages/source-metronome/src/logger.ts new file mode 100644 index 000000000..e41c400f2 --- /dev/null +++ b/packages/source-metronome/src/logger.ts @@ -0,0 +1,4 @@ +import { createLogger } from '@stripe/sync-logger' +import type { Logger } from '@stripe/sync-logger' + +export const log: Logger = createLogger({ name: 'source-metronome' }) diff --git a/packages/source-metronome/src/resources.ts b/packages/source-metronome/src/resources.ts new file mode 100644 index 000000000..03fcf4ce5 --- /dev/null +++ b/packages/source-metronome/src/resources.ts @@ -0,0 +1,194 @@ +export interface ResourceDefinition { + /** Stream/table name */ + name: string + /** API endpoint path */ + endpoint: string + /** HTTP method */ + method: 'GET' | 'POST' + /** JSON Schema for the record shape */ + jsonSchema: Record + /** Primary key field paths */ + primaryKey: string[][] + /** If true, requires iterating parent customers first */ + perCustomer?: boolean + /** If true, requires iterating parent customers AND their contracts */ + perContract?: boolean +} + +export const resources: ResourceDefinition[] = [ + { + name: 'customers', + endpoint: '/v1/customers', + method: 'GET', + primaryKey: [['id']], + jsonSchema: { + type: 'object', + properties: { + id: { type: 'string' }, + name: { type: 'string' }, + external_id: { type: 'string' }, + ingest_aliases: { type: 'array', items: { type: 'string' } }, + created_at: { type: 'string' }, + updated_at: { type: 'string' }, + archived_at: { type: ['string', 'null'] }, + custom_fields: { type: 'object' }, + _synced_at: { type: 'integer' }, + }, + }, + }, + { + name: 'billable_metrics', + endpoint: '/v1/billable-metrics', + method: 'GET', + primaryKey: [['id']], + jsonSchema: { + type: 'object', + properties: { + id: { type: 'string' }, + name: { type: 'string' }, + group_keys: { type: 'array' }, + aggregation_type: { type: 'string' }, + aggregation_key: { type: ['string', 'null'] }, + event_type_filter: { type: 'object' }, + custom_fields: { type: 'object' }, + _synced_at: { type: 'integer' }, + }, + }, + }, + { + name: 'plans', + endpoint: '/v1/plans', + method: 'GET', + primaryKey: [['id']], + jsonSchema: { + type: 'object', + properties: { + id: { type: 'string' }, + name: { type: 'string' }, + description: { type: ['string', 'null'] }, + custom_fields: { type: 'object' }, + _synced_at: { type: 'integer' }, + }, + }, + }, + { + name: 'contracts', + endpoint: '/v2/contracts/list', + method: 'POST', + primaryKey: [['id']], + perCustomer: true, + jsonSchema: { + type: 'object', + properties: { + id: { type: 'string' }, + customer_id: { type: 'string' }, + rate_card_id: { type: ['string', 'null'] }, + starting_at: { type: 'string' }, + ending_before: { type: ['string', 'null'] }, + name: { type: ['string', 'null'] }, + custom_fields: { type: 'object' }, + _synced_at: { type: 'integer' }, + }, + }, + }, + { + name: 'products', + endpoint: '/v1/contract-pricing/products/list', + method: 'POST', + primaryKey: [['id']], + jsonSchema: { + type: 'object', + properties: { + id: { type: 'string' }, + name: { type: 'string' }, + type: { type: 'string' }, + custom_fields: { type: 'object' }, + _synced_at: { type: 'integer' }, + }, + }, + }, + { + name: 'rate_cards', + endpoint: '/v1/contract-pricing/rate-cards/list', + method: 'POST', + primaryKey: [['id']], + jsonSchema: { + type: 'object', + properties: { + id: { type: 'string' }, + name: { type: 'string' }, + description: { type: ['string', 'null'] }, + custom_fields: { type: 'object' }, + _synced_at: { type: 'integer' }, + }, + }, + }, + { + name: 'credit_grants', + endpoint: '/v1/credits/listGrants', + method: 'POST', + primaryKey: [['id']], + jsonSchema: { + type: 'object', + properties: { + id: { type: 'string' }, + name: { type: 'string' }, + customer_id: { type: 'string' }, + reason: { type: ['string', 'null'] }, + effective_at: { type: 'string' }, + expires_at: { type: ['string', 'null'] }, + priority: { type: 'number' }, + credit_grant_type: { type: ['string', 'null'] }, + balance: { type: 'object' }, + custom_fields: { type: 'object' }, + _synced_at: { type: 'integer' }, + }, + }, + }, + { + name: 'invoices', + endpoint: '/v1/invoices', + method: 'GET', + primaryKey: [['id']], + jsonSchema: { + type: 'object', + properties: { + id: { type: 'string' }, + customer_id: { type: 'string' }, + status: { type: 'string' }, + total: { type: 'number' }, + credit_type: { type: 'object' }, + start_timestamp: { type: 'string' }, + end_timestamp: { type: 'string' }, + line_items: { type: 'array' }, + custom_fields: { type: 'object' }, + _synced_at: { type: 'integer' }, + }, + }, + }, + { + name: 'entitlements', + endpoint: '/v1/contracts/getContractRateSchedule', + method: 'POST', + primaryKey: [['customer_id'], ['contract_id'], ['product_id']], + perContract: true, + jsonSchema: { + type: 'object', + properties: { + customer_id: { type: 'string' }, + contract_id: { type: 'string' }, + product_id: { type: 'string' }, + product_name: { type: 'string' }, + product_tags: { type: 'array', items: { type: 'string' } }, + product_custom_fields: { type: 'object' }, + rate_card_id: { type: 'string' }, + entitled: { type: 'boolean' }, + starting_at: { type: 'string' }, + ending_before: { type: ['string', 'null'] }, + list_rate: { type: 'object' }, + override_rate: { type: 'object' }, + _synced_at: { type: 'integer' }, + }, + }, + }, +] diff --git a/packages/source-metronome/src/spec.ts b/packages/source-metronome/src/spec.ts new file mode 100644 index 000000000..7a9509a20 --- /dev/null +++ b/packages/source-metronome/src/spec.ts @@ -0,0 +1,48 @@ +import { z } from 'zod' +import type { ConnectorSpecification } from '@stripe/sync-protocol' + +export const configSchema = z.object({ + api_key: z.string().describe('Metronome API bearer token'), + base_url: z + .string() + .url() + .optional() + .describe('Override the Metronome API base URL (default: https://api.metronome.com)'), + rate_limit: z + .number() + .int() + .positive() + .optional() + .describe('Max requests per second (default: no limit)'), + backfill_limit: z + .number() + .int() + .positive() + .optional() + .describe('Max records to fetch per stream (useful for testing)'), + webhook_secret: z + .string() + .optional() + .describe('Webhook signing secret for HMAC-SHA256 signature verification'), + webhook_port: z + .number() + .int() + .optional() + .describe('Port for built-in webhook HTTP listener (e.g. 4243)'), +}) + +export type Config = z.infer + +export const streamStateSpec = z.object({ + next_page: z + .string() + .nullable() + .describe('Cursor token for pagination. Null means stream is complete.'), +}) + +export type StreamState = z.infer + +export default { + config: z.toJSONSchema(configSchema), + source_state_stream: z.toJSONSchema(streamStateSpec), +} satisfies ConnectorSpecification diff --git a/packages/source-metronome/src/webhook.ts b/packages/source-metronome/src/webhook.ts new file mode 100644 index 000000000..78b9bd5fc --- /dev/null +++ b/packages/source-metronome/src/webhook.ts @@ -0,0 +1,97 @@ +import { createHmac, timingSafeEqual } from 'node:crypto' +import http from 'node:http' +import { log } from './logger.js' + +/** + * Verify a Metronome webhook signature. + * Metronome signs: HMAC-SHA256(secret, date + "\n" + body) + * Header: Metronome-Webhook-Signature + */ +export function verifyWebhookSignature( + body: string, + signature: string, + date: string, + secret: string +): void { + const expected = createHmac('sha256', secret).update(`${date}\n${body}`).digest('hex') + + const sigBuffer = Buffer.from(signature, 'hex') + const expectedBuffer = Buffer.from(expected, 'hex') + + if (sigBuffer.length !== expectedBuffer.length || !timingSafeEqual(sigBuffer, expectedBuffer)) { + throw new Error('Webhook signature verification failed') + } +} + +export interface MetronomeWebhookEvent { + type: string + id?: string + customer_id?: string + contract_id?: string + timestamp?: string + properties?: Record +} + +export interface WebhookInput { + event: MetronomeWebhookEvent + raw_body: string + verified: boolean +} + +export type WebhookPushFn = (input: WebhookInput) => void + +/** + * Start an HTTP server that receives Metronome webhook events. + * Verifies signatures if a secret is provided, then pushes parsed events. + */ +export function startWebhookServer( + port: number, + secret: string | undefined, + push: WebhookPushFn +): http.Server { + const server = http.createServer((req, res) => { + if (req.method !== 'POST') { + res.writeHead(405).end() + return + } + + const chunks: Buffer[] = [] + req.on('data', (chunk: Buffer) => chunks.push(chunk)) + req.on('end', () => { + const body = Buffer.concat(chunks).toString('utf8') + + try { + let verified = false + if (secret) { + const signature = req.headers['metronome-webhook-signature'] as string | undefined + const date = req.headers['date'] as string | undefined + if (!signature || !date) { + res.writeHead(400).end('Missing Metronome-Webhook-Signature or Date header') + return + } + verifyWebhookSignature(body, signature, date, secret) + verified = true + } + + const event = JSON.parse(body) as MetronomeWebhookEvent + log.info( + { eventType: event.type, eventId: event.id, verified }, + 'metronome: webhook received' + ) + + push({ event, raw_body: body, verified }) + res.writeHead(200).end('{"received":true}') + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + log.error({ error: message }, 'metronome: webhook processing error') + res.writeHead(400).end(message) + } + }) + }) + + server.listen(port, () => { + log.info({ port }, 'metronome: webhook server listening') + }) + + return server +} diff --git a/packages/source-metronome/tsconfig.json b/packages/source-metronome/tsconfig.json new file mode 100644 index 000000000..2481fe545 --- /dev/null +++ b/packages/source-metronome/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts", "src/**/__tests__/**"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e6ae53cb3..8759b2401 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -164,6 +164,9 @@ importers: '@stripe/sync-destination-postgres': specifier: workspace:* version: link:../../packages/destination-postgres + '@stripe/sync-destination-redis': + specifier: workspace:* + version: link:../../packages/destination-redis '@stripe/sync-hono-zod-openapi': specifier: workspace:* version: link:../../packages/hono-zod-openapi @@ -173,6 +176,9 @@ importers: '@stripe/sync-protocol': specifier: workspace:* version: link:../../packages/protocol + '@stripe/sync-source-metronome': + specifier: workspace:* + version: link:../../packages/source-metronome '@stripe/sync-source-stripe': specifier: workspace:* version: link:../../packages/source-stripe @@ -529,6 +535,28 @@ importers: specifier: ^3.2.4 version: 3.2.4(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) + packages/destination-redis: + dependencies: + '@stripe/sync-logger': + specifier: workspace:* + version: link:../logger + '@stripe/sync-protocol': + specifier: workspace:* + version: link:../protocol + ioredis: + specifier: ^5.6.1 + version: 5.10.1 + zod: + specifier: ^4.3.6 + version: 4.3.6 + devDependencies: + '@types/node': + specifier: ^24.5.0 + version: 24.10.1 + vitest: + specifier: ^3.2.4 + version: 3.2.4(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) + packages/hono-zod-openapi: dependencies: '@hono/zod-validator': @@ -608,6 +636,25 @@ importers: specifier: ^3.2.1 version: 3.2.4(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) + packages/source-metronome: + dependencies: + '@stripe/sync-logger': + specifier: workspace:* + version: link:../logger + '@stripe/sync-protocol': + specifier: workspace:* + version: link:../protocol + zod: + specifier: ^4.3.6 + version: 4.3.6 + devDependencies: + '@types/node': + specifier: ^24.5.0 + version: 24.10.1 + vitest: + specifier: ^3.2.4 + version: 3.2.4(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) + packages/source-stripe: dependencies: '@stripe/sync-logger': @@ -1391,6 +1438,9 @@ packages: cpu: [x64] os: [win32] + '@ioredis/commands@1.5.1': + resolution: {integrity: sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==} + '@isaacs/balanced-match@4.0.1': resolution: {integrity: sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==} engines: {node: 20 || >=22} @@ -3005,6 +3055,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + code-excerpt@4.0.0: resolution: {integrity: sha512-xxodCmBen3iy2i0WtAK8FlFNrRzjUqjRsMfho58xT/wvZU1YTM3fCnRjcy1gJPMepaRlgm/0e6w8SpWHpn3/cA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -3116,6 +3170,10 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} @@ -3582,6 +3640,10 @@ packages: react-devtools-core: optional: true + ioredis@5.10.1: + resolution: {integrity: sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==} + engines: {node: '>=12.22.0'} + is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} @@ -3783,6 +3845,12 @@ packages: lodash.camelcase@4.3.0: resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + lodash.merge@4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} @@ -4225,6 +4293,14 @@ packages: resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} engines: {node: '>= 12.13.0'} + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} @@ -4380,6 +4456,9 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + std-env@3.9.0: resolution: {integrity: sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw==} @@ -5843,6 +5922,8 @@ snapshots: '@img/sharp-win32-x64@0.34.5': optional: true + '@ioredis/commands@1.5.1': {} + '@isaacs/balanced-match@4.0.1': {} '@isaacs/brace-expansion@5.0.0': @@ -7593,6 +7674,8 @@ snapshots: clsx@2.1.1: {} + cluster-key-slot@1.1.2: {} + code-excerpt@4.0.0: dependencies: convert-to-spaces: 2.0.1 @@ -7680,6 +7763,8 @@ snapshots: delayed-stream@1.0.0: {} + denque@2.1.0: {} + detect-libc@2.1.2: {} detect-node-es@1.1.0: {} @@ -8213,6 +8298,20 @@ snapshots: - bufferutil - utf-8-validate + ioredis@5.10.1: + dependencies: + '@ioredis/commands': 1.5.1 + cluster-key-slot: 1.1.2 + debug: 4.4.3(supports-color@10.2.2) + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + is-extglob@2.1.1: {} is-fullwidth-code-point@3.0.0: {} @@ -8367,6 +8466,10 @@ snapshots: lodash.camelcase@4.3.0: {} + lodash.defaults@4.2.0: {} + + lodash.isarguments@3.1.0: {} + lodash.merge@4.6.2: {} long@5.3.2: {} @@ -8787,6 +8890,12 @@ snapshots: real-require@0.2.0: {} + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + require-directory@2.1.1: {} require-from-string@2.0.2: {} @@ -9000,6 +9109,8 @@ snapshots: stackback@0.0.2: {} + standard-as-callback@2.1.0: {} + std-env@3.9.0: {} string-width@4.2.3: diff --git a/scripts/e2e-metronome-redis.sh b/scripts/e2e-metronome-redis.sh new file mode 100755 index 000000000..b748d811a --- /dev/null +++ b/scripts/e2e-metronome-redis.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# End-to-end test: Metronome → source-metronome → destination-redis +# +# Proves the full pipeline works with real data: +# 1. Backfill credit grants + entitlements to Redis +# 2. Start webhook listener +# 3. Simulate customer usage (send events to Metronome ingest API) +# 4. Fire a webhook event → source re-fetches → Redis updates +# 5. Check Redis reflects current credit balance +# +# Prerequisites: +# - METRONOME_API_TOKEN env var set +# - Redis running on localhost:56379 (docker compose up redis) +# - Customer + contract + credit grant already exist in Metronome sandbox +# +# Usage: ./scripts/e2e-metronome-redis.sh +set -euo pipefail + +: "${METRONOME_API_TOKEN:?Set METRONOME_API_TOKEN}" + +CUSTOMER_ID="1a6de34e-ec68-46b0-a1c3-bb3d49f66bb3" +GRANT_ID="30ec9faa-3c5d-4cea-9e2a-b44a4e4446bd" +REDIS_PORT=56379 +WEBHOOK_PORT=4243 +KEY_PREFIX="sync:" + +echo "=== E2E: Metronome → source-metronome → destination-redis ===" +echo "" + +# Verify Redis is running +if ! redis-cli -p "$REDIS_PORT" ping &>/dev/null; then + echo "ERROR: Redis not running on port $REDIS_PORT. Run: docker compose up redis -d" + exit 1 +fi + +redis-cli -p "$REDIS_PORT" FLUSHDB >/dev/null + +CATALOG='{"streams":[{"stream":{"name":"credit_grants","primary_key":[["id"]],"newer_than_field":"_synced_at","json_schema":{}},"sync_mode":"full_refresh","destination_sync_mode":"append_dedup"},{"stream":{"name":"entitlements","primary_key":[["customer_id"],["contract_id"],["product_id"]],"newer_than_field":"_synced_at","json_schema":{}},"sync_mode":"full_refresh","destination_sync_mode":"append_dedup"}]}' +SOURCE_CONFIG="{\"api_key\": \"$METRONOME_API_TOKEN\", \"webhook_port\": $WEBHOOK_PORT}" +DEST_CONFIG="{\"url\":\"redis://localhost:$REDIS_PORT\",\"key_prefix\":\"$KEY_PREFIX\",\"batch_size\":1}" + +# Step 1: Start pipeline (backfill + webhook server) +echo "Step 1: Starting pipeline (backfill + webhook listener on port $WEBHOOK_PORT)..." +npx tsx --conditions bun packages/source-metronome/src/bin.ts read \ + --config "$SOURCE_CONFIG" --catalog "$CATALOG" 2>/dev/null | \ +npx tsx --conditions bun packages/destination-redis/src/bin.ts write \ + --config "$DEST_CONFIG" --catalog "$CATALOG" >/dev/null 2>/dev/null & +PIPE_PID=$! +trap "kill $PIPE_PID 2>/dev/null; wait $PIPE_PID 2>/dev/null" EXIT +sleep 5 + +echo "Step 1: Backfill complete." +echo "" + +# Step 2: Check initial state +echo "Step 2: Initial Redis state after backfill:" +BALANCE_BEFORE=$(redis-cli -p "$REDIS_PORT" GET "${KEY_PREFIX}credit_grants:$GRANT_ID" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['balance']['including_pending'])") +SYNCED_BEFORE=$(redis-cli -p "$REDIS_PORT" GET "${KEY_PREFIX}credit_grants:$GRANT_ID" | python3 -c "import sys,json; print(json.load(sys.stdin)['_synced_at'])") +echo " Credit balance: $BALANCE_BEFORE" +echo " Synced at: $SYNCED_BEFORE" +echo "" + +# Step 3: Simulate customer usage +echo "Step 3: Simulating customer usage (5 API calls)..." +TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) +curl -s -X POST https://api.metronome.com/v1/ingest \ + -H "Authorization: Bearer $METRONOME_API_TOKEN" \ + -H "Content-Type: application/json" \ + -d "[ + {\"customer_id\": \"$CUSTOMER_ID\", \"event_type\": \"api_call\", \"timestamp\": \"$TS\", \"transaction_id\": \"e2e_$(date +%s)_1\"}, + {\"customer_id\": \"$CUSTOMER_ID\", \"event_type\": \"api_call\", \"timestamp\": \"$TS\", \"transaction_id\": \"e2e_$(date +%s)_2\"}, + {\"customer_id\": \"$CUSTOMER_ID\", \"event_type\": \"api_call\", \"timestamp\": \"$TS\", \"transaction_id\": \"e2e_$(date +%s)_3\"}, + {\"customer_id\": \"$CUSTOMER_ID\", \"event_type\": \"api_call\", \"timestamp\": \"$TS\", \"transaction_id\": \"e2e_$(date +%s)_4\"}, + {\"customer_id\": \"$CUSTOMER_ID\", \"event_type\": \"api_call\", \"timestamp\": \"$TS\", \"transaction_id\": \"e2e_$(date +%s)_5\"} + ]" >/dev/null +echo " Sent 5 usage events to Metronome." +echo "" + +# Step 4: Trigger webhook (simulates Metronome firing a credit event) +echo "Step 4: Firing credit.segment.end webhook..." + +# Verify webhook server is listening +if ! curl -s -o /dev/null -w "%{http_code}" -X POST "http://localhost:$WEBHOOK_PORT" \ + -H "Content-Type: application/json" \ + -d "{\"type\":\"credit.segment.end\",\"id\":\"evt_e2e_$(date +%s)\",\"customer_id\":\"$CUSTOMER_ID\"}" | grep -q "200"; then + echo " WARNING: Webhook server returned non-200" +fi +sleep 5 +echo " Webhook processed." +echo "" + +# Step 5: Verify Redis updated +echo "Step 5: Redis state after webhook refresh:" +BALANCE_AFTER=$(redis-cli -p "$REDIS_PORT" GET "${KEY_PREFIX}credit_grants:$GRANT_ID" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['balance']['including_pending'])") +SYNCED_AFTER=$(redis-cli -p "$REDIS_PORT" GET "${KEY_PREFIX}credit_grants:$GRANT_ID" | python3 -c "import sys,json; print(json.load(sys.stdin)['_synced_at'])") +echo " Credit balance: $BALANCE_AFTER" +echo " Synced at: $SYNCED_AFTER" +echo "" + +# Step 6: Verify timestamp changed (proves webhook triggered a re-fetch) +if [ "$SYNCED_AFTER" -gt "$SYNCED_BEFORE" ]; then + echo "✓ SUCCESS: Redis was updated by webhook (synced_at $SYNCED_BEFORE → $SYNCED_AFTER)" +else + echo "✗ FAIL: Redis was NOT updated by webhook" + exit 1 +fi + +echo "" +echo "=== All Redis keys ===" +redis-cli -p "$REDIS_PORT" KEYS "${KEY_PREFIX}*" +echo "" +echo "=== E2E complete ===" diff --git a/scripts/generate-openapi-specs.ts b/scripts/generate-openapi-specs.ts index 27936f4be..6981b19a0 100644 --- a/scripts/generate-openapi-specs.ts +++ b/scripts/generate-openapi-specs.ts @@ -9,8 +9,10 @@ import { createApp, createConnectorResolver } from '../apps/engine/src/index.js' import { createApp as createServiceApp } from '../apps/service/src/api/app.js' import { memoryPipelineStore } from '../apps/service/src/lib/stores-memory.js' import sourceStripe from '../packages/source-stripe/src/index.js' +import sourceMetronome from '../packages/source-metronome/src/index.js' import destinationPostgres from '../packages/destination-postgres/src/index.js' import destinationGoogleSheets from '../packages/destination-google-sheets/src/index.js' +import destinationRedis from '../packages/destination-redis/src/index.js' const [engineOut, serviceOut] = process.argv.slice(2) if (!engineOut || !serviceOut) { @@ -19,10 +21,14 @@ if (!engineOut || !serviceOut) { } const resolver = await createConnectorResolver({ - sources: { stripe: (sourceStripe as any).default ?? sourceStripe }, + sources: { + stripe: (sourceStripe as any).default ?? sourceStripe, + metronome: (sourceMetronome as any).default ?? sourceMetronome, + }, destinations: { postgres: (destinationPostgres as any).default ?? destinationPostgres, google_sheets: (destinationGoogleSheets as any).default ?? destinationGoogleSheets, + redis: (destinationRedis as any).default ?? destinationRedis, }, }) From 5d351d5f74cfac2d80b6938afaffcf0dda9793d7 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 30 Apr 2026 11:23:34 -0700 Subject: [PATCH 02/11] =?UTF-8?q?feat:=20add=20pixel-app=20demo=20with=20M?= =?UTF-8?q?etronome=E2=86=92Redis=20entitlement=20checks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PixelDraw demo app: each pixel drawn sends a usage event to Metronome, credit balance is gated via Metronome-synced data in Redis (no local state). The sync pipeline keeps Redis up to date via webhooks. - Express server with /api/draw (hot path) and /api/credits - Canvas UI with color picker and live credit balance display - Usage events sent to Metronome ingest API per pixel - Balance checked from Metronome-synced credit_grants in Redis only Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- examples/pixel-app/package-lock.json | 953 +++++++++++++++++++++++++++ examples/pixel-app/package.json | 14 + examples/pixel-app/public/index.html | 216 ++++++ examples/pixel-app/server.js | 184 ++++++ scripts/webhook-relay.sh | 55 ++ 5 files changed, 1422 insertions(+) create mode 100644 examples/pixel-app/package-lock.json create mode 100644 examples/pixel-app/package.json create mode 100644 examples/pixel-app/public/index.html create mode 100644 examples/pixel-app/server.js create mode 100644 scripts/webhook-relay.sh diff --git a/examples/pixel-app/package-lock.json b/examples/pixel-app/package-lock.json new file mode 100644 index 000000000..52263cd72 --- /dev/null +++ b/examples/pixel-app/package-lock.json @@ -0,0 +1,953 @@ +{ + "name": "pixeldraw-metronome-redis", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "pixeldraw-metronome-redis", + "version": "1.0.0", + "dependencies": { + "express": "^4.21.0", + "ioredis": "^5.6.1" + } + }, + "node_modules/@ioredis/commands": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.5.1.tgz", + "integrity": "sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==", + "license": "MIT" + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "1.20.5", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", + "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/qs": { + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.14.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", + "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ioredis": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.1.tgz", + "integrity": "sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.5.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ioredis/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + } + } +} diff --git a/examples/pixel-app/package.json b/examples/pixel-app/package.json new file mode 100644 index 000000000..ac388eab4 --- /dev/null +++ b/examples/pixel-app/package.json @@ -0,0 +1,14 @@ +{ + "name": "pixeldraw-metronome-redis", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "start": "node server.js", + "sync": "node sync.js" + }, + "dependencies": { + "express": "^4.21.0", + "ioredis": "^5.6.1" + } +} diff --git a/examples/pixel-app/public/index.html b/examples/pixel-app/public/index.html new file mode 100644 index 000000000..2948e6d0b --- /dev/null +++ b/examples/pixel-app/public/index.html @@ -0,0 +1,216 @@ + + + + + + PixelDraw — Metronome + Redis + + + +
+
+

PixelDraw

+
Metronome + Redis entitlements
+
+
+

Credits

+
—
+
remaining credits
+
+
+
+
Out of credits! Drawing disabled.
+
+
+

Colors

+
+
+
+ + +
+
+
+ +
Ready
+
+ + + + diff --git a/examples/pixel-app/server.js b/examples/pixel-app/server.js new file mode 100644 index 000000000..5f01736be --- /dev/null +++ b/examples/pixel-app/server.js @@ -0,0 +1,184 @@ +/** + * PixelDraw — Metronome + Redis entitlement demo. + * + * Each pixel drawn sends a usage event to Metronome (color = event type). + * Credit balance is checked in Redis — synced from Metronome via sync-engine. + * NO local state in Redis. The only data is replicated from Metronome. + * + * Architecture: + * Browser → POST /api/draw → check Metronome-synced Redis balance → send usage to Metronome + * Metronome → webhook → source-metronome → destination-redis (keeps Redis fresh) + * + * Env vars: + * METRONOME_API_TOKEN — Metronome bearer token + * METRONOME_CUSTOMER_ID — Customer ID in Metronome + * REDIS_URL — Redis connection (default: redis://localhost:56379) + * PORT — Server port (default: 4000) + */ + +import express from 'express' +import { Redis } from 'ioredis' +import { fileURLToPath } from 'node:url' +import { dirname, join } from 'node:path' + +const __dirname = dirname(fileURLToPath(import.meta.url)) + +const app = express() +app.use(express.json()) +app.use(express.static(join(__dirname, 'public'))) + +const PORT = process.env.PORT || 4000 +const METRONOME_API_TOKEN = process.env.METRONOME_API_TOKEN +const METRONOME_CUSTOMER_ID = process.env.METRONOME_CUSTOMER_ID +const METRONOME_BASE_URL = process.env.METRONOME_BASE_URL || 'https://api.metronome.com' +const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:56379' +const KEY_PREFIX = process.env.KEY_PREFIX || 'sync:' + +if (!METRONOME_API_TOKEN) { + console.error('ERROR: Set METRONOME_API_TOKEN') + process.exit(1) +} +if (!METRONOME_CUSTOMER_ID) { + console.error('ERROR: Set METRONOME_CUSTOMER_ID') + process.exit(1) +} + +const redis = new Redis(REDIS_URL) + +// ---- Redis reads (Metronome-synced data only) ---- + +/** Get credit balance from Metronome-synced grant data in Redis */ +async function getCreditBalance() { + const keys = await scanKeys(`${KEY_PREFIX}credit_grants:*`) + let balance = 0 + + for (const key of keys) { + const raw = await redis.get(key) + if (!raw) continue + const grant = JSON.parse(raw) + if (grant.customer_id !== METRONOME_CUSTOMER_ID) continue + balance += grant.balance?.including_pending ?? 0 + } + + return balance +} + +/** Get entitlement for a specific product from Redis */ +async function getEntitlement(productName) { + const keys = await scanKeys(`${KEY_PREFIX}entitlements:${METRONOME_CUSTOMER_ID}:*`) + for (const key of keys) { + const raw = await redis.get(key) + if (!raw) continue + const ent = JSON.parse(raw) + if (ent.product_name === productName) { + return ent + } + } + return null +} + +/** Scan Redis keys matching a pattern */ +async function scanKeys(pattern) { + const keys = [] + let cursor = '0' + do { + const [next, batch] = await redis.scan(cursor, 'MATCH', pattern, 'COUNT', 100) + cursor = next + keys.push(...batch) + } while (cursor !== '0') + return keys +} + +// ---- Metronome usage ingestion ---- + +async function ingestUsage(color) { + const res = await fetch(`${METRONOME_BASE_URL}/v1/ingest`, { + method: 'POST', + headers: { + Authorization: `Bearer ${METRONOME_API_TOKEN}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify([ + { + customer_id: METRONOME_CUSTOMER_ID, + event_type: 'pixel_draw', + timestamp: new Date().toISOString(), + transaction_id: `px_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`, + properties: { color }, + }, + ]), + }) + + if (!res.ok) { + const text = await res.text() + throw new Error(`Metronome ingest failed: ${res.status} ${text}`) + } +} + +// ---- API routes ---- + +/** Health check */ +app.get('/api/health', async (_req, res) => { + try { + await redis.ping() + res.json({ ok: true, redis: 'connected' }) + } catch { + res.status(503).json({ ok: false, redis: 'disconnected' }) + } +}) + +/** Get current credit balance + entitlements from Metronome-synced Redis */ +app.get('/api/credits', async (_req, res) => { + const balance = await getCreditBalance() + const entitlement = await getEntitlement('API Access') + res.json({ + balance, + entitled: entitlement?.entitled ?? false, + product: entitlement?.product_name ?? null, + }) +}) + +/** Draw a pixel — the hot path */ +app.post('/api/draw', async (req, res) => { + const { color, x, y } = req.body + if (!color || x == null || y == null) { + return res.status(400).json({ error: 'color, x, y required' }) + } + + // 1. Check Metronome-synced credit balance in Redis + const balance = await getCreditBalance() + if (balance <= 0) { + return res.status(402).json({ + allowed: false, + error: 'Out of credits', + balance: 0, + }) + } + + // 2. Send usage event to Metronome (async, don't block response) + ingestUsage(color).catch((err) => { + console.error('Usage ingest error:', err.message) + }) + + res.json({ + allowed: true, + balance, + color, + x, + y, + }) +}) + +// ---- Start ---- + +app.listen(PORT, () => { + console.log(` ++==================================================+ +| PixelDraw — http://localhost:${PORT} | +| Metronome customer: ${METRONOME_CUSTOMER_ID.slice(0, 20)}... | +| Redis: ${REDIS_URL.padEnd(42)}| +| Balance: Metronome-synced only (no local state) | +| Usage sent to Metronome (async) | ++==================================================+ + `) +}) diff --git a/scripts/webhook-relay.sh b/scripts/webhook-relay.sh new file mode 100644 index 000000000..a424367ef --- /dev/null +++ b/scripts/webhook-relay.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Poll webhook.site for new requests and forward them to a local server. +# Usage: ./scripts/webhook-relay.sh +set -euo pipefail + +TOKEN="${1:?Usage: webhook-relay.sh }" +TARGET="${2:-http://localhost:4243}" +SEEN="" + +echo "Relaying webhook.site/$TOKEN → $TARGET" +echo "Polling every 2 seconds..." + +while true; do + REQUESTS=$(curl -s "https://webhook.site/token/$TOKEN/requests?sorting=newest&per_page=5" 2>/dev/null) + + # Extract request UUIDs and process new ones + echo "$REQUESTS" | python3 -c " +import sys, json, subprocess + +data = json.load(sys.stdin) +seen = set('''$SEEN'''.split()) + +for req in reversed(data.get('data', [])): + uuid = req['uuid'] + if uuid in seen: + continue + + # Forward the request body + headers to target + body = req.get('content', '') or '{}' + headers = req.get('headers', {}) + + cmd = ['curl', '-s', '-X', 'POST', '$TARGET', '-H', 'Content-Type: application/json'] + + # Forward relevant headers + for key in ['date', 'metronome-webhook-signature']: + for hdr_key, hdr_vals in headers.items(): + if hdr_key.lower() == key and hdr_vals: + val = hdr_vals[0] if isinstance(hdr_vals, list) else hdr_vals + cmd.extend(['-H', f'{hdr_key}: {val}']) + + cmd.extend(['-d', body]) + + result = subprocess.run(cmd, capture_output=True, text=True) + print(f'Relayed {uuid[:8]}... → {result.stdout[:80]}') + print(uuid) # Print UUID so we can track it +" 2>/dev/null | while IFS= read -r line; do + if [[ "$line" =~ ^[0-9a-f]{8}-[0-9a-f]{4} ]]; then + SEEN="$SEEN $line" + else + echo "$line" + fi + done + + sleep 2 +done From f2ca0c3f47b9b95164ba03a716e70191abb68436 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 30 Apr 2026 11:24:02 -0700 Subject: [PATCH 03/11] chore: regenerate OpenAPI specs with webhook config fields Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- apps/engine/src/__generated__/openapi.d.ts | 4 ++++ apps/engine/src/__generated__/openapi.json | 10 ++++++++++ apps/service/src/__generated__/openapi.d.ts | 4 ++++ apps/service/src/__generated__/openapi.json | 10 ++++++++++ 4 files changed, 28 insertions(+) diff --git a/apps/engine/src/__generated__/openapi.d.ts b/apps/engine/src/__generated__/openapi.d.ts index e0dd0a446..3d2b1e52d 100644 --- a/apps/engine/src/__generated__/openapi.d.ts +++ b/apps/engine/src/__generated__/openapi.d.ts @@ -344,6 +344,10 @@ export interface components { rate_limit?: number; /** @description Max records to fetch per stream (useful for testing) */ backfill_limit?: number; + /** @description Webhook signing secret for HMAC-SHA256 signature verification */ + webhook_secret?: string; + /** @description Port for built-in webhook HTTP listener (e.g. 4243) */ + webhook_port?: number; }; DestinationConfig: { /** @constant */ diff --git a/apps/engine/src/__generated__/openapi.json b/apps/engine/src/__generated__/openapi.json index 5002c7531..58e8938ce 100644 --- a/apps/engine/src/__generated__/openapi.json +++ b/apps/engine/src/__generated__/openapi.json @@ -1145,6 +1145,16 @@ "exclusiveMinimum": 0, "maximum": 9007199254740991, "description": "Max records to fetch per stream (useful for testing)" + }, + "webhook_secret": { + "type": "string", + "description": "Webhook signing secret for HMAC-SHA256 signature verification" + }, + "webhook_port": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Port for built-in webhook HTTP listener (e.g. 4243)" } }, "required": [ diff --git a/apps/service/src/__generated__/openapi.d.ts b/apps/service/src/__generated__/openapi.d.ts index f5c6b9022..8fce9e1d8 100644 --- a/apps/service/src/__generated__/openapi.d.ts +++ b/apps/service/src/__generated__/openapi.d.ts @@ -279,6 +279,10 @@ export interface components { rate_limit?: number; /** @description Max records to fetch per stream (useful for testing) */ backfill_limit?: number; + /** @description Webhook signing secret for HMAC-SHA256 signature verification */ + webhook_secret?: string; + /** @description Port for built-in webhook HTTP listener (e.g. 4243) */ + webhook_port?: number; }; DestinationConfig: { /** @constant */ diff --git a/apps/service/src/__generated__/openapi.json b/apps/service/src/__generated__/openapi.json index 8ebb73b80..3ed1e400d 100644 --- a/apps/service/src/__generated__/openapi.json +++ b/apps/service/src/__generated__/openapi.json @@ -1356,6 +1356,16 @@ "exclusiveMinimum": 0, "maximum": 9007199254740991, "description": "Max records to fetch per stream (useful for testing)" + }, + "webhook_secret": { + "type": "string", + "description": "Webhook signing secret for HMAC-SHA256 signature verification" + }, + "webhook_port": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Port for built-in webhook HTTP listener (e.g. 4243)" } }, "required": [ From eeb4dd918a97e01f72e345d79d9382c78b172a05 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Tue, 5 May 2026 14:35:44 -0700 Subject: [PATCH 04/11] feat: add destination-sqlite package using Node.js built-in node:sqlite Zero-dependency SQLite destination connector that uses Node.js 24's built-in node:sqlite module (DatabaseSync). Supports upsert with newer-than deduplication, hard deletes, and auto-creates tables on first write. Registered in both the engine default connectors and the service CLI. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- apps/engine/package.json | 1 + apps/engine/src/lib/default-connectors.ts | 2 + apps/service/package.json | 1 + apps/service/src/cli.ts | 3 +- packages/destination-sqlite/package.json | 29 ++ packages/destination-sqlite/src/bin.ts | 6 + packages/destination-sqlite/src/index.ts | 321 +++++++++++++++++++ packages/destination-sqlite/src/logger.ts | 4 + packages/destination-sqlite/src/spec.ts | 13 + packages/destination-sqlite/tsconfig.json | 9 + packages/destination-sqlite/vitest.config.ts | 7 + pnpm-lock.yaml | 18 ++ 12 files changed, 413 insertions(+), 1 deletion(-) create mode 100644 packages/destination-sqlite/package.json create mode 100644 packages/destination-sqlite/src/bin.ts create mode 100644 packages/destination-sqlite/src/index.ts create mode 100644 packages/destination-sqlite/src/logger.ts create mode 100644 packages/destination-sqlite/src/spec.ts create mode 100644 packages/destination-sqlite/tsconfig.json create mode 100644 packages/destination-sqlite/vitest.config.ts diff --git a/apps/engine/package.json b/apps/engine/package.json index 78ec30e3f..5d0692eaa 100644 --- a/apps/engine/package.json +++ b/apps/engine/package.json @@ -51,6 +51,7 @@ "@scalar/hono-api-reference": "^0.6", "@stripe/sync-destination-google-sheets": "workspace:*", "@stripe/sync-destination-postgres": "workspace:*", + "@stripe/sync-destination-sqlite": "workspace:*", "@stripe/sync-hono-zod-openapi": "workspace:*", "@stripe/sync-logger": "workspace:*", "@stripe/sync-protocol": "workspace:*", diff --git a/apps/engine/src/lib/default-connectors.ts b/apps/engine/src/lib/default-connectors.ts index a3414b114..add1ec088 100644 --- a/apps/engine/src/lib/default-connectors.ts +++ b/apps/engine/src/lib/default-connectors.ts @@ -1,5 +1,6 @@ import sourceStripe from '@stripe/sync-source-stripe' import destinationPostgres from '@stripe/sync-destination-postgres' +import destinationSqlite from '@stripe/sync-destination-sqlite' import destinationGoogleSheets from '@stripe/sync-destination-google-sheets' import type { RegisteredConnectors } from './resolver.js' @@ -8,6 +9,7 @@ export const defaultConnectors: RegisteredConnectors = { sources: { stripe: sourceStripe }, destinations: { postgres: destinationPostgres, + sqlite: destinationSqlite, google_sheets: destinationGoogleSheets, }, } diff --git a/apps/service/package.json b/apps/service/package.json index 83530916a..798b24091 100644 --- a/apps/service/package.json +++ b/apps/service/package.json @@ -32,6 +32,7 @@ "@scalar/hono-api-reference": "^0.6", "@stripe/sync-destination-google-sheets": "workspace:*", "@stripe/sync-destination-postgres": "workspace:*", + "@stripe/sync-destination-sqlite": "workspace:*", "@stripe/sync-engine": "workspace:*", "@stripe/sync-hono-zod-openapi": "workspace:*", "@stripe/sync-logger": "workspace:*", diff --git a/apps/service/src/cli.ts b/apps/service/src/cli.ts index 4c7b92e12..c3487db69 100644 --- a/apps/service/src/cli.ts +++ b/apps/service/src/cli.ts @@ -9,6 +9,7 @@ import { serve } from '@hono/node-server' import { createConnectorResolver, startApiServer, type ApiServerHandle } from '@stripe/sync-engine' import sourceStripe from '@stripe/sync-source-stripe' import destinationPostgres from '@stripe/sync-destination-postgres' +import destinationSqlite from '@stripe/sync-destination-sqlite' import destinationGoogleSheets from '@stripe/sync-destination-google-sheets' import { createApp } from './api/app.js' import { @@ -27,7 +28,7 @@ const defaultDataDir = process.env.DATA_DIR ?? `${homedir()}/.stripe-sync` const resolverPromise = createConnectorResolver({ sources: { stripe: sourceStripe }, - destinations: { postgres: destinationPostgres, google_sheets: destinationGoogleSheets }, + destinations: { postgres: destinationPostgres, sqlite: destinationSqlite, google_sheets: destinationGoogleSheets }, }) async function buildCliSpec() { diff --git a/packages/destination-sqlite/package.json b/packages/destination-sqlite/package.json new file mode 100644 index 000000000..6e70671b0 --- /dev/null +++ b/packages/destination-sqlite/package.json @@ -0,0 +1,29 @@ +{ + "name": "@stripe/sync-destination-sqlite", + "version": "0.1.0", + "private": false, + "type": "module", + "exports": { + ".": { + "bun": "./src/index.ts", + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "bin": { + "destination-sqlite": "./dist/bin.js" + }, + "scripts": { + "build": "tsc", + "test": "vitest" + }, + "files": [ + "src", + "dist" + ], + "dependencies": { + "@stripe/sync-logger": "workspace:*", + "@stripe/sync-protocol": "workspace:*", + "zod": "^4.3.6" + } +} diff --git a/packages/destination-sqlite/src/bin.ts b/packages/destination-sqlite/src/bin.ts new file mode 100644 index 000000000..80991693b --- /dev/null +++ b/packages/destination-sqlite/src/bin.ts @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import connector from './index.js' +import { configSchema } from './spec.js' +import { runConnectorCli } from '@stripe/sync-protocol/cli' + +runConnectorCli(connector, { name: 'destination-sqlite', configSchema }) diff --git a/packages/destination-sqlite/src/index.ts b/packages/destination-sqlite/src/index.ts new file mode 100644 index 000000000..116cc81b3 --- /dev/null +++ b/packages/destination-sqlite/src/index.ts @@ -0,0 +1,321 @@ +import { DatabaseSync } from 'node:sqlite' +import { mkdirSync } from 'node:fs' +import { dirname } from 'node:path' +import type { Destination } from '@stripe/sync-protocol' +import defaultSpec from './spec.js' +import { log } from './logger.js' +import type { Config } from './spec.js' + +export { configSchema, type Config } from './spec.js' + +function quoteIdent(value: string): string { + return `"${value.replaceAll('"', '""')}"` +} + +function openDatabase(config: Config): DatabaseSync { + if (config.path !== ':memory:') { + mkdirSync(dirname(config.path), { recursive: true }) + } + const db = new DatabaseSync(config.path) + db.exec('PRAGMA journal_mode = WAL') + db.exec('PRAGMA synchronous = NORMAL') + db.exec('PRAGMA busy_timeout = 5000') + return db +} + +function buildCreateTableSQL(tableName: string): string { + const qt = quoteIdent(tableName) + return `CREATE TABLE IF NOT EXISTS ${qt} ( + id TEXT NOT NULL PRIMARY KEY, + _raw_data TEXT NOT NULL, + _synced_at TEXT NOT NULL, + _updated_at TEXT NOT NULL +)` +} + +function buildUpsertSQL( + tableName: string, + entries: Record[], + primaryKeyColumns: string[], + newerThanField: string +): { sql: string; params: unknown[] } { + if (entries.length === 0) return { sql: '', params: [] } + + const qt = quoteIdent(tableName) + const pkCols = primaryKeyColumns.map(quoteIdent).join(', ') + const syncedAt = new Date().toISOString() + const params: unknown[] = [] + const valueRows: string[] = [] + + for (const entry of entries) { + const ts = entry[newerThanField] as number + const updatedAt = new Date(ts * 1000).toISOString() + const pkValues = primaryKeyColumns.map((pk) => String(entry[pk] ?? '')) + + for (const pk of pkValues) params.push(pk) + params.push(JSON.stringify(entry)) + params.push(syncedAt) + params.push(updatedAt) + + const placeholders = Array.from( + { length: primaryKeyColumns.length + 3 }, + (_, i) => `?` + ).join(', ') + valueRows.push(`(${placeholders})`) + } + + const allCols = [...primaryKeyColumns.map(quoteIdent), '"_raw_data"', '"_synced_at"', '"_updated_at"'] + + const sql = `INSERT INTO ${qt} (${allCols.join(', ')}) +VALUES ${valueRows.join(',\n')} +ON CONFLICT(${pkCols}) DO UPDATE SET + "_raw_data" = excluded."_raw_data", + "_synced_at" = excluded."_synced_at", + "_updated_at" = excluded."_updated_at" +WHERE json_extract(excluded."_raw_data", '$.${newerThanField}') >= json_extract(${qt}."_raw_data", '$.${newerThanField}')` + + return { sql, params } +} + +function buildDeleteSQL( + tableName: string, + entries: Record[], + primaryKeyColumns: string[] +): { sql: string; params: unknown[] } { + if (entries.length === 0) return { sql: '', params: [] } + + const qt = quoteIdent(tableName) + const params: unknown[] = [] + const conditions: string[] = [] + + for (const entry of entries) { + const pkConditions = primaryKeyColumns.map((pk) => { + params.push(String(entry[pk] ?? '')) + return `${quoteIdent(pk)} = ?` + }) + conditions.push(`(${pkConditions.join(' AND ')})`) + } + + const sql = `DELETE FROM ${qt} WHERE ${conditions.join(' OR ')}` + return { sql, params } +} + +export interface WriteManyResult { + written_count: number + deleted_count: number +} + +function writeMany( + db: DatabaseSync, + tableName: string, + entries: Record[], + primaryKeyColumns: string[], + newerThanField: string +): WriteManyResult { + const tombstones = entries.filter((e) => e.recordDeleted === true).map((r) => r.data as Record) + const liveRecords = entries.filter((e) => e.recordDeleted !== true).map((r) => r.data as Record) + + let written_count = 0 + let deleted_count = 0 + + if (liveRecords.length > 0) { + const { sql, params } = buildUpsertSQL(tableName, liveRecords, primaryKeyColumns, newerThanField) + if (sql) { + db.prepare(sql).run(...(params as Array)) + written_count = liveRecords.length + } + } + + if (tombstones.length > 0) { + const { sql, params } = buildDeleteSQL(tableName, tombstones, primaryKeyColumns) + if (sql) { + db.prepare(sql).run(...(params as Array)) + deleted_count = tombstones.length + } + } + + return { written_count, deleted_count } +} + +const destination = { + async *spec() { + yield { type: 'spec' as const, spec: defaultSpec } + }, + + async *check({ config }) { + try { + const db = openDatabase(config) + db.exec('SELECT 1') + db.close() + yield { + type: 'connection_status' as const, + connection_status: { status: 'succeeded' as const }, + } + } catch (err) { + yield { + type: 'connection_status' as const, + connection_status: { + status: 'failed' as const, + message: err instanceof Error ? err.message : String(err), + }, + } + } + }, + + async *setup({ config, catalog }) { + const db = openDatabase(config) + try { + log.info(`Creating ${catalog.streams.length} tables in ${config.path}`) + for (const cs of catalog.streams) { + const pkFields = (cs.stream.primary_key ?? [['id']]).map((pk) => pk[0]) + const pkCols = pkFields.map(quoteIdent).join(', ') + const qt = quoteIdent(cs.stream.name) + + db.exec(`CREATE TABLE IF NOT EXISTS ${qt} ( + ${pkFields.map((f) => `${quoteIdent(f)} TEXT NOT NULL`).join(',\n ')}, + "_raw_data" TEXT NOT NULL, + "_synced_at" TEXT NOT NULL, + "_updated_at" TEXT NOT NULL, + PRIMARY KEY (${pkCols}) +)`) + } + log.info('Setup complete') + } finally { + db.close() + } + }, + + async *teardown({ config }) { + const db = openDatabase(config) + try { + const tables = db + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'") + .all() as Array<{ name: string }> + for (const { name } of tables) { + db.exec(`DROP TABLE IF EXISTS ${quoteIdent(name)}`) + } + } finally { + db.close() + } + }, + + async *write({ config, catalog }, $stdin) { + const db = openDatabase(config) + const batchSize = config.batch_size + + // Auto-create tables (idempotent) + for (const cs of catalog.streams) { + const pkFields = (cs.stream.primary_key ?? [['id']]).map((pk) => pk[0]) + const pkCols = pkFields.map(quoteIdent).join(', ') + const qt = quoteIdent(cs.stream.name) + db.exec(`CREATE TABLE IF NOT EXISTS ${qt} ( + ${pkFields.map((f) => `${quoteIdent(f)} TEXT NOT NULL`).join(',\n ')}, + "_raw_data" TEXT NOT NULL, + "_synced_at" TEXT NOT NULL, + "_updated_at" TEXT NOT NULL, + PRIMARY KEY (${pkCols}) +)`) + } + + const streamBuffers = new Map[]>() + const streamKeyColumns = new Map( + catalog.streams.map((cs) => [ + cs.stream.name, + cs.stream.primary_key?.map((pk) => pk[0]) ?? ['id'], + ]) + ) + const streamNewerThanField = new Map( + catalog.streams.map((cs) => [cs.stream.name, cs.stream.newer_than_field]) + ) + const failedStreams = new Set() + + const flushStream = (streamName: string): string | undefined => { + if (failedStreams.has(streamName)) return undefined + const buffer = streamBuffers.get(streamName) + if (!buffer || buffer.length === 0) return undefined + const pk = streamKeyColumns.get(streamName) ?? ['id'] + const newerThan = streamNewerThanField.get(streamName)! + + try { + const stats = writeMany(db, streamName, buffer, pk, newerThan) + log.debug( + { + stream: streamName, + batch_size: buffer.length, + written: stats.written_count, + deleted: stats.deleted_count, + }, + `dest write: upsert ${streamName}` + ) + } catch (err) { + const errMsg = err instanceof Error ? err.message : String(err) + log.error({ stream: streamName, error: errMsg }, 'dest write: flush failed') + failedStreams.add(streamName) + streamBuffers.set(streamName, []) + return errMsg + } + streamBuffers.set(streamName, []) + return undefined + } + + function streamError(stream: string, error: string) { + return { + type: 'stream_status' as const, + stream_status: { stream, status: 'error' as const, error }, + } + } + + try { + for await (const msg of $stdin) { + if (msg.type === 'record') { + const { stream } = msg.record + if (failedStreams.has(stream)) continue + + if (!streamBuffers.has(stream)) streamBuffers.set(stream, []) + const buffer = streamBuffers.get(stream)! + buffer.push(msg.record as Record) + + if (buffer.length >= batchSize) { + const err = flushStream(stream) + if (err) { + yield streamError(stream, err) + continue + } + } + yield msg + } else if (msg.type === 'source_state') { + if (msg.source_state.state_type !== 'global') { + const stream = msg.source_state.stream + if (failedStreams.has(stream)) continue + const err = flushStream(stream) + if (err) { + yield streamError(stream, err) + continue + } + } + yield msg + } else { + yield msg + } + } + + for (const streamName of streamBuffers.keys()) { + const err = flushStream(streamName) + if (err) yield streamError(streamName, err) + } + + if (failedStreams.size > 0) { + log.error( + { failed_streams: [...failedStreams] }, + `SQLite destination: completed with ${failedStreams.size} failed stream(s)` + ) + } else { + log.debug(`SQLite destination: wrote to ${config.path}`) + } + } finally { + db.close() + } + }, +} satisfies Destination + +export default destination diff --git a/packages/destination-sqlite/src/logger.ts b/packages/destination-sqlite/src/logger.ts new file mode 100644 index 000000000..4f4f74834 --- /dev/null +++ b/packages/destination-sqlite/src/logger.ts @@ -0,0 +1,4 @@ +import { createLogger } from '@stripe/sync-logger' +import type { Logger } from '@stripe/sync-logger' + +export const log: Logger = createLogger({ name: 'destination-sqlite' }) diff --git a/packages/destination-sqlite/src/spec.ts b/packages/destination-sqlite/src/spec.ts new file mode 100644 index 000000000..114a8cfa7 --- /dev/null +++ b/packages/destination-sqlite/src/spec.ts @@ -0,0 +1,13 @@ +import { z } from 'zod' +import type { ConnectorSpecification } from '@stripe/sync-protocol' + +export const configSchema = z.object({ + path: z.string().describe('Path to the SQLite database file (use ":memory:" for in-memory)'), + batch_size: z.number().default(100).describe('Records to buffer before flushing'), +}) + +export type Config = z.infer + +export default { + config: z.toJSONSchema(configSchema), +} satisfies ConnectorSpecification diff --git a/packages/destination-sqlite/tsconfig.json b/packages/destination-sqlite/tsconfig.json new file mode 100644 index 000000000..2481fe545 --- /dev/null +++ b/packages/destination-sqlite/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts", "src/**/__tests__/**"] +} diff --git a/packages/destination-sqlite/vitest.config.ts b/packages/destination-sqlite/vitest.config.ts new file mode 100644 index 000000000..2b1c323fe --- /dev/null +++ b/packages/destination-sqlite/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + environment: 'node', + }, +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e6ae53cb3..d2955c56b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -164,6 +164,9 @@ importers: '@stripe/sync-destination-postgres': specifier: workspace:* version: link:../../packages/destination-postgres + '@stripe/sync-destination-sqlite': + specifier: workspace:* + version: link:../../packages/destination-sqlite '@stripe/sync-hono-zod-openapi': specifier: workspace:* version: link:../../packages/hono-zod-openapi @@ -249,6 +252,9 @@ importers: '@stripe/sync-destination-postgres': specifier: workspace:* version: link:../../packages/destination-postgres + '@stripe/sync-destination-sqlite': + specifier: workspace:* + version: link:../../packages/destination-sqlite '@stripe/sync-engine': specifier: workspace:* version: link:../engine @@ -529,6 +535,18 @@ importers: specifier: ^3.2.4 version: 3.2.4(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) + packages/destination-sqlite: + dependencies: + '@stripe/sync-logger': + specifier: workspace:* + version: link:../logger + '@stripe/sync-protocol': + specifier: workspace:* + version: link:../protocol + zod: + specifier: ^4.3.6 + version: 4.3.6 + packages/hono-zod-openapi: dependencies: '@hono/zod-validator': From d847199c29cf1a124c32d3a71832c85f8d0f5936 Mon Sep 17 00:00:00 2001 From: kdhillon-stripe <243457111+kdhillon-stripe@users.noreply.github.com> Date: Wed, 6 May 2026 23:54:43 -0400 Subject: [PATCH 05/11] Fix reverse ETL progress and connector schemas (#362) * Fix reverse ETL connector progress and schemas Committed-By-Agent: cursor * Fix reverse ETL regression with source schema and status handling Fix source-postgres connector specification so conformance sees a valid JSON Schema shape for `config`, prevent destination-stripe from forwarding non-error stream status once a stream has failed, and make the reverse ETL custom-object double-run test state progression deterministic. Committed-By-Agent: cursor --- apps/engine/src/__generated__/openapi.d.ts | 200 ++++++- apps/engine/src/__generated__/openapi.json | 507 +++++++++++++++++- apps/engine/src/lib/createSchemas.test.ts | 98 ++++ apps/engine/src/lib/createSchemas.ts | 17 +- apps/engine/src/lib/resolver.test.ts | 38 ++ apps/engine/src/lib/resolver.ts | 7 +- apps/engine/src/lib/reverse-etl.test.ts | 2 +- apps/service/package.json | 2 + apps/service/src/__generated__/openapi.d.ts | 196 +++++++ apps/service/src/__generated__/openapi.json | 495 +++++++++++++++++ apps/service/src/cli.test.ts | 32 +- apps/service/src/cli.ts | 10 +- .../src/lib/cli-connector-shorthand.test.ts | 56 +- .../src/lib/cli-connector-shorthand.ts | 110 ++-- apps/service/src/lib/createSchemas.test.ts | 99 ++++ apps/service/src/lib/createSchemas.ts | 13 +- e2e/reverse-etl-demo-loop.ts | 49 +- packages/destination-stripe/src/index.ts | 6 + packages/source-postgres/src/index.test.ts | 59 +- packages/source-postgres/src/index.ts | 4 + packages/source-postgres/src/spec.ts | 10 +- pnpm-lock.yaml | 16 +- scripts/generate-openapi-specs.ts | 8 +- 23 files changed, 1907 insertions(+), 127 deletions(-) create mode 100644 apps/engine/src/lib/createSchemas.test.ts create mode 100644 apps/service/src/lib/createSchemas.test.ts diff --git a/apps/engine/src/__generated__/openapi.d.ts b/apps/engine/src/__generated__/openapi.d.ts index d11dfe064..7636a8136 100644 --- a/apps/engine/src/__generated__/openapi.d.ts +++ b/apps/engine/src/__generated__/openapi.d.ts @@ -291,6 +291,10 @@ export interface components { /** @constant */ type: "stripe"; stripe: components["schemas"]["SourceStripeConfig"]; + } | { + /** @constant */ + type: "postgres"; + postgres: components["schemas"]["SourcePostgresConfig"]; }; SourceStripeConfig: { /** @description Stripe API key (sk_test_... or sk_live_...) */ @@ -328,6 +332,133 @@ export interface components { /** @description Override max requests per second (default: auto-derived from API key mode — 20 live, 10 test). */ rate_limit?: number; }; + SourcePostgresConfig: { + [key: string]: unknown; + } & ({ + /** + * @description Schema containing the source table + * @default public + */ + schema: string; + /** + * @description Columns that uniquely identify a row in this stream + * @default [ + * "id" + * ] + */ + primary_key: string[]; + /** @description Monotonic column used for incremental reads */ + cursor_field: string; + /** + * @description Rows to read per page + * @default 100 + */ + page_size: number; + /** @description PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA) */ + ssl_ca_pem?: string; + /** @description Postgres connection string */ + url: string; + /** @description Deprecated alias for url; prefer url */ + connection_string?: string; + /** @description Table to read from */ + table: string; + query?: unknown; + /** @description Stream name emitted in the catalog and records. Defaults to table name. */ + stream?: string; + } | { + /** + * @description Schema containing the source table + * @default public + */ + schema: string; + /** + * @description Columns that uniquely identify a row in this stream + * @default [ + * "id" + * ] + */ + primary_key: string[]; + /** @description Monotonic column used for incremental reads */ + cursor_field: string; + /** + * @description Rows to read per page + * @default 100 + */ + page_size: number; + /** @description PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA) */ + ssl_ca_pem?: string; + /** @description Postgres connection string */ + url?: string; + /** @description Deprecated alias for url; prefer url */ + connection_string: string; + /** @description Table to read from */ + table: string; + query?: unknown; + /** @description Stream name emitted in the catalog and records. Defaults to table name. */ + stream?: string; + } | { + /** + * @description Schema containing the source table + * @default public + */ + schema: string; + /** + * @description Columns that uniquely identify a row in this stream + * @default [ + * "id" + * ] + */ + primary_key: string[]; + /** @description Monotonic column used for incremental reads */ + cursor_field: string; + /** + * @description Rows to read per page + * @default 100 + */ + page_size: number; + /** @description PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA) */ + ssl_ca_pem?: string; + /** @description Postgres connection string */ + url: string; + /** @description Deprecated alias for url; prefer url */ + connection_string?: string; + table?: unknown; + /** @description SQL query to read from. Must expose the primary_key and cursor_field columns. */ + query: string; + /** @description Stream name emitted in the catalog and records. */ + stream: string; + } | { + /** + * @description Schema containing the source table + * @default public + */ + schema: string; + /** + * @description Columns that uniquely identify a row in this stream + * @default [ + * "id" + * ] + */ + primary_key: string[]; + /** @description Monotonic column used for incremental reads */ + cursor_field: string; + /** + * @description Rows to read per page + * @default 100 + */ + page_size: number; + /** @description PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA) */ + ssl_ca_pem?: string; + /** @description Postgres connection string */ + url?: string; + /** @description Deprecated alias for url; prefer url */ + connection_string: string; + table?: unknown; + /** @description SQL query to read from. Must expose the primary_key and cursor_field columns. */ + query: string; + /** @description Stream name emitted in the catalog and records. */ + stream: string; + }); DestinationConfig: { /** @constant */ type: "postgres"; @@ -336,6 +467,10 @@ export interface components { /** @constant */ type: "google_sheets"; google_sheets: components["schemas"]["DestinationGoogleSheetsConfig"]; + } | { + /** @constant */ + type: "stripe"; + stripe: components["schemas"]["DestinationStripeConfig"]; }; DestinationPostgresConfig: { /** @description Postgres connection string */ @@ -396,6 +531,67 @@ export interface components { */ batch_size: number; }; + DestinationStripeConfig: { + [key: string]: unknown; + } & ({ + /** @description Stripe API key (sk_test_... or sk_live_...) */ + api_key: string; + /** + * Format: uri + * @description Override the Stripe API base URL (e.g. http://localhost:12111 for tests) + */ + base_url?: string; + /** + * @description Retries for 429/5xx/network errors + * @default 3 + */ + max_retries: number; + /** @constant */ + api_version: "unsafe-development"; + /** @constant */ + object: "custom_object"; + /** @constant */ + write_mode: "create"; + /** @description Per-source-stream Custom Object write configuration. */ + streams: { + [key: string]: { + /** @description Stripe Custom Object api_name_plural */ + plural_name: string; + /** @description Mapping from Custom Object field names to source record fields. */ + field_mapping: { + [key: string]: string; + }; + }; + }; + } | { + /** @description Stripe API key (sk_test_... or sk_live_...) */ + api_key: string; + /** + * Format: uri + * @description Override the Stripe API base URL (e.g. http://localhost:12111 for tests) + */ + base_url?: string; + /** + * @description Retries for 429/5xx/network errors + * @default 3 + */ + max_retries: number; + /** @enum {string} */ + api_version: "2026-03-25.dahlia" | "2026-02-25.clover" | "2026-01-28.clover" | "2025-12-15.clover" | "2025-11-17.clover" | "2025-10-29.clover" | "2025-09-30.clover" | "2025-08-27.basil" | "2025-07-30.basil" | "2025-06-30.basil" | "2025-05-28.basil" | "2025-04-30.basil" | "2025-03-31.basil" | "2025-02-24.acacia" | "2025-01-27.acacia" | "2024-12-18.acacia" | "2024-11-20.acacia" | "2024-10-28.acacia" | "2024-09-30.acacia" | "2024-06-20" | "2024-04-10" | "2024-04-03" | "2023-10-16" | "2023-08-16" | "2022-11-15" | "2022-08-01" | "2020-08-27" | "2020-03-02" | "2019-12-03" | "2019-11-05" | "2019-10-17" | "2019-10-08" | "2019-09-09" | "2019-08-14" | "2019-05-16" | "2019-03-14" | "2019-02-19" | "2019-02-11" | "2018-11-08" | "2018-10-31" | "2018-09-24" | "2018-09-06" | "2018-08-23" | "2018-07-27" | "2018-05-21" | "2018-02-28" | "2018-02-06" | "2018-02-05" | "2018-01-23" | "2017-12-14" | "2017-08-15"; + /** @constant */ + object: "standard_object"; + /** @constant */ + write_mode: "create"; + /** @description Per-source-stream standard Stripe object create configuration. */ + streams: { + [key: string]: { + /** @description Mapping from Stripe create parameter names to source record fields. */ + field_mapping: { + [key: string]: string; + }; + }; + }; + }); RecordMessage: { /** @description Who emitted this message: "source/{type}", "destination/{type}", or "engine". Set by the engine. */ _emitted_by?: string; @@ -650,11 +846,11 @@ export interface components { control: { /** @constant */ control_type: "source_config"; - source_config: components["schemas"]["SourceStripeConfig"]; + source_config: components["schemas"]["SourceStripeConfig"] | components["schemas"]["SourcePostgresConfig"]; } | { /** @constant */ control_type: "destination_config"; - destination_config: components["schemas"]["DestinationPostgresConfig"] | components["schemas"]["DestinationGoogleSheetsConfig"]; + destination_config: components["schemas"]["DestinationPostgresConfig"] | components["schemas"]["DestinationGoogleSheetsConfig"] | components["schemas"]["DestinationStripeConfig"]; }; }; ProgressMessage: { diff --git a/apps/engine/src/__generated__/openapi.json b/apps/engine/src/__generated__/openapi.json index ace9d0e6b..9b1cfb4e0 100644 --- a/apps/engine/src/__generated__/openapi.json +++ b/apps/engine/src/__generated__/openapi.json @@ -969,6 +969,22 @@ "type", "stripe" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "postgres" + }, + "postgres": { + "$ref": "#/components/schemas/SourcePostgresConfig" + } + }, + "required": [ + "type", + "postgres" + ] } ], "type": "object", @@ -1106,6 +1122,265 @@ ], "additionalProperties": false }, + "SourcePostgresConfig": { + "allOf": [ + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + { + "anyOf": [ + { + "type": "object", + "properties": { + "schema": { + "default": "public", + "type": "string", + "description": "Schema containing the source table" + }, + "primary_key": { + "default": [ + "id" + ], + "minItems": 1, + "type": "array", + "items": { + "type": "string" + }, + "description": "Columns that uniquely identify a row in this stream" + }, + "cursor_field": { + "type": "string", + "description": "Monotonic column used for incremental reads" + }, + "page_size": { + "default": 100, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "description": "Rows to read per page" + }, + "ssl_ca_pem": { + "type": "string", + "description": "PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA)" + }, + "url": { + "type": "string", + "description": "Postgres connection string" + }, + "connection_string": { + "type": "string", + "description": "Deprecated alias for url; prefer url" + }, + "table": { + "type": "string", + "description": "Table to read from" + }, + "query": { + "not": {} + }, + "stream": { + "type": "string", + "description": "Stream name emitted in the catalog and records. Defaults to table name." + } + }, + "required": [ + "cursor_field", + "url", + "table" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "schema": { + "default": "public", + "type": "string", + "description": "Schema containing the source table" + }, + "primary_key": { + "default": [ + "id" + ], + "minItems": 1, + "type": "array", + "items": { + "type": "string" + }, + "description": "Columns that uniquely identify a row in this stream" + }, + "cursor_field": { + "type": "string", + "description": "Monotonic column used for incremental reads" + }, + "page_size": { + "default": 100, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "description": "Rows to read per page" + }, + "ssl_ca_pem": { + "type": "string", + "description": "PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA)" + }, + "url": { + "type": "string", + "description": "Postgres connection string" + }, + "connection_string": { + "type": "string", + "description": "Deprecated alias for url; prefer url" + }, + "table": { + "type": "string", + "description": "Table to read from" + }, + "query": { + "not": {} + }, + "stream": { + "type": "string", + "description": "Stream name emitted in the catalog and records. Defaults to table name." + } + }, + "required": [ + "cursor_field", + "connection_string", + "table" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "schema": { + "default": "public", + "type": "string", + "description": "Schema containing the source table" + }, + "primary_key": { + "default": [ + "id" + ], + "minItems": 1, + "type": "array", + "items": { + "type": "string" + }, + "description": "Columns that uniquely identify a row in this stream" + }, + "cursor_field": { + "type": "string", + "description": "Monotonic column used for incremental reads" + }, + "page_size": { + "default": 100, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "description": "Rows to read per page" + }, + "ssl_ca_pem": { + "type": "string", + "description": "PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA)" + }, + "url": { + "type": "string", + "description": "Postgres connection string" + }, + "connection_string": { + "type": "string", + "description": "Deprecated alias for url; prefer url" + }, + "table": { + "not": {} + }, + "query": { + "type": "string", + "description": "SQL query to read from. Must expose the primary_key and cursor_field columns." + }, + "stream": { + "type": "string", + "description": "Stream name emitted in the catalog and records." + } + }, + "required": [ + "cursor_field", + "url", + "query", + "stream" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "schema": { + "default": "public", + "type": "string", + "description": "Schema containing the source table" + }, + "primary_key": { + "default": [ + "id" + ], + "minItems": 1, + "type": "array", + "items": { + "type": "string" + }, + "description": "Columns that uniquely identify a row in this stream" + }, + "cursor_field": { + "type": "string", + "description": "Monotonic column used for incremental reads" + }, + "page_size": { + "default": 100, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "description": "Rows to read per page" + }, + "ssl_ca_pem": { + "type": "string", + "description": "PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA)" + }, + "url": { + "type": "string", + "description": "Postgres connection string" + }, + "connection_string": { + "type": "string", + "description": "Deprecated alias for url; prefer url" + }, + "table": { + "not": {} + }, + "query": { + "type": "string", + "description": "SQL query to read from. Must expose the primary_key and cursor_field columns." + }, + "stream": { + "type": "string", + "description": "Stream name emitted in the catalog and records." + } + }, + "required": [ + "cursor_field", + "connection_string", + "query", + "stream" + ], + "additionalProperties": false + } + ] + } + ] + }, "DestinationConfig": { "oneOf": [ { @@ -1139,6 +1414,22 @@ "type", "google_sheets" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "stripe" + }, + "stripe": { + "$ref": "#/components/schemas/DestinationStripeConfig" + } + }, + "required": [ + "type", + "stripe" + ] } ], "type": "object", @@ -1261,6 +1552,210 @@ ], "additionalProperties": false }, + "DestinationStripeConfig": { + "allOf": [ + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + { + "oneOf": [ + { + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Stripe API key (sk_test_... or sk_live_...)" + }, + "base_url": { + "type": "string", + "format": "uri", + "description": "Override the Stripe API base URL (e.g. http://localhost:12111 for tests)" + }, + "max_retries": { + "default": 3, + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991, + "description": "Retries for 429/5xx/network errors" + }, + "api_version": { + "type": "string", + "const": "unsafe-development" + }, + "object": { + "type": "string", + "const": "custom_object" + }, + "write_mode": { + "type": "string", + "const": "create" + }, + "streams": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "plural_name": { + "type": "string", + "description": "Stripe Custom Object api_name_plural" + }, + "field_mapping": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + }, + "description": "Mapping from Custom Object field names to source record fields." + } + }, + "required": [ + "plural_name", + "field_mapping" + ], + "additionalProperties": false + }, + "description": "Per-source-stream Custom Object write configuration." + } + }, + "required": [ + "api_key", + "api_version", + "object", + "write_mode", + "streams" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Stripe API key (sk_test_... or sk_live_...)" + }, + "base_url": { + "type": "string", + "format": "uri", + "description": "Override the Stripe API base URL (e.g. http://localhost:12111 for tests)" + }, + "max_retries": { + "default": 3, + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991, + "description": "Retries for 429/5xx/network errors" + }, + "api_version": { + "type": "string", + "enum": [ + "2026-03-25.dahlia", + "2026-02-25.clover", + "2026-01-28.clover", + "2025-12-15.clover", + "2025-11-17.clover", + "2025-10-29.clover", + "2025-09-30.clover", + "2025-08-27.basil", + "2025-07-30.basil", + "2025-06-30.basil", + "2025-05-28.basil", + "2025-04-30.basil", + "2025-03-31.basil", + "2025-02-24.acacia", + "2025-01-27.acacia", + "2024-12-18.acacia", + "2024-11-20.acacia", + "2024-10-28.acacia", + "2024-09-30.acacia", + "2024-06-20", + "2024-04-10", + "2024-04-03", + "2023-10-16", + "2023-08-16", + "2022-11-15", + "2022-08-01", + "2020-08-27", + "2020-03-02", + "2019-12-03", + "2019-11-05", + "2019-10-17", + "2019-10-08", + "2019-09-09", + "2019-08-14", + "2019-05-16", + "2019-03-14", + "2019-02-19", + "2019-02-11", + "2018-11-08", + "2018-10-31", + "2018-09-24", + "2018-09-06", + "2018-08-23", + "2018-07-27", + "2018-05-21", + "2018-02-28", + "2018-02-06", + "2018-02-05", + "2018-01-23", + "2017-12-14", + "2017-08-15" + ] + }, + "object": { + "type": "string", + "const": "standard_object" + }, + "write_mode": { + "type": "string", + "const": "create" + }, + "streams": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "field_mapping": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + }, + "description": "Mapping from Stripe create parameter names to source record fields." + } + }, + "required": [ + "field_mapping" + ], + "additionalProperties": false + }, + "description": "Per-source-stream standard Stripe object create configuration." + } + }, + "required": [ + "api_key", + "api_version", + "object", + "write_mode", + "streams" + ], + "additionalProperties": false + } + ] + } + ] + }, "RecordMessage": { "type": "object", "properties": { @@ -1815,7 +2310,14 @@ "const": "source_config" }, "source_config": { - "$ref": "#/components/schemas/SourceStripeConfig" + "oneOf": [ + { + "$ref": "#/components/schemas/SourceStripeConfig" + }, + { + "$ref": "#/components/schemas/SourcePostgresConfig" + } + ] } }, "required": [ @@ -1837,6 +2339,9 @@ }, { "$ref": "#/components/schemas/DestinationGoogleSheetsConfig" + }, + { + "$ref": "#/components/schemas/DestinationStripeConfig" } ] } diff --git a/apps/engine/src/lib/createSchemas.test.ts b/apps/engine/src/lib/createSchemas.test.ts new file mode 100644 index 000000000..5136e6f12 --- /dev/null +++ b/apps/engine/src/lib/createSchemas.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest' +import { z } from 'zod' +import { createConnectorSchemas } from './createSchemas.js' +import type { ConnectorResolver } from './resolver.js' + +const postgresLikeConfigSchema = { + anyOf: [ + { + type: 'object', + properties: { + url: { type: 'string' }, + table: { type: 'string' }, + cursor_field: { type: 'string' }, + primary_key: { type: 'array', items: { type: 'string' } }, + }, + required: ['url', 'table', 'cursor_field'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + url: { type: 'string' }, + query: { type: 'string' }, + stream: { type: 'string' }, + cursor_field: { type: 'string' }, + }, + required: ['url', 'query', 'stream', 'cursor_field'], + additionalProperties: false, + }, + ], +} satisfies Record + +function resolverWithUnionConfig(): ConnectorResolver { + return { + async resolveSource() { + throw new Error('not used') + }, + async resolveDestination() { + throw new Error('not used') + }, + sources() { + return new Map([ + [ + 'postgres', + { + connector: {} as never, + configSchema: z.any(), + rawConfigJsonSchema: postgresLikeConfigSchema, + }, + ], + ]) + }, + destinations() { + return new Map([ + [ + 'stripe', + { + connector: {} as never, + configSchema: z.any(), + rawConfigJsonSchema: { + type: 'object', + properties: { api_key: { type: 'string' } }, + required: ['api_key'], + additionalProperties: false, + }, + }, + ], + ]) + }, + } +} + +describe('createConnectorSchemas', () => { + it('preserves connector payload fields for anyOf config schemas', () => { + const { PipelineConfig } = createConnectorSchemas(resolverWithUnionConfig()) + + const parsed = PipelineConfig.parse({ + source: { + type: 'postgres', + postgres: { + url: 'postgres://localhost/db', + table: 'crm_customers', + cursor_field: 'updated_at', + primary_key: ['id'], + }, + }, + destination: { type: 'stripe', stripe: { api_key: 'sk_test_123' } }, + streams: [{ name: 'customer', sync_mode: 'incremental' }], + }) + + expect(parsed.source.postgres).toEqual({ + url: 'postgres://localhost/db', + table: 'crm_customers', + cursor_field: 'updated_at', + primary_key: ['id'], + }) + }) +}) diff --git a/apps/engine/src/lib/createSchemas.ts b/apps/engine/src/lib/createSchemas.ts index 5fe5769fd..04e3de224 100644 --- a/apps/engine/src/lib/createSchemas.ts +++ b/apps/engine/src/lib/createSchemas.ts @@ -46,6 +46,14 @@ const StreamConfig = z.object({ .describe('Cap backfill to this many records, then mark the stream complete.'), }) +function schemaFromJsonSchema(jsonSchema: Record): z.ZodType { + const schema = z.fromJSONSchema(jsonSchema) + // fromJSONSchema({}) returns ZodAny. Use an empty object only for that truly + // unconstrained shape; keep unions/intersections intact so validation does + // not strip connector payloads such as source-postgres anyOf configs. + return schema instanceof z.ZodAny ? z.object({}) : schema +} + /** * Build typed Zod schemas with `.meta({ id })` annotations from registered connectors. * @@ -58,16 +66,14 @@ const StreamConfig = z.object({ export function createConnectorSchemas(resolver: ConnectorResolver) { // Build inner config schemas and envelope variants in one pass per role const sources = [...resolver.sources()].map(([name, r]) => { - const base = z.fromJSONSchema(r.rawConfigJsonSchema) - const config = (base instanceof z.ZodObject ? base : z.object({})).meta({ + const config = schemaFromJsonSchema(r.rawConfigJsonSchema).meta({ id: connectorSchemaName(name, 'Source'), }) return { name, config, variant: z.object({ type: z.literal(name), [name]: config }) } }) const destinations = [...resolver.destinations()].map(([name, r]) => { - const base = z.fromJSONSchema(r.rawConfigJsonSchema) - const config = (base instanceof z.ZodObject ? base : z.object({})).meta({ + const config = schemaFromJsonSchema(r.rawConfigJsonSchema).meta({ id: connectorSchemaName(name, 'Destination'), }) return { name, config, variant: z.object({ type: z.literal(name), [name]: config }) } @@ -93,8 +99,7 @@ export function createConnectorSchemas(resolver: ConnectorResolver) { const inputSchemas = [...resolver.sources()] .filter(([, r]) => r.rawInputJsonSchema != null) .map(([name, r]) => { - const base = z.fromJSONSchema(r.rawInputJsonSchema!) - return (base instanceof z.ZodObject ? base : z.object({})).meta({ + return schemaFromJsonSchema(r.rawInputJsonSchema!).meta({ id: connectorInputSchemaName(name), }) }) diff --git a/apps/engine/src/lib/resolver.test.ts b/apps/engine/src/lib/resolver.test.ts index cf8e2546f..041f962e7 100644 --- a/apps/engine/src/lib/resolver.test.ts +++ b/apps/engine/src/lib/resolver.test.ts @@ -1,5 +1,6 @@ import { existsSync } from 'node:fs' import { describe, expect, it } from 'vitest' +import { z } from 'zod' import { resolveSpecifier, resolveBin, createConnectorResolver } from './resolver.js' import { sourceTest } from './source-test.js' import { destinationTest } from './destination-test.js' @@ -104,4 +105,41 @@ describe('createConnectorResolver', () => { const resolver = await createConnectorResolver({}) await expect(resolver.resolveDestination('nonexistent')).rejects.toThrow(/not found/) }) + + it('preserves non-object config schemas for validation', async () => { + const config = z.toJSONSchema( + z.union([ + z.object({ + url: z.string(), + table: z.string(), + cursor_field: z.string(), + }), + z.object({ + url: z.string(), + query: z.string(), + stream: z.string(), + cursor_field: z.string(), + }), + ]) + ) + const resolver = await createConnectorResolver({ + sources: { + postgres: { + async *spec() { + yield { type: 'spec' as const, spec: { config } } + }, + async *check() {}, + async *discover() {}, + async *read() {}, + }, + }, + }) + + const schema = resolver.sources().get('postgres')?.configSchema + + expect( + schema?.parse({ url: 'postgres://localhost/db', table: 'users', cursor_field: 'id' }) + ).toEqual({ url: 'postgres://localhost/db', table: 'users', cursor_field: 'id' }) + expect(() => schema?.parse({})).toThrow() + }) }) diff --git a/apps/engine/src/lib/resolver.ts b/apps/engine/src/lib/resolver.ts index 1bd83250d..aa11a7e07 100644 --- a/apps/engine/src/lib/resolver.ts +++ b/apps/engine/src/lib/resolver.ts @@ -191,7 +191,7 @@ export interface ConnectorResolver { destinations(): ReadonlyMap> } -/** Convert a connector's spec() async iterable output to a Zod object schema + raw JSON Schema. */ +/** Convert a connector's spec() async iterable output to a Zod schema + raw JSON Schema. */ async function configSchemaFromSpec(connector: { spec(): AsyncIterable<{ type: string; [k: string]: unknown }> }): Promise<{ @@ -208,8 +208,9 @@ async function configSchemaFromSpec(connector: { // rawConfigJsonSchema is clean for injection into OpenAPI 3.1 component schemas. const { $schema: _unused, ...rawConfigJsonSchema } = specPayload.config const schema = z.fromJSONSchema(rawConfigJsonSchema) - // fromJSONSchema({}) returns ZodAny — fall back to empty object for composability - const configSchema = schema instanceof z.ZodObject ? schema : z.object({}) + // fromJSONSchema({}) returns ZodAny. Use an empty object only for that + // unconstrained shape; preserve unions such as source-postgres anyOf configs. + const configSchema = schema instanceof z.ZodAny ? z.object({}) : schema let rawInputJsonSchema: Record | undefined if (specPayload.source_input) { const { $schema: _unused2, ...inputSchema } = specPayload.source_input diff --git a/apps/engine/src/lib/reverse-etl.test.ts b/apps/engine/src/lib/reverse-etl.test.ts index cc11f3ea9..523601bcb 100644 --- a/apps/engine/src/lib/reverse-etl.test.ts +++ b/apps/engine/src/lib/reverse-etl.test.ts @@ -347,7 +347,7 @@ describe('reverse ETL', () => { ] const second = await engine.pipeline_sync_batch(pipeline, { state: first.ending_state, - run_id: 'run_reverse_etl_custom_object_create_twice_test', + run_id: 'run_reverse_etl_custom_object_create_twice_test_next', }) expect(first.ending_state?.source.streams.devices).toEqual({ diff --git a/apps/service/package.json b/apps/service/package.json index 83530916a..be9c6a8f4 100644 --- a/apps/service/package.json +++ b/apps/service/package.json @@ -32,10 +32,12 @@ "@scalar/hono-api-reference": "^0.6", "@stripe/sync-destination-google-sheets": "workspace:*", "@stripe/sync-destination-postgres": "workspace:*", + "@stripe/sync-destination-stripe": "workspace:*", "@stripe/sync-engine": "workspace:*", "@stripe/sync-hono-zod-openapi": "workspace:*", "@stripe/sync-logger": "workspace:*", "@stripe/sync-protocol": "workspace:*", + "@stripe/sync-source-postgres": "workspace:*", "@stripe/sync-source-stripe": "workspace:*", "@stripe/sync-ts-cli": "workspace:*", "@temporalio/activity": "^1", diff --git a/apps/service/src/__generated__/openapi.d.ts b/apps/service/src/__generated__/openapi.d.ts index d9a416097..c45a8407c 100644 --- a/apps/service/src/__generated__/openapi.d.ts +++ b/apps/service/src/__generated__/openapi.d.ts @@ -226,6 +226,10 @@ export interface components { /** @constant */ type: "stripe"; stripe: components["schemas"]["SourceStripeConfig"]; + } | { + /** @constant */ + type: "postgres"; + postgres: components["schemas"]["SourcePostgresConfig"]; }; SourceStripeConfig: { /** @description Stripe API key (sk_test_... or sk_live_...) */ @@ -263,6 +267,133 @@ export interface components { /** @description Override max requests per second (default: auto-derived from API key mode — 20 live, 10 test). */ rate_limit?: number; }; + SourcePostgresConfig: { + [key: string]: unknown; + } & ({ + /** + * @description Schema containing the source table + * @default public + */ + schema: string; + /** + * @description Columns that uniquely identify a row in this stream + * @default [ + * "id" + * ] + */ + primary_key: string[]; + /** @description Monotonic column used for incremental reads */ + cursor_field: string; + /** + * @description Rows to read per page + * @default 100 + */ + page_size: number; + /** @description PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA) */ + ssl_ca_pem?: string; + /** @description Postgres connection string */ + url: string; + /** @description Deprecated alias for url; prefer url */ + connection_string?: string; + /** @description Table to read from */ + table: string; + query?: unknown; + /** @description Stream name emitted in the catalog and records. Defaults to table name. */ + stream?: string; + } | { + /** + * @description Schema containing the source table + * @default public + */ + schema: string; + /** + * @description Columns that uniquely identify a row in this stream + * @default [ + * "id" + * ] + */ + primary_key: string[]; + /** @description Monotonic column used for incremental reads */ + cursor_field: string; + /** + * @description Rows to read per page + * @default 100 + */ + page_size: number; + /** @description PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA) */ + ssl_ca_pem?: string; + /** @description Postgres connection string */ + url?: string; + /** @description Deprecated alias for url; prefer url */ + connection_string: string; + /** @description Table to read from */ + table: string; + query?: unknown; + /** @description Stream name emitted in the catalog and records. Defaults to table name. */ + stream?: string; + } | { + /** + * @description Schema containing the source table + * @default public + */ + schema: string; + /** + * @description Columns that uniquely identify a row in this stream + * @default [ + * "id" + * ] + */ + primary_key: string[]; + /** @description Monotonic column used for incremental reads */ + cursor_field: string; + /** + * @description Rows to read per page + * @default 100 + */ + page_size: number; + /** @description PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA) */ + ssl_ca_pem?: string; + /** @description Postgres connection string */ + url: string; + /** @description Deprecated alias for url; prefer url */ + connection_string?: string; + table?: unknown; + /** @description SQL query to read from. Must expose the primary_key and cursor_field columns. */ + query: string; + /** @description Stream name emitted in the catalog and records. */ + stream: string; + } | { + /** + * @description Schema containing the source table + * @default public + */ + schema: string; + /** + * @description Columns that uniquely identify a row in this stream + * @default [ + * "id" + * ] + */ + primary_key: string[]; + /** @description Monotonic column used for incremental reads */ + cursor_field: string; + /** + * @description Rows to read per page + * @default 100 + */ + page_size: number; + /** @description PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA) */ + ssl_ca_pem?: string; + /** @description Postgres connection string */ + url?: string; + /** @description Deprecated alias for url; prefer url */ + connection_string: string; + table?: unknown; + /** @description SQL query to read from. Must expose the primary_key and cursor_field columns. */ + query: string; + /** @description Stream name emitted in the catalog and records. */ + stream: string; + }); DestinationConfig: { /** @constant */ type: "postgres"; @@ -271,6 +402,10 @@ export interface components { /** @constant */ type: "google_sheets"; google_sheets: components["schemas"]["DestinationGoogleSheetsConfig"]; + } | { + /** @constant */ + type: "stripe"; + stripe: components["schemas"]["DestinationStripeConfig"]; }; DestinationPostgresConfig: { /** @description Postgres connection string */ @@ -331,6 +466,67 @@ export interface components { */ batch_size: number; }; + DestinationStripeConfig: { + [key: string]: unknown; + } & ({ + /** @description Stripe API key (sk_test_... or sk_live_...) */ + api_key: string; + /** + * Format: uri + * @description Override the Stripe API base URL (e.g. http://localhost:12111 for tests) + */ + base_url?: string; + /** + * @description Retries for 429/5xx/network errors + * @default 3 + */ + max_retries: number; + /** @constant */ + api_version: "unsafe-development"; + /** @constant */ + object: "custom_object"; + /** @constant */ + write_mode: "create"; + /** @description Per-source-stream Custom Object write configuration. */ + streams: { + [key: string]: { + /** @description Stripe Custom Object api_name_plural */ + plural_name: string; + /** @description Mapping from Custom Object field names to source record fields. */ + field_mapping: { + [key: string]: string; + }; + }; + }; + } | { + /** @description Stripe API key (sk_test_... or sk_live_...) */ + api_key: string; + /** + * Format: uri + * @description Override the Stripe API base URL (e.g. http://localhost:12111 for tests) + */ + base_url?: string; + /** + * @description Retries for 429/5xx/network errors + * @default 3 + */ + max_retries: number; + /** @enum {string} */ + api_version: "2026-03-25.dahlia" | "2026-02-25.clover" | "2026-01-28.clover" | "2025-12-15.clover" | "2025-11-17.clover" | "2025-10-29.clover" | "2025-09-30.clover" | "2025-08-27.basil" | "2025-07-30.basil" | "2025-06-30.basil" | "2025-05-28.basil" | "2025-04-30.basil" | "2025-03-31.basil" | "2025-02-24.acacia" | "2025-01-27.acacia" | "2024-12-18.acacia" | "2024-11-20.acacia" | "2024-10-28.acacia" | "2024-09-30.acacia" | "2024-06-20" | "2024-04-10" | "2024-04-03" | "2023-10-16" | "2023-08-16" | "2022-11-15" | "2022-08-01" | "2020-08-27" | "2020-03-02" | "2019-12-03" | "2019-11-05" | "2019-10-17" | "2019-10-08" | "2019-09-09" | "2019-08-14" | "2019-05-16" | "2019-03-14" | "2019-02-19" | "2019-02-11" | "2018-11-08" | "2018-10-31" | "2018-09-24" | "2018-09-06" | "2018-08-23" | "2018-07-27" | "2018-05-21" | "2018-02-28" | "2018-02-06" | "2018-02-05" | "2018-01-23" | "2017-12-14" | "2017-08-15"; + /** @constant */ + object: "standard_object"; + /** @constant */ + write_mode: "create"; + /** @description Per-source-stream standard Stripe object create configuration. */ + streams: { + [key: string]: { + /** @description Mapping from Stripe create parameter names to source record fields. */ + field_mapping: { + [key: string]: string; + }; + }; + }; + }); /** @description Full sync checkpoint with separate sections for source, destination, and sync run. Connectors only see their own section; the engine manages routing. */ SyncState: { source: components["schemas"]["SourceState"]; diff --git a/apps/service/src/__generated__/openapi.json b/apps/service/src/__generated__/openapi.json index 377ec6ed7..15797765f 100644 --- a/apps/service/src/__generated__/openapi.json +++ b/apps/service/src/__generated__/openapi.json @@ -1180,6 +1180,22 @@ "type", "stripe" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "postgres" + }, + "postgres": { + "$ref": "#/components/schemas/SourcePostgresConfig" + } + }, + "required": [ + "type", + "postgres" + ] } ], "type": "object", @@ -1317,6 +1333,265 @@ ], "additionalProperties": false }, + "SourcePostgresConfig": { + "allOf": [ + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + { + "anyOf": [ + { + "type": "object", + "properties": { + "schema": { + "default": "public", + "type": "string", + "description": "Schema containing the source table" + }, + "primary_key": { + "default": [ + "id" + ], + "minItems": 1, + "type": "array", + "items": { + "type": "string" + }, + "description": "Columns that uniquely identify a row in this stream" + }, + "cursor_field": { + "type": "string", + "description": "Monotonic column used for incremental reads" + }, + "page_size": { + "default": 100, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "description": "Rows to read per page" + }, + "ssl_ca_pem": { + "type": "string", + "description": "PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA)" + }, + "url": { + "type": "string", + "description": "Postgres connection string" + }, + "connection_string": { + "type": "string", + "description": "Deprecated alias for url; prefer url" + }, + "table": { + "type": "string", + "description": "Table to read from" + }, + "query": { + "not": {} + }, + "stream": { + "type": "string", + "description": "Stream name emitted in the catalog and records. Defaults to table name." + } + }, + "required": [ + "cursor_field", + "url", + "table" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "schema": { + "default": "public", + "type": "string", + "description": "Schema containing the source table" + }, + "primary_key": { + "default": [ + "id" + ], + "minItems": 1, + "type": "array", + "items": { + "type": "string" + }, + "description": "Columns that uniquely identify a row in this stream" + }, + "cursor_field": { + "type": "string", + "description": "Monotonic column used for incremental reads" + }, + "page_size": { + "default": 100, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "description": "Rows to read per page" + }, + "ssl_ca_pem": { + "type": "string", + "description": "PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA)" + }, + "url": { + "type": "string", + "description": "Postgres connection string" + }, + "connection_string": { + "type": "string", + "description": "Deprecated alias for url; prefer url" + }, + "table": { + "type": "string", + "description": "Table to read from" + }, + "query": { + "not": {} + }, + "stream": { + "type": "string", + "description": "Stream name emitted in the catalog and records. Defaults to table name." + } + }, + "required": [ + "cursor_field", + "connection_string", + "table" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "schema": { + "default": "public", + "type": "string", + "description": "Schema containing the source table" + }, + "primary_key": { + "default": [ + "id" + ], + "minItems": 1, + "type": "array", + "items": { + "type": "string" + }, + "description": "Columns that uniquely identify a row in this stream" + }, + "cursor_field": { + "type": "string", + "description": "Monotonic column used for incremental reads" + }, + "page_size": { + "default": 100, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "description": "Rows to read per page" + }, + "ssl_ca_pem": { + "type": "string", + "description": "PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA)" + }, + "url": { + "type": "string", + "description": "Postgres connection string" + }, + "connection_string": { + "type": "string", + "description": "Deprecated alias for url; prefer url" + }, + "table": { + "not": {} + }, + "query": { + "type": "string", + "description": "SQL query to read from. Must expose the primary_key and cursor_field columns." + }, + "stream": { + "type": "string", + "description": "Stream name emitted in the catalog and records." + } + }, + "required": [ + "cursor_field", + "url", + "query", + "stream" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "schema": { + "default": "public", + "type": "string", + "description": "Schema containing the source table" + }, + "primary_key": { + "default": [ + "id" + ], + "minItems": 1, + "type": "array", + "items": { + "type": "string" + }, + "description": "Columns that uniquely identify a row in this stream" + }, + "cursor_field": { + "type": "string", + "description": "Monotonic column used for incremental reads" + }, + "page_size": { + "default": 100, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "description": "Rows to read per page" + }, + "ssl_ca_pem": { + "type": "string", + "description": "PEM-encoded CA certificate for SSL verification (required for verify-ca / verify-full with a private CA)" + }, + "url": { + "type": "string", + "description": "Postgres connection string" + }, + "connection_string": { + "type": "string", + "description": "Deprecated alias for url; prefer url" + }, + "table": { + "not": {} + }, + "query": { + "type": "string", + "description": "SQL query to read from. Must expose the primary_key and cursor_field columns." + }, + "stream": { + "type": "string", + "description": "Stream name emitted in the catalog and records." + } + }, + "required": [ + "cursor_field", + "connection_string", + "query", + "stream" + ], + "additionalProperties": false + } + ] + } + ] + }, "DestinationConfig": { "oneOf": [ { @@ -1350,6 +1625,22 @@ "type", "google_sheets" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "stripe" + }, + "stripe": { + "$ref": "#/components/schemas/DestinationStripeConfig" + } + }, + "required": [ + "type", + "stripe" + ] } ], "type": "object", @@ -1472,6 +1763,210 @@ ], "additionalProperties": false }, + "DestinationStripeConfig": { + "allOf": [ + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + { + "oneOf": [ + { + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Stripe API key (sk_test_... or sk_live_...)" + }, + "base_url": { + "type": "string", + "format": "uri", + "description": "Override the Stripe API base URL (e.g. http://localhost:12111 for tests)" + }, + "max_retries": { + "default": 3, + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991, + "description": "Retries for 429/5xx/network errors" + }, + "api_version": { + "type": "string", + "const": "unsafe-development" + }, + "object": { + "type": "string", + "const": "custom_object" + }, + "write_mode": { + "type": "string", + "const": "create" + }, + "streams": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "plural_name": { + "type": "string", + "description": "Stripe Custom Object api_name_plural" + }, + "field_mapping": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + }, + "description": "Mapping from Custom Object field names to source record fields." + } + }, + "required": [ + "plural_name", + "field_mapping" + ], + "additionalProperties": false + }, + "description": "Per-source-stream Custom Object write configuration." + } + }, + "required": [ + "api_key", + "api_version", + "object", + "write_mode", + "streams" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Stripe API key (sk_test_... or sk_live_...)" + }, + "base_url": { + "type": "string", + "format": "uri", + "description": "Override the Stripe API base URL (e.g. http://localhost:12111 for tests)" + }, + "max_retries": { + "default": 3, + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991, + "description": "Retries for 429/5xx/network errors" + }, + "api_version": { + "type": "string", + "enum": [ + "2026-03-25.dahlia", + "2026-02-25.clover", + "2026-01-28.clover", + "2025-12-15.clover", + "2025-11-17.clover", + "2025-10-29.clover", + "2025-09-30.clover", + "2025-08-27.basil", + "2025-07-30.basil", + "2025-06-30.basil", + "2025-05-28.basil", + "2025-04-30.basil", + "2025-03-31.basil", + "2025-02-24.acacia", + "2025-01-27.acacia", + "2024-12-18.acacia", + "2024-11-20.acacia", + "2024-10-28.acacia", + "2024-09-30.acacia", + "2024-06-20", + "2024-04-10", + "2024-04-03", + "2023-10-16", + "2023-08-16", + "2022-11-15", + "2022-08-01", + "2020-08-27", + "2020-03-02", + "2019-12-03", + "2019-11-05", + "2019-10-17", + "2019-10-08", + "2019-09-09", + "2019-08-14", + "2019-05-16", + "2019-03-14", + "2019-02-19", + "2019-02-11", + "2018-11-08", + "2018-10-31", + "2018-09-24", + "2018-09-06", + "2018-08-23", + "2018-07-27", + "2018-05-21", + "2018-02-28", + "2018-02-06", + "2018-02-05", + "2018-01-23", + "2017-12-14", + "2017-08-15" + ] + }, + "object": { + "type": "string", + "const": "standard_object" + }, + "write_mode": { + "type": "string", + "const": "create" + }, + "streams": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "field_mapping": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + }, + "description": "Mapping from Stripe create parameter names to source record fields." + } + }, + "required": [ + "field_mapping" + ], + "additionalProperties": false + }, + "description": "Per-source-stream standard Stripe object create configuration." + } + }, + "required": [ + "api_key", + "api_version", + "object", + "write_mode", + "streams" + ], + "additionalProperties": false + } + ] + } + ] + }, "SyncState": { "type": "object", "properties": { diff --git a/apps/service/src/cli.test.ts b/apps/service/src/cli.test.ts index 412257d02..9f79951ec 100644 --- a/apps/service/src/cli.test.ts +++ b/apps/service/src/cli.test.ts @@ -405,13 +405,13 @@ describe('generated pipeline CLI', () => { rawArgs: [ 'pipelines', 'create', - '--stripe.api-key', + '--source.stripe.api-key', 'sk_test_123', - '--stripe.api-version', + '--source.stripe.api-version', '2025-03-31.basil', - '--postgres.url', + '--destination.postgres.url', 'postgres://localhost/db', - '--postgres.schema', + '--destination.postgres.schema', 'public', ], }) @@ -442,13 +442,13 @@ describe('generated pipeline CLI', () => { rawArgs: [ 'pipelines', 'create', - '--stripe.api-key', + '--source.stripe.api-key', 'sk_test_123', - '--stripe.api-version', + '--source.stripe.api-version', '2025-03-31.basil', - '--postgres.connection-string', + '--destination.postgres.connection-string', 'postgres://localhost/db', - '--postgres.schema', + '--destination.postgres.schema', 'public', ], }) @@ -801,13 +801,13 @@ describe('generated pipeline CLI', () => { rawArgs: [ 'pipelines', 'create', - '--stripe.api-key', + '--source.stripe.api-key', 'sk_test_123', - '--stripe.api-version', + '--source.stripe.api-version', '2025-03-31.basil', - '--google_sheets.access-token', + '--destination.google_sheets.access-token', 'ya29.token', - '--google_sheets.refresh-token', + '--destination.google_sheets.refresh-token', 'refresh-token', ], }) @@ -836,13 +836,13 @@ describe('generated pipeline CLI', () => { rawArgs: [ 'pipelines', 'create', - '--stripe.api-key', + '--source.stripe.api-key', 'sk_test_123', - '--stripe.api-version', + '--source.stripe.api-version', 'not-a-real-version', - '--postgres.url', + '--destination.postgres.url', 'postgres://localhost/db', - '--postgres.schema', + '--destination.postgres.schema', 'public', ], }) diff --git a/apps/service/src/cli.ts b/apps/service/src/cli.ts index 4c7b92e12..b10df73c2 100644 --- a/apps/service/src/cli.ts +++ b/apps/service/src/cli.ts @@ -8,8 +8,10 @@ import { createPrettyFormatter } from './cli/pretty-output.js' import { serve } from '@hono/node-server' import { createConnectorResolver, startApiServer, type ApiServerHandle } from '@stripe/sync-engine' import sourceStripe from '@stripe/sync-source-stripe' +import sourcePostgres from '@stripe/sync-source-postgres' import destinationPostgres from '@stripe/sync-destination-postgres' import destinationGoogleSheets from '@stripe/sync-destination-google-sheets' +import destinationStripe from '@stripe/sync-destination-stripe' import { createApp } from './api/app.js' import { wrapPipelineConnectorShorthand, @@ -26,8 +28,12 @@ import { log } from './logger.js' const defaultDataDir = process.env.DATA_DIR ?? `${homedir()}/.stripe-sync` const resolverPromise = createConnectorResolver({ - sources: { stripe: sourceStripe }, - destinations: { postgres: destinationPostgres, google_sheets: destinationGoogleSheets }, + sources: { stripe: sourceStripe, postgres: sourcePostgres }, + destinations: { + postgres: destinationPostgres, + google_sheets: destinationGoogleSheets, + stripe: destinationStripe, + }, }) async function buildCliSpec() { diff --git a/apps/service/src/lib/cli-connector-shorthand.test.ts b/apps/service/src/lib/cli-connector-shorthand.test.ts index c365bba3b..b3b31a945 100644 --- a/apps/service/src/lib/cli-connector-shorthand.test.ts +++ b/apps/service/src/lib/cli-connector-shorthand.test.ts @@ -1,11 +1,10 @@ import { describe, expect, it } from 'vitest' import { applyConnectorShorthand, - assertNoAmbiguousConnectorNames, + extractConnectorOverrides, normalizeCliKey, parseCliValue, setNestedValue, - wrapPipelineConnectorShorthand, } from './cli-connector-shorthand.js' describe('cli connector shorthand', () => { @@ -33,11 +32,11 @@ describe('cli connector shorthand', () => { expect(applyConnectorShorthand(args, 'source', ['stripe'])).toEqual(args) }) - it('builds a source body from shorthand flags', () => { + it('builds a source body from scoped shorthand flags', () => { const result = applyConnectorShorthand( { - 'stripe.api-key': 'sk_test_123', - 'stripe.api-version': '2025-03-31.basil', + 'source.stripe.api-key': 'sk_test_123', + 'source.stripe.api-version': '2025-03-31.basil', }, 'source', ['stripe'] @@ -52,12 +51,12 @@ describe('cli connector shorthand', () => { it('supports nested shorthand keys and JSON values', () => { const result = applyConnectorShorthand( { - 'postgres.url': 'postgres://localhost/db', - 'postgres.schema': 'public', - 'postgres.aws.region': 'us-west-2', - 'postgres.aws.role-arn': 'arn:aws:iam::123:role/demo', - 'postgres.aws.port': '6543', - 'postgres.ssl-ca-pem': '{"pem":"value"}', + 'destination.postgres.url': 'postgres://localhost/db', + 'destination.postgres.schema': 'public', + 'destination.postgres.aws.region': 'us-west-2', + 'destination.postgres.aws.role-arn': 'arn:aws:iam::123:role/demo', + 'destination.postgres.aws.port': '6543', + 'destination.postgres.ssl-ca-pem': '{"pem":"value"}', }, 'destination', ['postgres', 'google_sheets'] @@ -82,7 +81,7 @@ describe('cli connector shorthand', () => { const result = applyConnectorShorthand( { destination: '{"type":"postgres","postgres":{"schema":"public"}}', - 'postgres.url': 'postgres://localhost/db', + 'destination.postgres.url': 'postgres://localhost/db', }, 'destination', ['postgres', 'google_sheets'] @@ -101,8 +100,8 @@ describe('cli connector shorthand', () => { expect(() => applyConnectorShorthand( { - 'postgres.schema': 'public', - 'google_sheets.access-token': 'token', + 'destination.postgres.schema': 'public', + 'destination.google_sheets.access-token': 'token', }, 'destination', ['postgres', 'google_sheets'] @@ -115,7 +114,7 @@ describe('cli connector shorthand', () => { applyConnectorShorthand( { destination: '{"type":"google_sheets","google_sheets":{"access_token":"token"}}', - 'postgres.schema': 'public', + 'destination.postgres.schema': 'public', }, 'destination', ['postgres', 'google_sheets'] @@ -123,21 +122,18 @@ describe('cli connector shorthand', () => { ).toThrow('--destination type google_sheets conflicts with shorthand flags for postgres') }) - it('rejects connector names that appear in both source and destination sets', () => { - expect(() => - assertNoAmbiguousConnectorNames( - ['stripe', 'shared_connector'], - ['postgres', 'shared-connector'] - ) - ).toThrow('Connector names cannot exist in both source and destination sets') - }) + it('lets the same connector name be used on both sides without ambiguity', () => { + const overrides = extractConnectorOverrides( + { + 'source.postgres.url': 'postgres://src/db', + 'destination.postgres.url': 'postgres://dst/db', + }, + { sources: ['postgres'], destinations: ['postgres'] } + ) - it('fails wrapper creation when source and destination connector names overlap', () => { - expect(() => - wrapPipelineConnectorShorthand({} as any, { - sources: ['shared_connector'], - destinations: ['shared-connector'], - }) - ).toThrow('Connector names cannot exist in both source and destination sets') + expect(overrides).toEqual({ + source: { type: 'postgres', postgres: { url: 'postgres://src/db' } }, + destination: { type: 'postgres', postgres: { url: 'postgres://dst/db' } }, + }) }) }) diff --git a/apps/service/src/lib/cli-connector-shorthand.ts b/apps/service/src/lib/cli-connector-shorthand.ts index 2936b8532..027149727 100644 --- a/apps/service/src/lib/cli-connector-shorthand.ts +++ b/apps/service/src/lib/cli-connector-shorthand.ts @@ -40,17 +40,18 @@ export function applyConnectorShorthand( const shorthandConfigs = new Map>() const connectorByPrefix = new Map(connectorNames.map((name) => [normalizeCliKey(name), name])) + // Shorthand keys are scoped to a side: `..`. + // This makes the side explicit so connector names that exist on both sides + // (e.g. `postgres`, `stripe`) are unambiguous. for (const [rawKey, rawValue] of Object.entries(args)) { - const dotIndex = rawKey.indexOf('.') - if (dotIndex === -1) continue + const segments = rawKey.split('.') + if (segments.length < 3) continue + if (normalizeCliKey(segments[0]!) !== bodyKey) continue - const connector = connectorByPrefix.get(normalizeCliKey(rawKey.slice(0, dotIndex))) + const connector = connectorByPrefix.get(normalizeCliKey(segments[1]!)) if (!connector) continue - const path = rawKey - .slice(dotIndex + 1) - .split('.') - .map((segment) => normalizeCliKey(segment)) + const path = segments.slice(2).map((segment) => normalizeCliKey(segment)) if (path.length === 0) continue const config = shorthandConfigs.get(connector) ?? {} @@ -107,8 +108,10 @@ export function applyConnectorShorthand( } /** - * Extracts connector override objects from CLI args (e.g. --postgres.url → destination override). - * Returns `{ source?, destination? }` suitable for merging into pipeline configs or POST bodies. + * Extracts connector override objects from CLI args. + * Recognizes scoped shorthand of the form `--source..` and + * `--destination..`. Returns `{ source?, destination? }` + * suitable for merging into pipeline configs or POST bodies. */ export function extractConnectorOverrides( args: Record, @@ -116,35 +119,44 @@ export function extractConnectorOverrides( ): { source?: Record; destination?: Record } { const result: { source?: Record; destination?: Record } = {} - const allConnectors = [...options.sources, ...options.destinations] - const connectorByPrefix = new Map(allConnectors.map((name) => [normalizeCliKey(name), name])) - const sourceSet = new Set(options.sources.map(normalizeCliKey)) + const sourceByPrefix = new Map(options.sources.map((name) => [normalizeCliKey(name), name])) + const destinationByPrefix = new Map( + options.destinations.map((name) => [normalizeCliKey(name), name]) + ) - assertNoDottedUnknownFlags(args, allConnectors) + assertNoDottedUnknownFlags(args, options) - const grouped = new Map>() + const grouped: { + source: Map> + destination: Map> + } = { + source: new Map(), + destination: new Map(), + } for (const [rawKey, rawValue] of Object.entries(args)) { - const dotIndex = rawKey.indexOf('.') - if (dotIndex === -1) continue + const segments = rawKey.split('.') + if (segments.length < 3) continue + + const side = normalizeCliKey(segments[0]!) as ConnectorBodyKey + if (side !== 'source' && side !== 'destination') continue - const connector = connectorByPrefix.get(normalizeCliKey(rawKey.slice(0, dotIndex))) + const lookup = side === 'source' ? sourceByPrefix : destinationByPrefix + const connector = lookup.get(normalizeCliKey(segments[1]!)) if (!connector) continue - const path = rawKey - .slice(dotIndex + 1) - .split('.') - .map((segment) => normalizeCliKey(segment)) + const path = segments.slice(2).map((segment) => normalizeCliKey(segment)) if (path.length === 0) continue - const config = grouped.get(connector) ?? {} + const config = grouped[side].get(connector) ?? {} setNestedValue(config, path, parseCliValue(rawValue)) - grouped.set(connector, config) + grouped[side].set(connector, config) } - for (const [connectorName, config] of grouped) { - const bodyKey = sourceSet.has(normalizeCliKey(connectorName)) ? 'source' : 'destination' - result[bodyKey] = { type: connectorName, [connectorName]: config } + for (const side of ['source', 'destination'] as const) { + for (const [connectorName, config] of grouped[side]) { + result[side] = { type: connectorName, [connectorName]: config } + } } return result @@ -196,32 +208,36 @@ export function mergeConnectorOverrides( export function assertNoDottedUnknownFlags( args: Record, - knownConnectors: string[] + options: { sources: string[]; destinations: string[] } ) { - const known = new Set(knownConnectors.map(normalizeCliKey)) + const sources = new Set(options.sources.map(normalizeCliKey)) + const destinations = new Set(options.destinations.map(normalizeCliKey)) + for (const rawKey of Object.keys(args)) { - const dotIndex = rawKey.indexOf('.') - if (dotIndex === -1) continue - const prefix = normalizeCliKey(rawKey.slice(0, dotIndex)) - if (!known.has(prefix)) { + const segments = rawKey.split('.') + if (segments.length < 2) continue + + const side = normalizeCliKey(segments[0]!) + if (side !== 'source' && side !== 'destination') { throw new Error( - `Unknown connector flag --${rawKey}: "${prefix}" is not a known connector. ` + - `Available connectors: ${knownConnectors.join(', ')}` + `Unknown connector flag --${rawKey}: must start with "source." or "destination.".` ) } - } -} -export function assertNoAmbiguousConnectorNames(sources: string[], destinations: string[]) { - const sourceNames = new Map(sources.map((name) => [normalizeCliKey(name), name])) - const overlaps = destinations - .filter((name) => sourceNames.has(normalizeCliKey(name))) - .map((name) => `${sourceNames.get(normalizeCliKey(name))} / ${name}`) + if (segments.length < 3) { + throw new Error(`Unknown connector flag --${rawKey}: expected --${side}...`) + } - if (overlaps.length > 0) { - throw new Error( - `Connector names cannot exist in both source and destination sets: ${overlaps.join(', ')}` - ) + const connector = normalizeCliKey(segments[1]!) + const known = side === 'source' ? sources : destinations + if (!known.has(connector)) { + throw new Error( + `Unknown connector flag --${rawKey}: "${connector}" is not a known ${side} connector. ` + + `Available ${side} connectors: ${ + side === 'source' ? options.sources.join(', ') : options.destinations.join(', ') + }` + ) + } } } @@ -229,8 +245,6 @@ export function wrapPipelineConnectorShorthand( command: CommandDef, options: { sources: string[]; destinations: string[] } ): CommandDef { - assertNoAmbiguousConnectorNames(options.sources, options.destinations) - const args = { ...((command.args ?? {}) as Record) } as Record if (args.source && typeof args.source === 'object') { args.source = { ...args.source, required: false } @@ -283,7 +297,7 @@ export function wrapPipelineConnectorShorthand( } } - assertNoDottedUnknownFlags(resolvedArgs, [...options.sources, ...options.destinations]) + assertNoDottedUnknownFlags(resolvedArgs, options) const argsWithSource = applyConnectorShorthand(resolvedArgs, 'source', options.sources) const argsWithDestination = applyConnectorShorthand( argsWithSource, diff --git a/apps/service/src/lib/createSchemas.test.ts b/apps/service/src/lib/createSchemas.test.ts new file mode 100644 index 000000000..058cbace9 --- /dev/null +++ b/apps/service/src/lib/createSchemas.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest' +import { z } from 'zod' +import { createSchemas } from './createSchemas.js' +import type { ConnectorResolver } from '@stripe/sync-engine' + +const postgresLikeConfigSchema = { + anyOf: [ + { + type: 'object', + properties: { + url: { type: 'string' }, + table: { type: 'string' }, + cursor_field: { type: 'string' }, + primary_key: { type: 'array', items: { type: 'string' } }, + }, + required: ['url', 'table', 'cursor_field'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + url: { type: 'string' }, + query: { type: 'string' }, + stream: { type: 'string' }, + cursor_field: { type: 'string' }, + }, + required: ['url', 'query', 'stream', 'cursor_field'], + additionalProperties: false, + }, + ], +} satisfies Record + +function resolverWithUnionConfig(): ConnectorResolver { + return { + async resolveSource() { + throw new Error('not used') + }, + async resolveDestination() { + throw new Error('not used') + }, + sources() { + return new Map([ + [ + 'postgres', + { + connector: {} as never, + configSchema: z.any(), + rawConfigJsonSchema: postgresLikeConfigSchema, + }, + ], + ]) + }, + destinations() { + return new Map([ + [ + 'stripe', + { + connector: {} as never, + configSchema: z.any(), + rawConfigJsonSchema: { + type: 'object', + properties: { api_key: { type: 'string' } }, + required: ['api_key'], + additionalProperties: false, + }, + }, + ], + ]) + }, + } +} + +describe('createSchemas', () => { + it('preserves connector payload fields for anyOf config schemas', () => { + const { CreatePipeline } = createSchemas(resolverWithUnionConfig()) + + const parsed = CreatePipeline.parse({ + id: 'my_pipe', + source: { + type: 'postgres', + postgres: { + url: 'postgres://localhost/db', + table: 'crm_customers', + cursor_field: 'updated_at', + primary_key: ['id'], + }, + }, + destination: { type: 'stripe', stripe: { api_key: 'sk_test_123' } }, + streams: [{ name: 'customer', sync_mode: 'incremental' }], + }) + + expect(parsed.source.postgres).toEqual({ + url: 'postgres://localhost/db', + table: 'crm_customers', + cursor_field: 'updated_at', + primary_key: ['id'], + }) + }) +}) diff --git a/apps/service/src/lib/createSchemas.ts b/apps/service/src/lib/createSchemas.ts index 07bf79b32..c70753d6f 100644 --- a/apps/service/src/lib/createSchemas.ts +++ b/apps/service/src/lib/createSchemas.ts @@ -64,6 +64,13 @@ export const LogEntry = z.object({ timestamp: z.string().describe('ISO 8601 timestamp when the log entry was produced.'), }) +function schemaFromJsonSchema(jsonSchema: Record): z.ZodType { + const schema = z.fromJSONSchema(jsonSchema) + // Preserve non-object schemas such as source-postgres anyOf unions. Falling + // back for all non-ZodObject values strips valid connector payload fields. + return schema instanceof z.ZodAny ? z.object({}) : schema +} + // MARK: - Dynamic schema factory (depends on registered connectors) /** @@ -79,8 +86,7 @@ export const LogEntry = z.object({ export function createSchemas(resolver: ConnectorResolver) { // Build source config discriminated union with .meta({ id }) for OAS component registration const sourceVariants = [...resolver.sources()].map(([name, r]) => { - const base = z.fromJSONSchema(r.rawConfigJsonSchema) - const obj = (base instanceof z.ZodObject ? base : z.object({})).meta({ + const obj = schemaFromJsonSchema(r.rawConfigJsonSchema).meta({ id: connectorSchemaName(name, 'Source'), }) return z.object({ type: z.literal(name), [name]: obj }) @@ -96,8 +102,7 @@ export function createSchemas(resolver: ConnectorResolver) { // Build destination config discriminated union const destVariants = [...resolver.destinations()].map(([name, r]) => { - const base = z.fromJSONSchema(r.rawConfigJsonSchema) - const obj = (base instanceof z.ZodObject ? base : z.object({})).meta({ + const obj = schemaFromJsonSchema(r.rawConfigJsonSchema).meta({ id: connectorSchemaName(name, 'Destination'), }) return z.object({ type: z.literal(name), [name]: obj }) diff --git a/e2e/reverse-etl-demo-loop.ts b/e2e/reverse-etl-demo-loop.ts index c52862ea2..eac47de47 100644 --- a/e2e/reverse-etl-demo-loop.ts +++ b/e2e/reverse-etl-demo-loop.ts @@ -81,6 +81,7 @@ import { createStripeDestination } from '../packages/destination-stripe/src/inde type DemoState = { customer?: SyncState device?: SyncState + product?: SyncState } type PipelineRunner = ( @@ -158,12 +159,53 @@ async function preparePostgres() { ) `) - log('Demo tables are ready', { tables: ['crm_customers', 'devices'] }) + await client.query(` + CREATE TABLE IF NOT EXISTS my_products ( + id text PRIMARY KEY, + name text NOT NULL, + updated_at timestamptz(3) NOT NULL DEFAULT date_trunc('milliseconds', clock_timestamp()) + ) + `) + + log('Demo tables are ready', { tables: ['crm_customers', 'devices', 'my_products'] }) } finally { await client.end() } } +function productPipeline(): PipelineConfig { + return { + source: { + type: 'postgres', + postgres: { + url: databaseUrl, + table: 'my_products', + stream: 'product', + primary_key: ['id'], + cursor_field: 'updated_at', + page_size: 100, + }, + }, + destination: { + type: 'stripe', + stripe: { + api_key: stripeApiKey, + api_version: stripeApiVersion, + object: 'standard_object', + write_mode: 'create', + streams: { + product: { + field_mapping: { + name: 'name', + }, + }, + }, + }, + }, + streams: [{ name: 'product', sync_mode: 'incremental' }], + } +} + function customerPipeline(): PipelineConfig { return { source: { @@ -290,6 +332,7 @@ async function main() { await preparePostgres() const runner = await createRunner() const pipelines = { + product: productPipeline(), customer: customerPipeline(), device: devicePipeline(), } @@ -312,6 +355,10 @@ async function main() { state = { ...state, device: deviceResult.ending_state } log('Device poll complete', streamSummary(deviceResult, 'devices')) + const productResult = await runner(pipelines.product, state.product) + state = { ...state, product: productResult.ending_state } + log('Product poll complete', streamSummary(productResult, 'product')) + await saveState(state) } catch (err) { console.error(`[${now()}] Demo poll failed`, err instanceof Error ? err.stack : err) diff --git a/packages/destination-stripe/src/index.ts b/packages/destination-stripe/src/index.ts index cf0b02194..2bdd1dd0c 100644 --- a/packages/destination-stripe/src/index.ts +++ b/packages/destination-stripe/src/index.ts @@ -652,6 +652,12 @@ export function createStripeDestination(deps: StripeDestinationDeps = {}): Desti continue } yield input + } else if ( + input.type === 'stream_status' && + failedStreams.has(input.stream_status.stream) && + input.stream_status.status !== 'error' + ) { + continue } else { yield input } diff --git a/packages/source-postgres/src/index.test.ts b/packages/source-postgres/src/index.test.ts index 9d6e50eb7..d9820a2e4 100644 --- a/packages/source-postgres/src/index.test.ts +++ b/packages/source-postgres/src/index.test.ts @@ -234,13 +234,19 @@ describe('source-postgres', () => { const messages = await collect(source.read({ config, catalog })) expect(messages.map((message) => message.type)).toEqual([ + 'stream_status', 'record', 'record', 'source_state', 'record', 'source_state', + 'stream_status', ]) - expect(messages[2]).toMatchObject({ + expect(messages[0]).toMatchObject({ + type: 'stream_status', + stream_status: { stream: 'crm_customers', status: 'start' }, + }) + expect(messages[3]).toMatchObject({ type: 'source_state', source_state: { state_type: 'stream', @@ -248,11 +254,60 @@ describe('source-postgres', () => { data: { cursor: '2026-01-02T00:00:00.000Z', primary_key: ['crm_2'] }, }, }) - expect(messages[4]).toMatchObject({ + expect(messages[5]).toMatchObject({ type: 'source_state', source_state: { data: { cursor: '2026-01-03T00:00:00.000Z', primary_key: ['crm_3'] }, }, }) + expect(messages[6]).toMatchObject({ + type: 'stream_status', + stream_status: { stream: 'crm_customers', status: 'complete' }, + }) + }) + + it('marks an empty selected stream complete without advancing source state', async () => { + const config = configSchema.parse({ + url: 'postgres://example', + table: 'crm_customers', + primary_key: ['id'], + cursor_field: 'updated_at', + page_size: 100, + }) + const catalog: ConfiguredCatalog = { + streams: [ + { + stream: { + name: 'crm_customers', + primary_key: [['id']], + newer_than_field: 'updated_at', + }, + sync_mode: 'incremental', + destination_sync_mode: 'append', + }, + ], + } + + const source = createPostgresSource({ + createPool: () => ({ + async query() { + return queryResult([]) + }, + async end() {}, + }), + }) + + const messages = await collect(source.read({ config, catalog })) + + expect(messages).toEqual([ + { + type: 'stream_status', + stream_status: { stream: 'crm_customers', status: 'start' }, + }, + { + type: 'stream_status', + stream_status: { stream: 'crm_customers', status: 'complete' }, + }, + ]) }) }) diff --git a/packages/source-postgres/src/index.ts b/packages/source-postgres/src/index.ts index 44bea4738..b03f77171 100644 --- a/packages/source-postgres/src/index.ts +++ b/packages/source-postgres/src/index.ts @@ -234,6 +234,8 @@ export function createPostgresSource(deps: PostgresSourceDeps = {}): Source +function asObjectJsonSchema(jsonSchema: Record): Record { + return { + type: 'object', + properties: {}, + ...jsonSchema, + } +} + export default { - config: z.toJSONSchema(configSchema), + config: asObjectJsonSchema(z.toJSONSchema(configSchema)), source_state_stream: z.toJSONSchema(streamStateSpec), } satisfies ConnectorSpecification diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 537f9f18e..dbae9f5b1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -255,6 +255,9 @@ importers: '@stripe/sync-destination-postgres': specifier: workspace:* version: link:../../packages/destination-postgres + '@stripe/sync-destination-stripe': + specifier: workspace:* + version: link:../../packages/destination-stripe '@stripe/sync-engine': specifier: workspace:* version: link:../engine @@ -267,6 +270,9 @@ importers: '@stripe/sync-protocol': specifier: workspace:* version: link:../../packages/protocol + '@stripe/sync-source-postgres': + specifier: workspace:* + version: link:../../packages/source-postgres '@stripe/sync-source-stripe': specifier: workspace:* version: link:../../packages/source-stripe @@ -7321,14 +7327,6 @@ snapshots: chai: 5.2.0 tinyrainbow: 2.0.0 - '@vitest/mocker@3.2.4(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1))': - dependencies: - '@vitest/spy': 3.2.4 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) - '@vitest/mocker@3.2.4(vite@7.2.2(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1))': dependencies: '@vitest/spy': 3.2.4 @@ -9391,7 +9389,7 @@ snapshots: dependencies: '@types/chai': 5.2.2 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1)) + '@vitest/mocker': 3.2.4(vite@7.2.2(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 diff --git a/scripts/generate-openapi-specs.ts b/scripts/generate-openapi-specs.ts index 27936f4be..55f8df946 100644 --- a/scripts/generate-openapi-specs.ts +++ b/scripts/generate-openapi-specs.ts @@ -9,8 +9,10 @@ import { createApp, createConnectorResolver } from '../apps/engine/src/index.js' import { createApp as createServiceApp } from '../apps/service/src/api/app.js' import { memoryPipelineStore } from '../apps/service/src/lib/stores-memory.js' import sourceStripe from '../packages/source-stripe/src/index.js' +import sourcePostgres from '../packages/source-postgres/src/index.js' import destinationPostgres from '../packages/destination-postgres/src/index.js' import destinationGoogleSheets from '../packages/destination-google-sheets/src/index.js' +import destinationStripe from '../packages/destination-stripe/src/index.js' const [engineOut, serviceOut] = process.argv.slice(2) if (!engineOut || !serviceOut) { @@ -19,10 +21,14 @@ if (!engineOut || !serviceOut) { } const resolver = await createConnectorResolver({ - sources: { stripe: (sourceStripe as any).default ?? sourceStripe }, + sources: { + stripe: (sourceStripe as any).default ?? sourceStripe, + postgres: (sourcePostgres as any).default ?? sourcePostgres, + }, destinations: { postgres: (destinationPostgres as any).default ?? destinationPostgres, google_sheets: (destinationGoogleSheets as any).default ?? destinationGoogleSheets, + stripe: (destinationStripe as any).default ?? destinationStripe, }, }) From e12d69b3f11ebbbb0762eb814c6188c276c3457f Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 6 May 2026 21:08:06 -0700 Subject: [PATCH 06/11] docs: update architecture docs to match current codebase (#361) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: update architecture docs to match current codebase - packages.md: add missing packages (logger, openapi, hono-zod-openapi, test-utils, dashboard, visualizer), fix stale dependency claims (yesql removed, stripe SDK → undici, protocol deps updated) - service/ARCHITECTURE.md: rewrite for Temporal-based architecture - engine/ARCHITECTURE.md: fix stale export names (forward→pipe, etc.) - principles.md: add logger+openapi to approved shared utilities - AGENTS.md: update package table and isolation rule - Remove deprecated docs (impl-status.md, overview.md) Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude * got the outline going * More outline update * docs(slides): add knowledge-transfer deck and bump node to 24 Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude * Align KT slides with the current sync protocol and runtime surfaces The knowledge-transfer deck had drifted away from the current engine, webhook, and connector behavior. This updates the slides to match the actual code paths, adds concrete protocol examples, and marks deprecated protocol message wrappers so the presentation teaches the current direction instead of legacy shapes. Constraint: Slides needed to stay faithful to current code paths while remaining presentation-readable Rejected: Keep generic interface aliases and legacy protocol examples | too ambiguous for a codebase handoff Rejected: Present webhook flow through Temporal/liveLoop | newer /pipeline_handle_events direction is the simpler story to teach Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep slides aligned with the actual protocol and API surfaces; if the message union or webhook entrypoints change again, update the deck at the same time Tested: Built Slidev deck in Docker; verified preview served over HTTP on localhost:3030 Not-tested: Runtime behavior of the deprecated protocol annotations beyond type/docs usage Committed-By-Agent: codex Co-authored-by: codex * Keep generated OpenAPI specs aligned with protocol deprecation docs The protocol annotation updates changed generated schema descriptions for deprecated eof payloads. The pre-push hook rejects drift, so this records the regenerated engine and service OpenAPI artifacts alongside the slide/protocol update already on the branch. Constraint: Repo pre-push hook requires generated OpenAPI files to match current source annotations Rejected: Push docs/protocol changes without regenerating specs | hook blocks on generated drift Confidence: high Scope-risk: narrow Reversibility: clean Directive: Any protocol description or schema metadata change that feeds OpenAPI should be followed immediately by ./scripts/generate-openapi.sh Tested: Ran ./scripts/generate-openapi.sh and inspected generated engine/service OpenAPI diffs Not-tested: Full downstream client compatibility beyond generated type refresh Committed-By-Agent: codex Co-authored-by: codex * docs(slides): refine knowledge-transfer deck and add styles Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude * update slides * fix(demo): align write-to-postgres with protocol message format Records must be nested as `{"type":"record","record":{...}}` per the RecordMessage schema, and the catalog needs `newer_than_field` with a matching timestamp in record data for destination-postgres upserts. Also adds dummy-source demo scripts and reorders KT slides. Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude * Polish the KT deck so the presentation reads cleanly live This pass simplifies the opening slide, tightens the webhook and subdivision explanations, and adjusts layout choices so the deck behaves better in Slidev during an actual presentation rather than just reading well in source form. Constraint: Slides need to fit the live Slidev rendering model, not just look correct in markdown Rejected: Leave the slide structure as-is and rely on browser refreshes | rendering/layout issues were slide-source problems, not just stale client state Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep presentation changes grouped and verify them in the running Slidev preview before committing Tested: Rebuilt Slidev deck in Docker repeatedly and verified preview responses on localhost:3030 Not-tested: Remote browser rendering differences beyond the local preview Committed-By-Agent: codex Co-authored-by: codex --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: codex --- AGENTS.md | 28 +- apps/engine/src/__generated__/openapi.d.ts | 2 +- apps/engine/src/__generated__/openapi.json | 2 +- apps/service/src/__generated__/openapi.d.ts | 2 +- apps/service/src/__generated__/openapi.json | 2 +- .../temporal/workflows/pipeline-backfill.ts | 2 +- demo/README.md | 16 + demo/dummy-source-to-dummy-destination.sh | 4 + demo/dummy-source.ndjson | 3 + demo/dummy-source.sh | 4 + demo/read-from-stripe.sh | 2 +- demo/write-to-postgres.sh | 10 +- docs/architecture/impl-status.md | 186 ------- docs/architecture/packages.md | 181 +++++-- docs/architecture/principles.md | 2 +- docs/engine/ARCHITECTURE.md | 2 +- docs/service/ARCHITECTURE.md | 32 +- docs/service/overview.md | 325 ----------- docs/service/scenarios.md | 2 +- docs/slides/Dockerfile | 4 +- docs/slides/knowledge-transfer-outline.md | 53 ++ docs/slides/knowledge-transfer.md | 504 ++++++++++++++++++ docs/slides/package.json | 3 +- docs/slides/style.css | 33 +- packages/protocol/src/protocol.ts | 10 +- 25 files changed, 808 insertions(+), 606 deletions(-) create mode 100755 demo/dummy-source-to-dummy-destination.sh create mode 100644 demo/dummy-source.ndjson create mode 100755 demo/dummy-source.sh delete mode 100644 docs/architecture/impl-status.md delete mode 100644 docs/service/overview.md create mode 100644 docs/slides/knowledge-transfer-outline.md create mode 100644 docs/slides/knowledge-transfer.md diff --git a/AGENTS.md b/AGENTS.md index a1e3ca3b2..4e0070b88 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,8 @@ If you add a migration, register it in `packages/state-postgres/src/migrations/i ## Architecture at a Glance -Sources and destinations are isolated connectors that only depend on `protocol`. +Sources and destinations are isolated connectors that only depend on `protocol` +and approved shared utilities (`logger`, `openapi`, `util-postgres`). The engine loads connectors (in-process or subprocess), pipes source output through destination input, and manages state checkpoints. See [docs/architecture/packages.md](docs/architecture/packages.md) for the full dependency graph. @@ -37,23 +38,28 @@ for the full dependency graph. | Package | Purpose | Depends on | | ------------------------------------ | --------------------------------------------------------- | ---------------------------------------- | -| `packages/protocol` | Message types, Source/Destination interfaces, Zod schemas | `zod` only | -| `packages/openapi` | Stripe OpenAPI spec fetching and parsing | standalone | -| `packages/source-stripe` | Stripe API source connector | `protocol`, `openapi` | -| `packages/destination-postgres` | Postgres destination connector | `protocol`, `util-postgres` | -| `packages/destination-google-sheets` | Google Sheets destination connector | `protocol` | -| `packages/state-postgres` | Postgres state store + migrations | `util-postgres` | -| `packages/util-postgres` | Shared Postgres utilities (upsert, rate limiter) | standalone | -| `packages/ts-cli` | Generic TypeScript module CLI runner | standalone | +| `packages/protocol` | Message types, Source/Destination interfaces, Zod schemas | `zod`, `citty`, `ix` | +| `packages/openapi` | Stripe OpenAPI spec fetching and parsing | `zod` | +| `packages/logger` | Structured logging (pino) + progress UI (ink) | `pino`, `ink`; peer: `protocol` | +| `packages/source-stripe` | Stripe API source connector | `protocol`, `openapi`, `logger` | +| `packages/destination-postgres` | Postgres destination connector | `protocol`, `util-postgres`, `logger` | +| `packages/destination-google-sheets` | Google Sheets destination connector | `protocol`, `logger` | +| `packages/state-postgres` | Postgres state store + migrations | `util-postgres`, `logger` | +| `packages/util-postgres` | Shared Postgres utilities (upsert, rate limiter) | `logger`, `pg` | +| `packages/hono-zod-openapi` | Hono + zod-openapi integration for spec generation | `hono`, `zod`, `zod-openapi` | +| `packages/test-utils` | Shared test helpers (servers, seeds, fixtures) | `destination-postgres`, `openapi`, `pg` | +| `packages/ts-cli` | Generic TypeScript module CLI runner | `citty` | | `apps/engine` | Sync engine library + stateless CLI + HTTP API | `protocol`, connectors, `state-postgres` | -| `apps/service` | Stateful service (credentials, state management) | `engine` | +| `apps/service` | Pipeline management + Temporal workflows | `engine`, Temporal SDK | +| `apps/dashboard` | React web UI for pipeline management | `openapi-fetch`, `radix-ui` | +| `apps/visualizer` | Next.js data visualization tool | `next`, `source-stripe`, `pglite` | | `apps/supabase` | Supabase edge functions (Deno runtime) | `protocol`, `engine`, connectors | | `e2e/` | Cross-package conformance and layer tests | all packages | ## Key Rules 0. **This file is an index, not a rulebook** — before adding anything here, check if it belongs in [docs/architecture/principles.md](docs/architecture/principles.md), [docs/architecture/decisions.md](docs/architecture/decisions.md), or another doc first. Only add to AGENTS.md if no better home exists. -1. **Connector isolation** — sources never import destinations, both depend only on `protocol`. Enforced by `e2e/layers.test.ts`. +1. **Connector isolation** — sources never import destinations, both depend only on `protocol` + approved shared utilities. Enforced by `e2e/layers.test.ts`. 2. **State is a message** — connectors never access state storage directly. State in = `cursor_in`; state out = `SourceStateMessage`. 3. **Snake_case on the wire** — all Zod schemas and JSON wire format use snake_case. 4. **api_version is required** — always mandatory in Stripe source config. Never optional. diff --git a/apps/engine/src/__generated__/openapi.d.ts b/apps/engine/src/__generated__/openapi.d.ts index 7636a8136..f97b5d0d7 100644 --- a/apps/engine/src/__generated__/openapi.d.ts +++ b/apps/engine/src/__generated__/openapi.d.ts @@ -951,7 +951,7 @@ export interface components { type: "eof"; eof: components["schemas"]["EofPayload"]; }; - /** @description Terminal message signaling end of this request. */ + /** @description Deprecated terminal message signaling end of this request. Prefer explicit request/response results via pipeline_sync_batch. */ EofPayload: { /** @description Terminal run status derived from stream outcomes. */ status: components["schemas"]["RunStatus"]; diff --git a/apps/engine/src/__generated__/openapi.json b/apps/engine/src/__generated__/openapi.json index 9b1cfb4e0..27deccd65 100644 --- a/apps/engine/src/__generated__/openapi.json +++ b/apps/engine/src/__generated__/openapi.json @@ -2628,7 +2628,7 @@ "run_progress", "request_progress" ], - "description": "Terminal message signaling end of this request." + "description": "Deprecated terminal message signaling end of this request. Prefer explicit request/response results via pipeline_sync_batch." }, "SyncState": { "type": "object", diff --git a/apps/service/src/__generated__/openapi.d.ts b/apps/service/src/__generated__/openapi.d.ts index c45a8407c..0ee6fcf1e 100644 --- a/apps/service/src/__generated__/openapi.d.ts +++ b/apps/service/src/__generated__/openapi.d.ts @@ -655,7 +655,7 @@ export interface components { /** @description Latest full sync checkpoint emitted by the engine. Includes source, destination, and sync-run state for the next request. */ sync_state?: components["schemas"]["SyncState"]; }; - /** @description Terminal message signaling end of this request. */ + /** @description Deprecated terminal message signaling end of this request. Prefer explicit request/response results via pipeline_sync_batch. */ EofPayload: { /** @description Terminal run status derived from stream outcomes. */ status: components["schemas"]["RunStatus"]; diff --git a/apps/service/src/__generated__/openapi.json b/apps/service/src/__generated__/openapi.json index 15797765f..964e1e6ae 100644 --- a/apps/service/src/__generated__/openapi.json +++ b/apps/service/src/__generated__/openapi.json @@ -2333,7 +2333,7 @@ "request_progress" ], "additionalProperties": false, - "description": "Terminal message signaling end of this request." + "description": "Deprecated terminal message signaling end of this request. Prefer explicit request/response results via pipeline_sync_batch." } } } diff --git a/apps/service/src/temporal/workflows/pipeline-backfill.ts b/apps/service/src/temporal/workflows/pipeline-backfill.ts index 3ce8a62fe..ae7bcfff0 100644 --- a/apps/service/src/temporal/workflows/pipeline-backfill.ts +++ b/apps/service/src/temporal/workflows/pipeline-backfill.ts @@ -39,7 +39,7 @@ export async function pipelineBackfill( operationCount++ if (!result.eof.has_more) { - if (result.eof.run_progress.derived.status === 'failed') { + if (result.eof.status === 'failed') { const message = result.eof.run_progress.connection_status?.message ?? 'Sync failed' throw ApplicationFailure.nonRetryable(message, 'SyncFailed') } diff --git a/demo/README.md b/demo/README.md index 79e0f5fd9..289f3af62 100644 --- a/demo/README.md +++ b/demo/README.md @@ -58,6 +58,22 @@ export STRIPE_API_KEY=sk_test_... You'll see a stream of JSON records — one per line — for each Stripe object. +## Dummy protocol demo + +For presentation or protocol-only walkthroughs, there is also a zero-dependency dummy stream: + +```sh +./demo/dummy-source.sh +./demo/dummy-source-to-dummy-destination.sh +``` + +This prints a tiny NDJSON stream with: + +- two `record` messages +- one trailing `source_state` message + +It is useful when you want to explain the wire protocol without needing Stripe or Postgres. + ## Step 2: Write to Postgres The destination connector reads NDJSON from stdin and writes to Postgres: diff --git a/demo/dummy-source-to-dummy-destination.sh b/demo/dummy-source-to-dummy-destination.sh new file mode 100755 index 000000000..dfcf900b9 --- /dev/null +++ b/demo/dummy-source-to-dummy-destination.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")" +./dummy-source.sh | cat diff --git a/demo/dummy-source.ndjson b/demo/dummy-source.ndjson new file mode 100644 index 000000000..22e3d2df4 --- /dev/null +++ b/demo/dummy-source.ndjson @@ -0,0 +1,3 @@ +{"type":"record","record":{"stream":"demo_customers","data":{"id":"cus_demo_1","email":"alice@example.com"},"emitted_at":"2026-01-01T00:00:00.000Z"}} +{"type":"record","record":{"stream":"demo_customers","data":{"id":"cus_demo_2","email":"bob@example.com"},"emitted_at":"2026-01-01T00:00:01.000Z"}} +{"type":"source_state","source_state":{"state_type":"stream","stream":"demo_customers","data":{"cursor":"cus_demo_2"}}} diff --git a/demo/dummy-source.sh b/demo/dummy-source.sh new file mode 100755 index 000000000..3fdc95091 --- /dev/null +++ b/demo/dummy-source.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")" +cat dummy-source.ndjson diff --git a/demo/read-from-stripe.sh b/demo/read-from-stripe.sh index 5499c7f60..e817c781b 100755 --- a/demo/read-from-stripe.sh +++ b/demo/read-from-stripe.sh @@ -17,4 +17,4 @@ echo "Stripe: $ACCT" >&2 $RUN packages/source-stripe/src/bin.ts read \ --config "{\"api_key\": \"$STRIPE_API_KEY\", \"backfill_limit\": 10}" \ - --catalog '{"streams":[{"stream":{"name":"product","primary_key":[["id"]]},"sync_mode":"full_refresh","destination_sync_mode":"append"}]}' + --catalog '{"streams":[{"stream":{"name":"products","primary_key":[["id"]]},"sync_mode":"full_refresh","destination_sync_mode":"append"}]}' diff --git a/demo/write-to-postgres.sh b/demo/write-to-postgres.sh index 97b3ae433..94145555c 100755 --- a/demo/write-to-postgres.sh +++ b/demo/write-to-postgres.sh @@ -19,19 +19,19 @@ CONFIG="{\"connection_string\": \"$DATABASE_URL\", \"schema\": \"public\"}" if [ -t 0 ]; then # No pipe — use sample data - CATALOG='{"streams":[{"stream":{"name":"demo","primary_key":[["id"]]},"sync_mode":"full_refresh","destination_sync_mode":"append"}]}' + CATALOG='{"streams":[{"stream":{"name":"demo","primary_key":[["id"]],"newer_than_field":"_updated_at"},"sync_mode":"full_refresh","destination_sync_mode":"append"}]}' $DEST setup --config "$CONFIG" --catalog "$CATALOG" printf '%s\n' \ - '{"type":"record","stream":"demo","data":{"id":"1","name":"Alice","email":"alice@example.com"},"emitted_at":"2024-01-01T00:00:00.000Z"}' \ - '{"type":"record","stream":"demo","data":{"id":"2","name":"Bob","email":"bob@example.com"},"emitted_at":"2024-01-01T00:00:00.000Z"}' \ + '{"type":"record","record":{"stream":"demo","data":{"id":"1","name":"Alice","email":"alice@example.com","_updated_at":1704067200},"emitted_at":"2024-01-01T00:00:00.000Z"}}' \ + '{"type":"record","record":{"stream":"demo","data":{"id":"2","name":"Bob","email":"bob@example.com","_updated_at":1704067200},"emitted_at":"2024-01-01T00:00:00.000Z"}}' \ | $DEST write --config "$CONFIG" --catalog "$CATALOG" else # Piped — buffer stdin, extract stream names, setup, then write DATA=$(cat) STREAMS=$(echo "$DATA" | node -e " let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>{ - const names=[...new Set(d.split('\n').filter(Boolean).map(l=>JSON.parse(l)).filter(m=>m.type==='record').map(m=>m.stream))]; - const catalog={streams:names.map(n=>({stream:{name:n,primary_key:[['id']]},sync_mode:'full_refresh',destination_sync_mode:'append'}))}; + const names=[...new Set(d.split('\n').filter(Boolean).map(l=>JSON.parse(l)).filter(m=>m.type==='record').map(m=>m.record.stream))]; + const catalog={streams:names.map(n=>({stream:{name:n,primary_key:[['id']],newer_than_field:'_updated_at'},sync_mode:'full_refresh',destination_sync_mode:'append'}))}; console.log(JSON.stringify(catalog)); })") echo "Streams: $(echo "$STREAMS" | node -e "let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>console.log(JSON.parse(d).streams.map(s=>s.stream.name).join(', ')))")" >&2 diff --git a/docs/architecture/impl-status.md b/docs/architecture/impl-status.md deleted file mode 100644 index 797a8d8f1..000000000 --- a/docs/architecture/impl-status.md +++ /dev/null @@ -1,186 +0,0 @@ -# Implementation Status - -Single source of truth for what's done, what's in progress, and what's left. - -**Last updated**: 2026-03-25 -**Branch**: `v2` - -## Packages - -Target structure from `packages.md`. Status: EXISTS / MISSING / WRONG. - -| Package | Status | Notes | -| ------------------------------------ | ------ | ----------------------------------------------------------------------------------------------- | -| `packages/protocol` | EXISTS | Zero deps. Defines Source, Destination interfaces. Message types + Zod schemas. | -| `packages/source-stripe` | EXISTS | Has `backfill.ts`, `streams/`, `openapi/`, `cli.ts`. Three input modes: stdin, WebSocket, HTTP. | -| `packages/destination-postgres` | EXISTS | Has real `write()`. No Stripe-specific knowledge (openapi moved out). | -| `packages/destination-google-sheets` | EXISTS | Fully implemented. `write()` works. E2E test. | -| `packages/state-postgres` | EXISTS | Migration runner + embedded migrations. | -| `packages/util-postgres` | EXISTS | Shared Postgres helpers (upsert, rate limiter). | -| `packages/ts-cli` | EXISTS | Generic TypeScript module CLI runner. | -| `apps/engine` | EXISTS | Engine (`createEngine`), connector loader, pipeline utils, CLI, HTTP API. | -| `apps/service` | EXISTS | `StatefulSync` class, store interfaces + file-based implementations. | -| `apps/supabase` | EXISTS | Supabase integration (edge functions). | - -## Architecture compliance - -### Isolation: source never imports destination - -`source-stripe` library code (`src/index.ts` entrypoint) has zero imports from `@stripe/sync-destination-postgres`. - -- Fastify server, `WebhookWriter`, `StripeSyncWebhook` all deleted — webhook HTTP listener now lives inside `read()` using Node `http.createServer` -- No `optionalDependencies` on `@stripe/sync-destination-postgres` - -### Isolation: destination has no Stripe-specific knowledge - -The source/destination boundary follows the protocol: `source.discover()` produces a catalog with `json_schema` (derived from the Stripe OpenAPI spec via `SpecParser` + `parsedTableToJsonSchema`), and `destination-postgres` reads `json_schema` to produce Postgres DDL via `schemaProjection.ts` (`buildCreateTableWithSchema`, `applySchemaFromCatalog`). - -Callers (`apps/sync-engine`) call `runMigrations()` for bootstrap, then `source.discover()` + `applySchemaFromCatalog()` for Stripe-specific schema. - -### Protocol interfaces - -`packages/protocol` defines both core interfaces: - -- `Source` — `spec()`, `check()`, `discover()`, `read(params, $stdin?)`; optional `setup?()`, `teardown?()` -- `Destination` — `spec()`, `check()`, `write(params, $stdin)`; optional `setup?()`, `teardown?()` - -There is no `Orchestrator` interface — orchestration is handled by `createEngine()` in `@stripe/sync-engine`. - -## Interface implementations - -### source-stripe - -| Method | Status | Test coverage | -| ---------------------------------------- | ------- | ----------------------------------------------------------------------------------- | -| `discover()` | REAL | 3 tests (catalog, filtering, empty) | -| `read()` backfill mode | REAL | 8 tests (pagination, resume, errors) | -| `read(params, $stdin)` webhook/live mode | REAL | 15 tests — full pipeline: sig verify, delete, revalidation, entitlements, sub items | -| `read()` HTTP server mode | REAL | Built-in `http.createServer` on `webhook_port`, backpressure via async generator | -| `read()` backfill→live transition | MISSING | .todo test stub | -| `fromWebhookEvent()` | REAL | 7 tests (simpler path, still used by WebSocket drain queue) | -| `cli.ts` (source discover, source read) | REAL | Working argv-based entrypoint. | -| `openapi/` (Stripe schema→DDL) | REAL | Moved from destination-postgres. 4 test suites. | - -### destination-postgres - -| Method | Status | Test coverage | -| ------------------------------ | ------- | -------------------------------------------- | -| `write()` — schema setup | REAL | 3 tests | -| `write()` — batched upsert | REAL | 3 tests | -| `write()` — checkpoint re-emit | REAL | 1 test | -| `write()` — schema evolution | MISSING | No ALTER TABLE for new columns | -| `write()` — error protocol | REAL | 2 tests | -| `cli.ts` (dest write) | REAL | Reads NDJSON from stdin, writes to Postgres. | - -### destination-google-sheets - -| Method | Status | Test coverage | -| ------------------------------- | ------ | ------------------------------ | -| `write()` — full implementation | REAL | 1 E2E test (needs credentials) | -| Rate limit retry | REAL | Built into writer.ts | - -### apps/engine (engine) - -| Method / Function | Status | Test coverage | -| --------------------------- | ------ | --------------------------- | -| `createEngine()` | REAL | 10 tests (mock source/dest) | -| `forward()` | REAL | 10 tests | -| `collect()` | REAL | 4 tests | -| StreamStatusMessage routing | REAL | 1 test | -| `spawnSource()` | REAL | Subprocess adapter | -| `spawnDestination()` | REAL | Subprocess adapter | - -### apps/service (StatefulSync) - -| Method | Status | Test coverage | -| ----------------------------------- | ------ | ----------------- | -| `StatefulSync.run()` | REAL | Integration tests | -| State persistence (load from store) | REAL | Integration tests | -| Credential refresh on auth_error | REAL | Unit test | - -## Scenario test coverage - -### source-stripe scenarios - -| Scenario | Status | -| --------------------------------------------------------------- | --------------------------- | -| discover() returns CatalogMessage with known streams | PASS | -| read() backfill emits RecordMessage + StateMessage interleaving | PASS | -| read() with prior state resumes from cursor | PASS | -| read() transitions backfill → live | TODO | -| Live webhook emits RecordMessage + StateMessage per event | PASS (via fromWebhookEvent) | -| Live WebSocket same as webhook | PASS (via fromWebhookEvent) | -| read() ErrorMessage transient_error on rate limit | PASS | -| read() ErrorMessage config_error on bad API key | PASS | -| Source never imports destination | PASS | - -### destination-postgres scenarios - -| Scenario | Status | -| ------------------------------------------ | ------ | -| write() creates tables from CatalogMessage | PASS | -| write() upserts with primary_key dedup | PASS | -| write() re-emits StateMessage after commit | PASS | -| write() batches inserts (configurable) | PASS | -| write() schema evolution (new columns) | NONE | -| write() ErrorMessage on connection failure | PASS | -| Destination never imports source | PASS | - -### engine scenarios - -| Scenario | Status | -| --------------------------------------------- | ------ | -| Persists state per stream | PASS | -| Passes full state map on resume | PASS | -| Filters: only data messages reach destination | PASS | -| Routes ErrorMessage to error handling | PASS | -| Routes LogMessage to observability | PASS | -| Routes StreamStatusMessage to progress | PASS | - -### cross-cutting scenarios - -| Scenario | Status | -| -------------------------------------- | ------ | -| Same-DB: engine + dest on one Postgres | NONE | -| Supabase dashboard installation | NONE | - -## Remaining work (ordered by priority) - -### P0 — Architecture violations - -All P0 items resolved: - -- ~~source-stripe imports from destination-postgres~~ → Fixed: Fastify server, WebhookWriter, StripeSyncWebhook deleted -- ~~openapi/ in destination-postgres~~ → Moved to source-stripe -- ~~Monolithic stripeSource.ts~~ → Split into backfill.ts, streams/ -- ~~No engine in protocol~~ → Engine in stateless-sync (`createEngine`) - -### P1 — Should do (completeness) - -1. **Implement schema evolution** in destination-postgres. ALTER TABLE for new columns discovered in subsequent CatalogMessage. - -2. **Fill remaining .todo test stubs**: backfill→live transition. - -3. **Add dedicated tests for `liveReader`** in source-stripe. The async generator exists but has no standalone test suite. - -### Known limitations - -- **Entitlement reconciliation gap**: `read(params, $stdin)` for `entitlements.active_entitlement_summary.updated` yields the current active entitlement set but cannot delete stale entitlements — the source doesn't know what's in the destination. Stale entitlements accumulate until the next full refresh. Fix requires a new `StreamResetMessage` (or similar) that tells the destination to clear-and-replace a subset (e.g., `WHERE customer = :id`). - -## Completion criteria - -- [x] `packages/fastify-app` does not exist -- [x] `packages/source-stripe` has `backfill.ts` and `live.ts` (not monolithic stripeSource.ts) -- [x] `packages/source-stripe` has `streams/` (not schemas/) -- [x] `packages/source-stripe` webhook HTTP listener lives inside `read()` (no separate server/) -- [x] `packages/source-stripe` has `openapi/` (Stripe schema→DDL) -- [x] `packages/stateless-sync` exists with engine + connector loader -- [x] `packages/stateful-sync` exists with `StatefulSync` class -- [x] All CLI stubs replaced with real implementations -- [x] `Source` and `Destination` interfaces defined in `packages/protocol` -- [x] `createEngine()` in `@stripe/sync-engine` -- [x] Zero cross-boundary imports between source/destination library code -- [x] `pnpm build && pnpm lint` clean -- [ ] All .todo test stubs filled or removed with justification -- [ ] All scenarios.md tests at PASS or explicitly descoped with reason -- [ ] Schema evolution in destination-postgres diff --git a/docs/architecture/packages.md b/docs/architecture/packages.md index 507893059..aa05a9dd1 100644 --- a/docs/architecture/packages.md +++ b/docs/architecture/packages.md @@ -1,88 +1,127 @@ # Monorepo Packages -The sync engine decomposes into packages along the architecture's isolation boundaries. The rule is simple: **sources and destinations never depend on each other.** They only depend on the core protocol. +The sync engine decomposes into packages along the architecture's isolation boundaries. The rule is simple: **sources and destinations never depend on each other.** They only depend on the core protocol and approved shared utilities. ``` packages/ ├── protocol/ ← core protocol (message types, interfaces, Zod schemas) +├── openapi/ ← Stripe OpenAPI spec fetching and parsing +├── logger/ ← structured logging (pino) + progress UI (ink) ├── source-stripe/ ← Stripe API source connector ├── destination-postgres/ ← Postgres destination connector ├── destination-google-sheets/← Google Sheets destination connector ├── state-postgres/ ← Postgres state store (migration runner + embedded migrations) ├── util-postgres/ ← shared Postgres utilities (upsert, rate limiter) +├── hono-zod-openapi/ ← Hono + zod-openapi integration for spec generation +├── test-utils/ ← shared test helpers (servers, seeds, Postgres fixtures) └── ts-cli/ ← generic TypeScript module CLI runner (private) apps/ ├── engine/ ← sync engine library + stateless CLI + HTTP API -├── service/ ← stateful service (credential/state management) +├── service/ ← stateful service (pipeline management, Temporal workflows) +├── dashboard/ ← React web UI for pipeline management +├── visualizer/ ← Next.js data visualization tool └── supabase/ ← Supabase edge functions (Deno runtime) ``` ## Dependency graph ``` - ┌────────────────┐ ┌──────────────┐ ┌──────────────┐ - │ protocol │ │state-postgres │ │ util-postgres │ - │ (types only) │ │ (pg only) │ │ (pg only) │ - └───────┬────────┘ └──────────────┘ └──────────────┘ - │ standalone — no protocol dep - ┌─────┼───────────┐ injected by apps at composition time + ┌────────────────┐ ┌──────────────┐ ┌──────────────┐ ┌──────────┐ + │ protocol │ │state-postgres │ │ util-postgres │ │ logger │ + │ (types+schemas)│ │ (pg only) │ │ (pg only) │ │ (pino) │ + └───────┬────────┘ └──────────────┘ └──────────────┘ └──────────┘ + │ depends on depends on shared by + ┌─────┼───────────┐ util-postgres logger most pkgs │ │ │ sources │ destinations (stripe) │ (pg, sheets) │ │ │ │ apps/engine │ ← engine + connector loader + pipeline utils + CLI + API - │ (protocol only)│ (depends on protocol, state-postgres) + │ (protocol, │ (depends on protocol, state-postgres, hono-zod-openapi) + │ connectors) │ │ │ │ - │ apps/service │ ← store interfaces + StatefulSync coordinator - │ (engine only) │ (no pg dep — stores are injected by CLI/API) + │ apps/service │ ← pipeline management + webhook ingress via Temporal + │ (engine, │ (depends on engine, Temporal SDK) + │ temporal) │ │ │ │ │ │ NO ARROWS BETWEEN │ │ SOURCES ↔ DESTINATIONS │ │ + ├─ apps/dashboard ─→ React SPA consuming service API via openapi-fetch + ├─ apps/visualizer ─→ Next.js app for data exploration └─ apps/supabase ─→ protocol + source-stripe + destination-postgres + state-postgres + apps/engine ``` ### Canonical dependency layering -| Layer | Package | Depends on | -| ------------ | -------------------------------------------------------------------- | ------------------------------------------------------------------ | -| Core | `protocol` | nothing (only `zod`) | -| Connectors | `source-stripe`, `destination-postgres`, `destination-google-sheets` | `protocol` only | -| Pg utilities | `state-postgres`, `util-postgres` | `pg` only (no protocol dep) | -| Engine + CLI | `apps/engine` | `protocol`, `state-postgres`, connectors | -| Service | `apps/service` | `apps/engine` | -| Integration | `apps/supabase` | `protocol`, `source-stripe`, `destination-postgres`, `apps/engine` | +| Layer | Package | Depends on | +| -------------- | -------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| Core | `protocol` | `zod`, `citty`, `ix` | +| Shared utils | `logger` | `pino`, `ink`, `react`; peer: `protocol` | +| Shared utils | `openapi` | `zod` | +| Shared utils | `util-postgres` | `logger`, `pg` | +| Shared utils | `hono-zod-openapi` | `hono`, `zod`, `zod-openapi` | +| Connectors | `source-stripe`, `destination-postgres`, `destination-google-sheets` | `protocol` + approved shared utilities (`logger`, `openapi`, `util-pg`) | +| State | `state-postgres` | `util-postgres`, `logger`, `pg` | +| Engine + CLI | `apps/engine` | `protocol`, `state-postgres`, connectors, `hono-zod-openapi`, `logger` | +| Service | `apps/service` | `apps/engine`, Temporal SDK | +| Frontend | `apps/dashboard` | `openapi-fetch` (consumes service API) | +| Frontend | `apps/visualizer` | `next`, `source-stripe`, `pglite` | +| Integration | `apps/supabase` | `protocol`, `source-stripe`, `destination-postgres`, `apps/engine` | +| Test infra | `test-utils` | `destination-postgres`, `openapi`, `hono`, `pg` | **Key rules:** -- Connectors only depend on `protocol` — never on each other or on infrastructure. -- `state-postgres` and `util-postgres` are standalone `pg`-only packages — they have no sync-engine workspace dependencies. Apps inject them at composition time. +- Connectors depend on `protocol` and approved shared utilities (`logger`, `openapi`, `util-postgres`) — never on each other or on infrastructure. +- `state-postgres` and `util-postgres` are `pg`-based packages with no connector dependencies. Apps inject them at composition time. - `apps/service` does NOT depend directly on `pg`; Postgres stores are injected by the CLI/API entrypoints. ## Packages ### `protocol` — core protocol -The shared foundation. Every connector depends on this. It has **zero** dependencies on any source, destination, or infrastructure implementation. Contains only types, interfaces, and Zod schemas. - -Contains: message types (`RecordMessage`, `StateMessage`, `CatalogMessage`), `Source`/`Destination` interfaces, Zod schemas (`ConnectorSpecification`, `ConfiguredCatalog`), and message helper functions. +The shared foundation. Every connector depends on this. Contains message types, interfaces, Zod schemas, and async iterable utilities. **Package name:** `@stripe/sync-protocol` -**Exports:** Message types, Source/Destination interfaces, Zod schemas, message helpers. +**Exports:** `"."` (message types, Source/Destination interfaces, Zod schemas, message helpers), `"./cli"` (CLI argument definitions). + +**Dependencies:** `zod` (schema validation), `citty` (CLI argument types), `ix` (async iterable utilities). + +### `openapi` — Stripe OpenAPI spec parsing + +Fetches and parses Stripe OpenAPI specs. Provides version discovery and type generation utilities. + +**Package name:** `@stripe/sync-openapi` + +**Exports:** `"."` (main), `"./browser"` (browser-safe subset). + +**Dependencies:** `zod`. + +### `logger` — structured logging + progress UI -**Dependencies:** `zod` (for schema validation). +Structured logging via pino with AsyncLocalStorage context propagation. Also provides an ink-based terminal progress UI for the CLI. + +**Package name:** `@stripe/sync-logger` + +**Exports:** `"."` (main logger), `"./progress"` (ink progress renderer). + +**Binary:** `sync-pretty-log` (pino pretty-print transport). + +**Dependencies:** `pino`, `ink`, `react`. **Peer:** `@stripe/sync-protocol`. ### `source-stripe` — Stripe API source -Reads from the Stripe REST API via list endpoints (backfill), events API (incremental pull), webhooks (push), and WebSocket (live dev). Includes OpenAPI spec parsing for automatic catalog discovery. +Reads from the Stripe REST API via list endpoints (backfill), events API (incremental pull), webhooks (push), and WebSocket (live dev). Uses the raw HTTP client (undici) rather than the Stripe SDK. Includes OpenAPI spec parsing for automatic catalog discovery. **Package name:** `@stripe/sync-source-stripe` -**Exports:** `StripeSource` (implements `Source`), `spec` (Zod config schema), default export. +**Exports:** `"."` (main), `"./browser"` (browser-safe), `"./client"` (HTTP client). -**Dependencies:** `@stripe/sync-protocol`, `stripe` (Stripe SDK). +**Binary:** `source-stripe` + +**Dependencies:** `@stripe/sync-protocol`, `@stripe/sync-openapi`, `@stripe/sync-logger`, `undici`, `ws`, `zod`. **Must NOT depend on:** Any destination or infrastructure package. @@ -92,9 +131,11 @@ Writes records into a Postgres database. Creates tables from catalog, upserts re **Package name:** `@stripe/sync-destination-postgres` -**Exports:** `PostgresDestination` (implements `Destination`), `PostgresDestinationWriter`, `spec`, default export. +**Binary:** `destination-postgres` + +**Dependencies:** `@stripe/sync-protocol`, `@stripe/sync-util-postgres`, `@stripe/sync-logger`, `pg`, `zod`. -**Dependencies:** `@stripe/sync-protocol`, `pg`, `yesql`. +**Optional peers:** `@aws-sdk/client-sts`, `@aws-sdk/rds-signer` (for IAM auth). **Must NOT depend on:** Any source or infrastructure package. @@ -104,9 +145,9 @@ Writes records into a Google Sheets spreadsheet. **Package name:** `@stripe/sync-destination-google-sheets` -**Exports:** `SheetsDestination` (implements `Destination`), `spec`, default export. +**Binary:** `destination-google-sheets` -**Dependencies:** `@stripe/sync-protocol`, `googleapis`. +**Dependencies:** `@stripe/sync-protocol`, `@stripe/sync-logger`, `googleapis`, `zod`. **Must NOT depend on:** Any source or infrastructure package. @@ -116,9 +157,7 @@ Postgres-specific migration infrastructure. Runs bootstrap and Stripe-specific S **Package name:** `@stripe/sync-state-postgres` -**Exports:** `runMigrations`, `runMigrationsFromContent`, `embeddedMigrations`, `genericBootstrapMigrations`, `renderMigrationTemplate`. - -**Dependencies:** `pg`. +**Dependencies:** `@stripe/sync-util-postgres`, `@stripe/sync-logger`, `pg`. ### `util-postgres` — shared Postgres utilities @@ -126,19 +165,35 @@ Shared Postgres helpers used by multiple packages. Batched upsert with timestamp **Package name:** `@stripe/sync-util-postgres` -**Exports:** `upsertMany`, `createRateLimiter`. +**Dependencies:** `@stripe/sync-logger`, `pg`. + +### `hono-zod-openapi` — OpenAPI integration for Hono -**Dependencies:** `pg`, `yesql`. +Bridges Hono's HTTP framework with zod-openapi for automatic OpenAPI spec generation from Zod route schemas. + +**Package name:** `@stripe/sync-hono-zod-openapi` + +**Dependencies:** `@hono/zod-validator`, `hono`, `zod`, `zod-openapi`. + +### `test-utils` — shared test helpers + +Test utilities shared across packages: seed data, Postgres fixtures, mock HTTP servers, OpenAPI validation helpers. + +**Package name:** `@stripe/sync-test-utils` (private) + +**Binary:** `sync-test-utils-server` (test HTTP server). + +**Dependencies:** `@hono/node-server`, `@stripe/sync-destination-postgres`, `@stripe/sync-openapi`, `hono`, `pg`. ### `ts-cli` — TypeScript module CLI runner -Generic CLI tool that can call any exported function/method from a TypeScript module, with support for stdin piping, positional args, and named args. Used for ad-hoc testing and scripting. +Generic CLI tool that can call any exported function/method from a TypeScript module, with support for stdin piping, positional args, and named args. Also provides NDJSON streaming and OpenAPI client helpers. **Package name:** `@stripe/sync-ts-cli` (private) -**Exports:** `run` (CLI entrypoint). +**Exports:** `"."` (CLI entrypoint), `"./config"`, `"./ndjson"`, `"./openapi"`, `"./env-proxy"`. -**Dependencies:** None. +**Dependencies:** `citty`. ### `apps/engine` — sync engine library + stateless CLI + HTTP API @@ -150,28 +205,46 @@ Published as the user-facing npm package. **Exports:** -- `"."` — library: `createEngine`, `createConnectorResolver`, `SyncParams`, `forward`, `collect`, `filterDataMessages`, `sourceTest`, `destinationTest`, everything from `protocol` +- `"."` — library: `createEngine`, `createConnectorResolver`, `pipe`, `collect`, `filterType`, `enforceCatalog`, `sourceTest`, `destinationTest`, everything from `protocol` - `"./cli"` — CLI `CommandDef` (citty program, no side effects) - `"./api"` — `createApp` + `startApiServer` (side-effect-free module surface) +- `"./api/openapi-utils"` — connector schema injection for runtime OAS spec +- `"./progress"` — progress rendering utilities **Binaries:** - `sync-engine` → `dist/bin/sync-engine.js` - `sync-engine-serve` → `dist/bin/serve.js` -**Dependencies:** `@stripe/sync-protocol`, `@stripe/sync-state-postgres`, connectors, `citty`, `hono`, `dotenv`. +**Dependencies:** `@stripe/sync-protocol`, `@stripe/sync-state-postgres`, `@stripe/sync-hono-zod-openapi`, `@stripe/sync-logger`, connectors, `hono`, `openapi-fetch`, `pg`, `ws`. -### `apps/service` — stateful sync service +### `apps/service` — pipeline management service -Wraps the engine with credential management and state persistence. Defines store interfaces (`CredentialStore`, `StateStore`, `LogSink`) with file-based implementations. The `StatefulSync` coordinator loads config → credentials → state, resolves connectors, runs the engine, and persists output state. +Manages sync pipelines with credential storage, state persistence, and Temporal workflow orchestration. Provides a REST API for pipeline CRUD, webhook ingress, and sync triggering. Temporal workflows handle long-running syncs with retry, scheduling, and cancellation. **Package name:** `@stripe/sync-service` -**Exports:** Store interfaces + implementations, `StatefulSync`. +**Exports:** `createSchemas`, `createApp`, `createActivities`, `createWorker`, pipeline types. **Binary:** `sync-service` -**Dependencies:** `@stripe/sync-engine`. +**Dependencies:** `@stripe/sync-engine`, `@temporalio/activity`, `@temporalio/client`, `@temporalio/worker`, `@temporalio/workflow`, `hono`, `openapi-fetch`. + +### `apps/dashboard` — web UI + +React + Vite single-page application for managing sync pipelines. Consumes the service REST API via `openapi-fetch`. Uses Radix UI + Tailwind for styling. + +**Package name:** `@stripe/sync-dashboard` (private) + +**Dependencies:** `@radix-ui/*`, `lucide-react`, `openapi-fetch`, `react`, `tailwindcss`. + +### `apps/visualizer` — data visualization + +Next.js application for exploring and visualizing synced data. Embeds PGlite for client-side SQL and CodeMirror for query editing. + +**Package name:** `@stripe/sync-visualizer` (private) + +**Dependencies:** `@codemirror/*`, `@electric-sql/pglite`, `@stripe/sync-source-stripe`, `next`, `react`, `tailwindcss`. ### `apps/supabase` — Supabase integration @@ -183,13 +256,13 @@ Deployment target for the Supabase installation flow. Bundles edge functions (De ## Isolation rules -| Rule | Enforced by | -| -------------------------------------------------------------- | -------------------------------- | -| `source-*` packages never import from `destination-*` packages | CI lint: disallowed import paths | -| `destination-*` packages never import from `source-*` packages | CI lint: disallowed import paths | -| `source-*` and `destination-*` only depend on `protocol` | package.json audit | -| `protocol` has zero runtime deps beyond `zod` | package.json audit | -| `apps/service` does not depend directly on `pg` | package.json audit | +| Rule | Enforced by | +| --------------------------------------------------------------------------------- | -------------------------------- | +| `source-*` packages never import from `destination-*` packages | CI lint: disallowed import paths | +| `destination-*` packages never import from `source-*` packages | CI lint: disallowed import paths | +| `source-*` and `destination-*` only depend on `protocol` + approved shared utils | package.json audit | +| `protocol` has zero workspace dependencies | package.json audit | +| `apps/service` does not depend directly on `pg` | package.json audit | ## pnpm workspace diff --git a/docs/architecture/principles.md b/docs/architecture/principles.md index 1e83443b1..92da02eab 100644 --- a/docs/architecture/principles.md +++ b/docs/architecture/principles.md @@ -8,7 +8,7 @@ All data flows as typed async iterables of messages (`RecordMessage`, `SourceSta ## 2. Connector isolation -Sources never import destinations. Destinations never import sources. Both depend only on `@stripe/sync-protocol` and approved shared utilities (e.g. `@stripe/sync-util-postgres`). This is enforced by `e2e/layers.test.ts`. +Sources never import destinations. Destinations never import sources. Both depend only on `@stripe/sync-protocol` and approved shared utilities (`@stripe/sync-logger`, `@stripe/sync-openapi`, `@stripe/sync-util-postgres`). This is enforced by `e2e/layers.test.ts`. ## 3. State is a message diff --git a/docs/engine/ARCHITECTURE.md b/docs/engine/ARCHITECTURE.md index 4741dafc0..39d06499b 100644 --- a/docs/engine/ARCHITECTURE.md +++ b/docs/engine/ARCHITECTURE.md @@ -207,7 +207,7 @@ for await (const msg of engine.run()) { } ``` -`createEngine` lives in `@stripe/sync-engine`. Pipeline utilities (`forward`, `collect`, `filterDataMessages`) also live there. +`createEngine` lives in `@stripe/sync-engine`. Pipeline utilities (`pipe`, `collect`, `filterType`, `enforceCatalog`) also live there. ### Validation boundary diff --git a/docs/service/ARCHITECTURE.md b/docs/service/ARCHITECTURE.md index 2fe965325..df593149f 100644 --- a/docs/service/ARCHITECTURE.md +++ b/docs/service/ARCHITECTURE.md @@ -5,13 +5,13 @@ Stripe Sync lets merchants create **sync pipelines** that continuously move data ## System layers ``` -StatefulSync (credential management + state persistence + scheduling) +Service (pipeline management + Temporal workflows + webhook ingress) └── Engine (wires source → destination, persists state) ├── Source (reads upstream data) └── Destination (writes downstream data) ``` -- **StatefulSync** — the stateful layer. Manages the four stores (credentials, config, state, logs), resolves stored config into engine-ready `SyncParams`, and calls the engine. Exposed via `apps/stateful` (REST API + CLI). +- **Service** — the stateful layer. Manages pipelines (CRUD), credentials, state persistence, and orchestrates long-running syncs via Temporal workflows. Exposed via `apps/service` (REST API + CLI). - **Engine** — the runtime that pipes a source to a destination. Filters messages (only data messages reach the destination), persists committed state checkpoints, handles errors, and routes logs. See [`../engine/ARCHITECTURE.md`](../engine/ARCHITECTURE.md). @@ -19,11 +19,12 @@ StatefulSync (credential management + state persistence + scheduling) ## Core Model -A **Sync** (aka sync pipeline) connects a **source** to a **destination**. Both may reference a **credential** for authentication. +A **Pipeline** connects a **source** to a **destination**. Both may reference a **credential** for authentication. Pipelines are the unit of management — they can be created, synced, checked, set up, and torn down. - **SourceConfig** — where data comes from (e.g. Stripe API) - **DestinationConfig** — where data lands (e.g. Postgres, Google Sheets) - **Credential** — stored connection secrets (API keys, database passwords, OAuth tokens) +- **PipelineStatus** — lifecycle state tracking for the pipeline ## Why "source" and not just "Stripe"? @@ -33,13 +34,22 @@ The source isn't always Stripe. Other data providers may have their own source i A Stripe organization may want to sync from a specific Stripe account. The source needs a credential (API key) to authenticate. Third-party sources will always need a user-supplied credential. +## Temporal workflows + +Long-running sync operations are orchestrated via Temporal. This provides: + +- **Durable execution** — syncs survive process restarts +- **Retry with backoff** — transient failures are retried automatically +- **Cancellation** — running syncs can be cleanly stopped +- **Scheduling** — periodic syncs via Temporal schedules + ## Files -| File | Description | -| ----------------------------------- | ------------------------------------------------------------ | -| `packages/protocol/src/protocol.ts` | TypeScript interfaces for Source, Destination; message types | -| `apps/engine/src/lib/engine.ts` | `createEngine()` — engine factory | -| `apps/service/src/lib/service.ts` | `StatefulSync` class — the composition root | -| `apps/service/src/lib/stores.ts` | Store interfaces: CredentialStore, StateStore, LogSink | -| `apps/service/src/api/app.ts` | Service HTTP API | -| `apps/service/src/cli/index.ts` | Service CLI entrypoint | +| File | Description | +| ----------------------------------- | -------------------------------------------------------------- | +| `packages/protocol/src/protocol.ts` | TypeScript interfaces for Source, Destination; message types | +| `apps/engine/src/lib/engine.ts` | `createEngine()` — engine factory | +| `apps/service/src/lib/stores.ts` | Store interfaces: CredentialStore, StateStore, LogSink | +| `apps/service/src/api/app.ts` | Service HTTP API (pipeline CRUD, webhook ingress, sync trigger)| +| `apps/service/src/cli/index.ts` | Service CLI entrypoint | +| `apps/service/src/temporal/` | Temporal workflows, activities, and worker | diff --git a/docs/service/overview.md b/docs/service/overview.md deleted file mode 100644 index 9249bf233..000000000 --- a/docs/service/overview.md +++ /dev/null @@ -1,325 +0,0 @@ -# Sync Service Architecture - -## Context - -The sync engine is a pure function: `createEngine(params, { source, destination })`. It deliberately ignores four concerns that any real deployment needs: **credentials**, **config**, **state**, and **logs**. The sync service (`StatefulSync` in `packages/stateful-sync`) is the stateful layer that manages these concerns and calls the engine. - ---- - -## The Four Concerns - -| Concern | Sensitivity | Mutability | Access pattern | Lifetime | -| --------------- | ------------------------- | ----------------------------- | ------------------------------------ | ------------------------------------ | -| **Credentials** | High (encrypted, audited) | Rarely — except token refresh | Read on sync start, refresh mid-sync | Outlives syncs (shared across syncs) | -| **Config** | Low (user-editable) | User-initiated only | Read on sync start | Tied to sync definition | -| **State** | None (opaque cursors) | Every checkpoint (~seconds) | Read on resume, write continuously | Tied to sync progress | -| **Logs** | Low | Append-only, never updated | Write continuously, read for debug | Ephemeral (retention-bounded) | - -These must be stored separately because they have different: - -- **Security requirements** — credentials encrypted at rest + audit-logged; config/state/logs are not -- **Write frequency** — state writes every few seconds; credentials almost never; config only on user action -- **Sharing** — one credential serves many syncs; config/state are per-sync -- **Retention** — logs are pruned; state is cleared on full-refresh; credentials persist until revoked - ---- - -## Stored Form vs Resolved Form - -Two distinct shapes exist for a sync: - -### SyncConfig (stored form) - -What lives in the config store. Has credential **references**, no embedded state: - -```ts -type SyncConfig = { - id: string - source: { - type: string // e.g. "stripe" - credential_id?: string // reference → CredentialStore - [key: string]: unknown // non-sensitive source config - } - destination: { - type: string // e.g. "postgres" - credential_id?: string // reference → CredentialStore - [key: string]: unknown // non-sensitive destination config - } - streams?: Array<{ name: string; sync_mode?: 'incremental' | 'full_refresh' }> -} -``` - -### SyncParams (resolved form) - -What the engine receives. Credentials inlined, state passed separately. This is the `SyncParams` type from `@stripe/sync-engine`: - -```ts -type SyncParams = { - source: { name: string; [key: string]: unknown } // name + credential fields + config merged - destination: { name: string; [key: string]: unknown } // name + credential fields + config merged - streams?: Array<{ name: string; sync_mode?: 'incremental' | 'full_refresh' }> - state?: Record -} -``` - -### Resolution - -The service resolves stored → resolved before calling the engine: - -```ts -function resolve(opts: { - config: SyncConfig - sourceCred?: Credential - destCred?: Credential - state?: Record -}): SyncParams { - return { - source: { name: opts.config.source.type, ...opts.config.source, ...opts.sourceCred?.fields }, - destination: { - name: opts.config.destination.type, - ...opts.config.destination, - ...opts.destCred?.fields, - }, - streams: opts.config.streams, - state: opts.state, - } -} -``` - -The engine never sees credential IDs, never knows where config came from, never persists state. It's a pure transformation. - ---- - -## Four Store Interfaces - -Each concern gets a minimal generic interface. `StatefulSync` depends on these interfaces, not implementations. - -### CredentialStore - -```ts -interface CredentialStore { - get(id: string): Promise - set(id: string, credential: Credential): Promise - delete(id: string): Promise - list(): Promise -} - -type Credential = { - id: string - type: string // "stripe", "postgres", "google" - fields: Record // type-specific fields (api_key, tokens, etc.) - created_at: string - updated_at: string -} -``` - -Implementations: file-backed JSON (dev/CLI), encrypted Postgres table (cloud), Vault. - -### ConfigStore - -```ts -interface ConfigStore { - get(id: string): Promise - set(id: string, config: SyncConfig): Promise - delete(id: string): Promise - list(): Promise -} -``` - -Implementations: file-backed JSON (dev/CLI), Postgres table (cloud). - -### StateStore - -```ts -interface StateStore { - get(syncId: string): Promise | undefined> - set(syncId: string, stream: string, data: unknown): Promise - clear(syncId: string): Promise -} -``` - -Implementations: file-backed JSON (dev/CLI), Postgres `_sync_state` table (cloud). - -### LogSink - -```ts -interface LogSink { - write(syncId: string, entry: LogEntry): void // fire-and-forget, non-blocking -} - -type LogEntry = { - level: 'debug' | 'info' | 'warn' | 'error' - message: string - stream?: string - timestamp: string -} -``` - -Implementations: stderr (CLI), NDJSON file (dev), Postgres table (cloud). - ---- - -## Credential Refresh (Service-Level Retry) - -Token refresh is handled by the **service**, not the source. The source interface stays pure — `read(params)` with a plain config object, no credential providers or functions. - -### Why service-level, not per-request - -An alternative is injecting a `credentialProvider` into the source for per-request refresh (zero wasted work on 401). This was rejected because it changes the Source interface. The coarse retry tradeoff is acceptable: - -- Stripe API keys don't expire (most common case — refresh never triggers) -- OAuth tokens have ~1-hour lifetimes — a single retry handles it -- State is checkpointed, so re-runs resume near where they left off -- Upserts are idempotent — re-fetching a few pages causes no data corruption - -### How it works - -``` -1. Service resolves credentials → SyncParams (access_token inlined) -2. engine.run() runs, source paginates normally -3. On page N, source gets 401 → yields ErrorMessage { failure_type: 'auth_error' } -4. Service detects auth_error in the output stream -5. Service refreshes: - a. refreshToken(cred.fields.refresh_token) → new access_token - b. credentialStore.set(credId, { ...cred, fields: { ...fields, access_token } }) -6. Service re-resolves SyncParams with fresh token -7. Service re-runs engine — resumes from last checkpoint (near page N) -``` - -### Protocol change - -The `ErrorMessage.failure_type` enum includes `'auth_error'`: - -```ts -// packages/protocol/src/protocol.ts — ErrorMessage -failure_type: z.enum(['config_error', 'system_error', 'transient_error', 'auth_error']) -``` - -Sources yield `auth_error` on HTTP 401 or equivalent credential failures. - ---- - -## StatefulSync — The Composition Root - -`StatefulSync` wires everything together. It's not an abstraction — it's the caller code. - -```ts -class StatefulSync { - private credentials: CredentialStore - private configs: ConfigStore - private states: StateStore - private logs: LogSink - private connectors: ConnectorResolver - - constructor(opts: { - credentials: CredentialStore - configs: ConfigStore - states: StateStore - logs: LogSink - connectors: ConnectorResolver - }) { - Object.assign(this, opts) - } - - async *run(syncId: string): AsyncIterable { - const config = await this.configs.get(syncId) - const source = await this.connectors.loadSource(config.source.type) - const destination = await this.connectors.loadDestination(config.destination.type) - - let retries = 0 - const MAX_AUTH_RETRIES = 2 - - while (retries <= MAX_AUTH_RETRIES) { - // Load credentials (fresh on each attempt — may have been refreshed) - const sourceCred = config.source.credential_id - ? await this.credentials.get(config.source.credential_id) - : undefined - const destCred = config.destination.credential_id - ? await this.credentials.get(config.destination.credential_id) - : undefined - - // Load state (picks up checkpoints from previous attempt) - const state = await this.states.get(syncId) - - // Resolve to SyncParams - const params = resolve({ config, sourceCred, destCred, state }) - - // Create engine and run - const engine = createEngine(params, { source, destination }) - - let authError = false - - for await (const msg of engine.run()) { - if (msg.type === 'error' && msg.failure_type === 'auth_error') { - authError = true - break // exit pipeline, will retry - } - // Persist state checkpoint - if (msg.type === 'state') { - await this.states.set(syncId, msg.stream, msg.data) - } - yield msg - } - - if (!authError) return // success — all streams completed - - // Refresh the failed credential and retry - await this.refreshCredential(config.source.credential_id!) - retries++ - } - - throw new Error(`Auth failed after ${MAX_AUTH_RETRIES} refresh attempts`) - } -} -``` - -The four stores are injected via a named options object — the service doesn't know if they're Postgres, files, or in-memory. - -> **Note on `createEngine()`**: `StatefulSync` uses `createEngine()` directly (from `@stripe/sync-engine`). The engine is the real interface — `StatefulSync` adds only the store-loading and state-persistence wrapper around it. - ---- - -## Deployment Configurations - -The same `StatefulSync` class works across all deployment modes by swapping store implementations. Both CLI and API use 4 file-based stores under `--data-dir` / `DATA_DIR` / `~/.stripe-sync`: - -### Local dev / CLI / API (file-backed) - -```ts -const service = new StatefulSync({ - credentials: fileCredentialStore(path.join(dataDir, 'credentials.json')), - configs: fileConfigStore(path.join(dataDir, 'syncs.json')), - states: fileStateStore(path.join(dataDir, 'state.json')), - logs: fileLogSink(path.join(dataDir, 'logs.ndjson')), - connectors: createConnectorResolver(), -}) -``` - -### Cloud (Postgres + encrypted) - -```ts -const service = new StatefulSync({ - credentials: pgCredentialStore({ pool, encryptionKey }), // encrypted at rest - configs: pgConfigStore({ pool }), - states: pgStateStore({ pool }), - logs: pgLogSink({ pool }), - connectors: cachedConnectorResolver(), -}) -``` - -### What doesn't change - -The engine (`createEngine`), the source/destination connectors, the `SyncParams` shape, and the `StatefulSync.run()` method are identical across all deployment modes. Only the four store implementations vary. - ---- - -## Key Files - -| File | Role | -| --------------------------------------- | ------------------------------------------------------------------------------------ | -| `packages/protocol/src/protocol.ts` | `Source`/`Destination` interfaces, `ErrorMessage` (with `auth_error`), message types | -| `packages/stateless-sync/src/engine.ts` | `createEngine()` — the engine factory the service calls | -| `packages/stateful-sync/src/service.ts` | `StatefulSync` class, `resolve()` function | -| `packages/stateful-sync/src/stores.ts` | Store interfaces: `CredentialStore`, `ConfigStore`, `StateStore`, `LogSink` | -| `apps/stateful/src/cli/index.ts` | Stateful CLI entrypoint (wires file stores + StatefulSync) | -| `apps/stateful/src/api/app.ts` | Stateful HTTP API (CRUD + SSE sync execution) | diff --git a/docs/service/scenarios.md b/docs/service/scenarios.md index b21c042a2..ca6a84d79 100644 --- a/docs/service/scenarios.md +++ b/docs/service/scenarios.md @@ -97,4 +97,4 @@ Both target the same Postgres host with different schemas. | `scenarios.md` | This document | | `ARCHITECTURE.md` | System layers, core model, source/destination types | | `packages/protocol/src/protocol.ts` | Source, Destination interfaces; message types | -| `apps/service/src/lib/service.ts` | `StatefulSync` class — credential + state management | +| `apps/service/src/lib/stores.ts` | Store interfaces — credential + state management | diff --git a/docs/slides/Dockerfile b/docs/slides/Dockerfile index eae90a46f..3da3cc2f3 100644 --- a/docs/slides/Dockerfile +++ b/docs/slides/Dockerfile @@ -2,7 +2,7 @@ # locked-down environments (e.g. corp machines with restricted exec policies). # We work around this by doing a full static build inside Docker where esbuild # can run freely, then serving the output with a plain HTTP server. -FROM node:20-alpine AS build +FROM node:24-alpine AS build WORKDIR /app COPY package.json . RUN npm install --legacy-peer-deps @@ -11,7 +11,7 @@ COPY ${SLIDES} slides.md COPY style.css . RUN NODE_OPTIONS="--max-old-space-size=4096" ./node_modules/.bin/slidev build --base / slides.md -FROM node:20-alpine +FROM node:24-alpine WORKDIR /app RUN npm install -g serve COPY --from=build /app/dist . diff --git a/docs/slides/knowledge-transfer-outline.md b/docs/slides/knowledge-transfer-outline.md new file mode 100644 index 000000000..aba6e2e4f --- /dev/null +++ b/docs/slides/knowledge-transfer-outline.md @@ -0,0 +1,53 @@ +# Sync Engine Knowledge Transfer Outline + +- sync engine is ... +- many deployment targets + - as a library (replit, supabasea) + - as cli (direct users, contributors) + - docker (kubernetes, internal stripe) +- core of sync engine is the protocol + - walk through different message types and what they are for +- 3 components and their roles + - source (interface) + - destination (interface) + - engine (interface) +- transport agnostic + - in-memory + - stdin / out + - http + +- dummy source & destination, simply using the file system + - let's try it out. echo source | cat destination + - now add the engine in there, and we get to keep track of the progress of sync +- source-stripe | dummy destination + - deep dive +- dummy source | postgres destination + - deep dive into postgres +- role of the engine + - concept of a single sync run + - progress tracking +- source-stripe | postgres destiination +- Putting it all together + - scheduled sync + - start new sync run + - 1000 state messages per pipeline_sync requedst + - keep going until has_more: false + - done + - `/pipeline_sync_batch` + - real time + - `/pipeline_handle_events` +- Experimental + - When we started the sync engine, the vision is to become an ubiquitous utility to help user “get your data where you need it, in real time, with a consistent schema” + - https://docs.google.com/document/d/1S4ELi0jZfCWupoi1m8iS49XmecPHgLOapi0xojtRDeM/edit?tab=t.0#heading=h.ihr1ujskb3dc +- source-metronome | destination-redis + - single seconds data sync latency + - single ms query latency without relying on the internet +- source-postgres | destination-stripe + - Sync both standard AND custom objects into Stripe + - standard + - customer + - products + - custom + - locations + - devices + diff --git a/docs/slides/knowledge-transfer.md b/docs/slides/knowledge-transfer.md new file mode 100644 index 000000000..93f723de8 --- /dev/null +++ b/docs/slides/knowledge-transfer.md @@ -0,0 +1,504 @@ +--- +theme: default +title: Sync Engine Knowledge Transfer +transition: slide-left +mdc: true +--- + +## Sync Engine + +- a message-driven runtime for moving data between systems +- transport agnostic +- supports both pull (iterable) and push (events) + +--- + +## many deployment targets + +- as a library + - embed the runtime inside another product or integration surface + - useful when the caller already owns scheduling, persistence, and API boundaries + - examples mentioned in the outline: Replit, Supabase + - usage example + ```ts + import { createEngine } from '@stripe/sync-engine' + + const engine = await createEngine(resolver) + const eof = await engine.pipeline_sync_batch(pipeline, { run_id: 'run_demo' }) + ``` +- as cli + - easiest surface for direct users and contributors + - good for local debugging, one-off syncs, and connector development + - usage example + ```sh + npx @stripe/sync-engine \ + --stripe.api_key sk_live_xxx \ + --postgres.url postgresql://... + ``` +- docker + - package the runtime for Kubernetes and internal Stripe deployment targets + - makes the runtime environment consistent across machines and services + - usage example + ```sh + docker run --rm -p 4010:4010 stripe/sync-engine + ``` + - what that means + - the engine image entrypoint is `node --use-env-proxy dist/bin/serve.js` + - running the container starts the stateless HTTP API server by default + - the service image is a different entrypoint (`dist/bin/sync-service.js`) + +--- + +## Sync Message Protocol + +
+ +- `record` — carries one data record for one stream + ```json + {"type":"record","record":{"stream":"customers","data":{"id":"cus_123","email":"a@b.com"},"emitted_at":"2026-01-01T00:00:00.000Z"}} + ``` +- `source_state` — checkpoints source progress so the sync can resume safely + ```json + {"type":"source_state","source_state":{"state_type":"stream","stream":"customers","data":{"cursor":"cus_123"}}} + ``` +- `catalog` — advertises available streams and their schemas + ```json + {"type":"catalog","catalog":{"streams":[{"name":"customers","primary_key":[["id"]],"newer_than_field":"_updated_at"}]}} + ``` +- `log` — emits diagnostics without polluting the data stream + ```json + {"type":"log","log":{"level":"info","message":"Fetched page 1","data":{"stream":"customers"}}} + ``` +- `spec` — returns connector configuration and state/input schemas + ```json + {"type":"spec","spec":{"config":{"type":"object","properties":{"api_key":{"type":"string"}}},"source_state_stream":{"type":"object"},"source_input":{"type":"object"}}} + ``` +- `connection_status` — reports whether a connector check succeeded or failed + ```json + {"type":"connection_status","connection_status":{"status":"failed","message":"invalid api key"}} + ``` +- `stream_status` — reports per-stream lifecycle and liveness + ```json + {"type":"stream_status","stream_status":{"stream":"customers","status":"complete"}} + ``` +- `control` — lets a connector ask the orchestrator to replace config + ```json + {"type":"control","control":{"control_type":"destination_config","destination_config":{"spreadsheet_title":"Stripe export","spreadsheet_id":"sheet_123","access_token":"ya29.a0AfH6SMAExampleAccessToken","refresh_token":"1//0gExampleRefreshToken"}}} + ``` + ```json + {"type":"control","control":{"control_type":"source_config","source_config":{"api_version":"2025-04-30.basil","webhook_url":"https://example.com/webhooks/pipe_123","webhook_secret":"whsec_123"}}} + ``` +- `progress` — engine-emitted full progress snapshot for the current run + ```json + {"type":"progress","progress":{"started_at":"2026-01-01T00:00:00.000Z","elapsed_ms":1200,"global_state_count":2,"derived":{"status":"started","records_per_second":80,"states_per_second":2,"total_record_count":96,"total_state_count":2},"streams":{"customers":{"status":"started","state_count":2,"record_count":96}}}} + ``` + +
+ +--- + +## 3 components and their roles + +
+
+

Source

+

+ Reads from an upstream system and emits protocol messages. +

+
+
+

Engine

+

+ Composes source and destination into a sync run and handles routing, + limits, and orchestration surfaces. +

+
+
+

Destination

+

+ Consumes messages and writes durably into a downstream system. +

+
+
+ +--- + +## source interface + +```ts +export interface Source { + spec(): AsyncIterable + check( + params: { config: TConfig } + ): AsyncIterable + discover( + params: { config: TConfig } + ): AsyncIterable + read( + params: { + config: TConfig + catalog: ConfiguredCatalog + state?: { streams: Record; global: Record } + }, + $stdin?: AsyncIterable + ): AsyncIterable + setup?(params: { + config: TConfig + catalog: ConfiguredCatalog + }): AsyncIterable + teardown?(params: { + config: TConfig + }): AsyncIterable +} +``` + +--- + +## destination interface + +```ts +export interface Destination { + spec(): AsyncIterable + check( + params: { config: TConfig } + ): AsyncIterable + write( + params: { + config: TConfig + catalog: ConfiguredCatalog + }, + $stdin: AsyncIterable + ): AsyncIterable + setup?(params: { + config: TConfig + catalog: ConfiguredCatalog + }): AsyncIterable + teardown?(params: { + config: TConfig + }): AsyncIterable +} +``` + +--- + +## engine interface + +```ts +export interface Engine { + source_discover(source: PipelineConfig['source']): AsyncIterable + pipeline_setup( + pipeline: PipelineConfig, + opts?: { only?: 'source' | 'destination' } + ): AsyncIterable + pipeline_teardown( + pipeline: PipelineConfig, + opts?: { only?: 'source' | 'destination' } + ): AsyncIterable + pipeline_sync_batch( + pipeline: PipelineConfig, + opts?: BatchSyncOptions + ): Promise +} +``` + +Example output: `source_discover` + +```json +{"type":"catalog","catalog":{"streams":[{"name":"customer","primary_key":[["id"]],"newer_than_field":"_updated_at"}]}} +``` + +Example output: `pipeline_setup` + +```json +{"type":"log","log":{"level":"info","message":"Starting pipeline setup","data":{"source_type":"stripe","destination_type":"postgres","run_source":true,"run_destination":true}}} +{"type":"control","control":{"control_type":"source_config","source_config":{"api_key":"sk_test_...","api_version":"2025-04-30.basil","account_id":"acct_test_123","account_created":1700000000}}} +``` + +Example output: `pipeline_teardown` + +```json +{"type":"log","log":{"level":"info","message":"Tearing down destination resources","data":{"destination_type":"postgres"}}} +``` + +Example output: `pipeline_sync_batch` + +```json +{ + "status": "succeeded", + "has_more": true, + "ending_state": { + "source": { "streams": { "customer": { "cursor": "2" } }, "global": {} }, + "destination": {}, + "sync_run": { "run_id": "run_batch" } + }, + "run_progress": { "derived": { "status": "started" }, "streams": {} }, + "request_progress": { "derived": { "status": "started" }, "streams": {} } +} +``` + +--- + +## source_state re-emit sequence + +```mermaid +sequenceDiagram + participant C as Client + participant E as Engine + participant S as source-stripe + participant D as destination-postgres + participant DB as Durable store + + C->>E: pipeline_sync(...) + E->>S: read(config, catalog, state) + S-->>E: record {stream: "customers", data: {...}} + E->>D: record {stream: "customers", data: {...}} + S-->>E: source_state {stream: "customers", data: {cursor: "cus_123"}} + E->>D: source_state {stream: "customers", data: {cursor: "cus_123"}} + D->>DB: persist all preceding records + DB-->>D: commit complete + D-->>E: source_state {stream: "customers", data: {cursor: "cus_123"}} + E-->>C: source_state {stream: "customers", data: {cursor: "cus_123"}} +``` + +`source_state` is a commit fence: destination re-emits the same payload only after preceding writes are durable. + +--- + +## transport agnostic + +- in-memory + - direct async iterable composition + - simplest path for tests and library usage +- stdin / out + - NDJSON over stdout/stdin for subprocess execution + - keeps connectors language-agnostic and process-isolated +- http + - remote execution surface for engine and service APIs + - useful when orchestration and execution live in separate processes + +--- + +## dummy source & destination, simply using the file system + +- let's try it out. echo source | cat destination + ```bash + ./demo/dummy-source.sh | cat + ``` +- now add the engine in there, and we get to keep track of the progress of sync + - the engine adds validation, routing, catalog enforcement, and checkpoint handling + - this is where “pipe some data” becomes “run a sync safely” + +--- + +## source-stripe + +- `discover` + - OpenAPI-driven and cached by API version + - stamps account-specific catalog details such as `_account_id` enum constraints +- `setup` + - resolves account metadata + - creates or reuses managed webhook configuration +- `read` + - supports both backfill and event-driven flows + - emits records, checkpoints, logs, and stream status events +- `rate limits` + - every list API page fetch goes through `withRateLimit(listFn, rateLimiter)` + - default token bucket is `config.rate_limit ?? (liveMode ? 50 : 10)` requests/sec + - HTTP retries are separate: `429`, `5xx`, and retryable network errors back off and honor `Retry-After` + +--- +layout: two-cols +--- + +## source-stripe: binary subdivision + +
+ +- only used when the Stripe endpoint supports created-time filtering +- oldest record on the fetched page becomes the boundary +- boundary slice keeps the cursor +- older remainder is split into equal time slices with `cursor = null` +- `streamingSubdivide` runs child ranges as a concurrent work queue +- if a child still has more data, split again +- if created filters are unsupported, fall back to sequential pagination +- density intuition: + ```text + [Jan ------------------------------- Apr) + sparse sparse dense dense dense + ^ + boundary + ``` + +
+ +::right:: + +
+ +```mermaid +flowchart TD + A["Initial time range\n[Jan 1 .. Apr 1)"] --> B["Fetch one page"] + RL["Rate limiter\nwait before every fetch"] -.-> B + B --> C{"Cursor returned?"} + C -->|No| D["Done"] + C -->|Yes| E["Oldest record on page\nbecomes boundary"] + E --> F["Keep boundary slice\nwith cursor"] + E --> G["Split older remainder\ninto equal time slices"] + F --> H["Boundary range\ncursor kept"] + G --> I["Older range A\ncursor = null"] + G --> J["Older range B\ncursor = null"] + H --> K["streamingSubdivide\nwork queue"] + I --> K + J --> K + K --> L["Fetch children in parallel"] + L --> M{"Child returns cursor?"} + M -->|Yes| G + M -->|No| D +``` + +
+ +--- + +## destination-postgres + +- `setup` + - projects schema from the discovered catalog + - does not need hardcoded table definitions +- `write` + - flows through upsert and delete paths + - enforces staleness gating and enum constraints at the database boundary +- important point + - behavior stays generic + - it relies on the protocol contract, the catalog, and source-stamped fields such as `_updated_at` + +--- + +## role of the engine + +- concept of a single sync run + - one bounded attempt to move state forward + - includes params, current checkpoint state, and a resulting output stream + - may stop at a safe continuation boundary instead of exhausting the source +- progress tracking + - progress comes from state, stream status, logs, and terminal output + - orchestration can only make good decisions if liveness is explicit + +--- + +## source-stripe | destination-postgres + +- main production-shaped path in the repo +- Stripe source discovers streams and emits source-owned state +- Postgres destination projects schema and applies durable writes +- engine and service glue the two into resumable sync behavior + +--- + +## Putting it all together: backfill + +```ts +pipelineBackfill(pipelineId, { syncState }) { + while (true) { + result = backfillStep({ pipelineSync }, pipelineId, { + syncState, + }) + + syncState = result.syncState + + if (!result.eof.has_more) { + return result.eof + } + } +} +``` + +--- + +## Putting it all together: webhook real time + +```ts +POST /webhooks/{pipeline_id} { + verifyStripeSignature(body, headers) + pipeline = pipelineStore.get(pipeline_id) + engine = createRemoteEngine(...) or createEngine(...) + + output = POST /pipeline_handle_events { + pipeline: { + source: pipeline.source, + destination: pipeline.destination, + streams: pipeline.streams, + }, + stdin: [verifiedEvent] + } + + return ndjsonResponse(output) +} +``` + +--- + +## developer workflow + +- forward proxy + - `source scripts/mitmweb-forward-proxy.sh` + - starts proxy on `127.0.0.1:9080`, UI on `127.0.0.1:9081` + - exports `HTTP_PROXY` / `HTTPS_PROXY` + - adds `NODE_OPTIONS=--use-env-proxy` because Node fetch otherwise bypasses the proxy +- reverse proxy + - `scripts/mitmweb-reverse-proxy.sh http://localhost:3000` + - starts proxy on `127.0.0.1:9090`, UI on `127.0.0.1:9091` + - forwards traffic to a local engine listening on `:3000` +- `--engine-mitm` + - `sync-service --engine-mitm` starts a local engine on `:3000` + - the service then points its engine traffic at `http://127.0.0.1:9090` + - this lets you inspect service → engine requests in the reverse proxy UI + +```mermaid +flowchart LR + Shell["dev shell"] -->|"HTTP_PROXY / HTTPS_PROXY\nNODE_OPTIONS=--use-env-proxy"| Fwd["mitmweb forward\n:9080\nUI :9081"] + Fwd --> Ext["external HTTP APIs"] + + Svc["sync-service --engine-mitm"] -->|"engine requests"| Rev["mitmweb reverse\n:9090\nUI :9091"] + Rev -->|"reverse to localhost:3000"| Eng["local engine\nserve.js\n:3000"] +``` + +--- + +## Experimental + +- When we started the sync engine, the vision is to become an ubiquitous utility to help user “get your data where you need it, in real time, with a consistent schema” + - long-term direction is broader than one Stripe-to-Postgres path + - the bet is that one protocol can support many connector pairs +- https://docs.google.com/document/d/1S4ELi0jZfCWupoi1m8iS49XmecPHgLOapi0xojtRDeM/edit?tab=t.0#heading=h.ihr1ujskb3dc + - broader product and vision context + - useful if you want the “why now / where next” story + +--- + +## source-metronome | destination-redis + +- single seconds data sync latency + - goal is near-real-time movement into a fast serving layer + - pressures orchestration overhead, batching, and transport choices +- single ms query latency without relying on the internet + - downstream shape can optimize for local low-latency reads + - shows the architecture is not limited to warehouse-style sinks + +--- + +## source-postgres | destination-stripe + +- Sync both standard AND custom objects into Stripe + - shows the architecture can also support reverse sync paths + - destination semantics matter as much as source semantics here +- standard + - customer + - canonical standard object example + - products + - another standard object with a different lifecycle and shape +- custom + - locations + - example tenant-specific entity + - devices + - another custom entity showing the pattern is reusable diff --git a/docs/slides/package.json b/docs/slides/package.json index 3933d61bb..04d322c99 100644 --- a/docs/slides/package.json +++ b/docs/slides/package.json @@ -3,7 +3,8 @@ "private": true, "scripts": { "demo": "docker build --build-arg SLIDES=demo.md -t slides-demo . && exec docker run --rm -p 3030:3030 slides-demo", - "arch": "docker build --build-arg SLIDES=architecture.md -t slides-arch . && exec docker run --rm -p 3030:3030 slides-arch" + "arch": "docker build --build-arg SLIDES=architecture.md -t slides-arch . && exec docker run --rm -p 3030:3030 slides-arch", + "kt": "docker build --build-arg SLIDES=knowledge-transfer.md -t slides-kt . && exec docker run --rm -p 3030:3030 slides-kt" }, "dependencies": { "@slidev/cli": "latest", diff --git a/docs/slides/style.css b/docs/slides/style.css index 36d502119..02afadb51 100644 --- a/docs/slides/style.css +++ b/docs/slides/style.css @@ -1,7 +1,38 @@ /* Constrain all mermaid diagrams to fit within slide bounds */ .slidev-layout .mermaid svg { - max-height: 320px !important; + max-height: 240px !important; max-width: 100% !important; height: auto !important; width: auto !important; } + +/* Some diagrams need a hard container cap because Mermaid can ignore svg sizing. */ +.slidev-layout .binary-subdivision-diagram { + display: flex; + justify-content: center; + align-items: flex-start; +} + +.slidev-layout .binary-subdivision-diagram .mermaid { + width: 100%; +} + +.slidev-layout .binary-subdivision-diagram .mermaid, +.slidev-layout .binary-subdivision-diagram .mermaid svg { + width: 100% !important; + height: auto !important; + max-height: 360px !important; + max-width: 100% !important; +} + +.slidev-layout .binary-subdivision-copy { + font-size: 0.92rem; + line-height: 1.45; +} + +/* Allow every slide to scroll vertically when content overflows. */ +.slidev-layout { + overflow-y: auto !important; + overflow-x: hidden !important; + scrollbar-gutter: stable; +} diff --git a/packages/protocol/src/protocol.ts b/packages/protocol/src/protocol.ts index 19e4314fb..a7581941e 100644 --- a/packages/protocol/src/protocol.ts +++ b/packages/protocol/src/protocol.ts @@ -433,6 +433,7 @@ export const SyncState = z .meta({ id: 'SyncState' }) export type SyncState = z.infer +/** @deprecated Legacy terminal payload. Prefer explicit request/response results via pipeline_sync_batch. */ export const EofPayload = z .object({ status: RunStatus.describe('Terminal run status derived from stream outcomes.'), @@ -451,8 +452,11 @@ export const EofPayload = z ), request_progress: ProgressPayload.describe('Progress for this specific request only.'), }) - .describe('Terminal message signaling end of this request.') + .describe( + 'Deprecated terminal message signaling end of this request. Prefer explicit request/response results via pipeline_sync_batch.' + ) .meta({ id: 'EofPayload' }) +/** @deprecated Legacy terminal payload. Prefer explicit request/response results via pipeline_sync_batch. */ export type EofPayload = z.infer // MARK: - Envelope messages (the wire format) @@ -532,10 +536,12 @@ export const ControlMessage = MessageBase.extend({ }).meta({ id: 'ControlMessage' }) export type ControlMessage = z.infer +/** @deprecated Legacy terminal message. Prefer explicit request/response results via pipeline_sync_batch. */ export const EofMessage = MessageBase.extend({ type: z.literal('eof'), eof: EofPayload, }).meta({ id: 'EofMessage' }) +/** @deprecated Legacy terminal message. Prefer explicit request/response results via pipeline_sync_batch. */ export type EofMessage = z.infer // MARK: - Pipeline params @@ -578,10 +584,12 @@ export type PipelineConfig = z.infer /** * Extended message types (engine-level, not emitted by connectors directly). */ +/** @deprecated Legacy push-input envelope. Prefer explicit event handling without a message wrapper. */ export const SourceInputMessage = MessageBase.extend({ type: z.literal('source_input'), source_input: z.unknown(), }).meta({ id: 'SourceInputMessage' }) +/** @deprecated Legacy push-input envelope. Prefer explicit event handling without a message wrapper. */ export type SourceInputMessage = z.infer /** From 80393e0aa89399c194570616fa3646b21eddd163 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 6 May 2026 21:21:35 -0700 Subject: [PATCH 07/11] ci: run CI on dev branch pushes Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b48e90069..9ab1a3615 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: CI on: pull_request: push: - branches: [main, v2] + branches: [main, dev, v2] workflow_dispatch: permissions: From 6ffdad4b7f67ca516117c3838445e72b3525bea2 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 6 May 2026 21:27:31 -0700 Subject: [PATCH 08/11] docs: add minimal README with v0 branch pointer Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..335cebaa5 --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +# Stripe Sync Engine + +> [!WARNING] +> The `dev` branch is experimental and under active development. + +For the original Supabase Stripe Sync Engine, see the [`v0` branch](https://github.com/stripe/sync-engine-fork/tree/v0). From 91d5f9b76c54b30af7911192aa9002cd0490fd58 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 6 May 2026 21:32:04 -0700 Subject: [PATCH 09/11] Update readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 335cebaa5..5346a6250 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Stripe Sync Engine > [!WARNING] -> The `dev` branch is experimental and under active development. +> The `dev` branch is experimental and under active development and not yet documented. + +For the original Supabase Stripe Sync Engine, see the [`og` branch](https://github.com/stripe/sync-engine/tree/og). -For the original Supabase Stripe Sync Engine, see the [`v0` branch](https://github.com/stripe/sync-engine-fork/tree/v0). From ff89368189c88ec69bfd85cecb70c85733234283 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 6 May 2026 21:38:10 -0700 Subject: [PATCH 10/11] ci: only tag Docker image as "latest" on main branch Co-Authored-By: Claude Opus 4.6 (1M context) Committed-By-Agent: claude --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ab1a3615..8a5f6c26b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -890,4 +890,4 @@ jobs: run: bash scripts/promote-to-dockerhub.sh env: GHCR_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} - DOCKERHUB_TAGS: ${{ steps.tag.outputs.name }} latest + DOCKERHUB_TAGS: ${{ steps.tag.outputs.name }}${{ github.ref_name == 'main' && ' latest' || '' }} From bc60431ddf63265fc83e9fae3fb900db5330878a Mon Sep 17 00:00:00 2001 From: Henry Liou Date: Thu, 21 May 2026 14:26:39 -0700 Subject: [PATCH 11/11] Add climate to SKIPPABLE_ERROR_MESSAGES --- packages/source-stripe/src/src-list-api.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/source-stripe/src/src-list-api.ts b/packages/source-stripe/src/src-list-api.ts index c14312775..6f8527b7a 100644 --- a/packages/source-stripe/src/src-list-api.ts +++ b/packages/source-stripe/src/src-list-api.ts @@ -157,6 +157,11 @@ const SKIPPABLE_ERROR_MESSAGES = [ // https://dashboard.stripe.com/identity to get started. // [GET /v1/identity/verification_reports (400)]" 'Your account is not set up to use Identity', + + // climate_order, climate_product + // This account is not eligible for Climate Orders. [GET /v1/climate/orders (400)] + // This account is not eligible for Climate Orders. [GET /v1/climate/products (400)] + 'This account is not eligible for Climate Orders', ] export function isSkippableError(err: unknown): boolean {