From 54161bda430f82387b2b0b99fd5f1f651e2af8af Mon Sep 17 00:00:00 2001 From: Yogesh Rao Date: Thu, 14 May 2026 09:33:21 +0530 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20improve=20generate-smoke-test=20ski?= =?UTF-8?q?ll=20score=20(85%=20=E2=86=92=2092%)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hey @gjtorikian 👋 I ran your skills through `tessl skill review` at work and found some targeted improvements for `generate-smoke-test`. Here's the full before/after: | Skill | Before | After | Change | |-------|--------|-------|--------| | generate-smoke-test | 85% | 92% | +7% | | review-operations | 100% | 100% | — | | generate-emitter | 93% | 93% | — | | check-emitter-parity | 90% | 90% | — | | generate-sdk | 90% | 90% | — | | verify-compat | 90% | 90% | — | | generate-extractor | 90% | 90% | — | | verify-smoke-test | 90% | 90% | — | | integrate | 85% | 85% | — |
Changes made to generate-smoke-test - **Consolidated operations map references** — the `.oagen-manifest.json` explanation was repeated across Steps 2, 3, 5, and the Emitter-Fixing Loop. Kept the authoritative explanation in Step 2 and replaced the others with concise back-references. - **Replaced interception prose with a table** — Step 1 listed language-specific interception libraries in prose format. A 5-row table conveys the same information more efficiently. - **Extracted implementation patterns to a reference file** — moved the concrete `buildArgs()` template and full smoke script skeleton to `references/implementation-patterns.md`, improving progressive disclosure while keeping the main SKILL.md focused on workflow. - **Trimmed redundant explanatory text** — removed a self-describing sentence in the overview that restated what the structure already communicates.
I also stress-tested your `generate-emitter` skill against a few real-world task evals and it held up really well on scaffolding a Go emitter with full `SdkBehavior` integration and operations map support. Kudos for that. Honest disclosure — I work at @tesslio where we build tooling around skills like these. Not a pitch — just saw room for improvement and wanted to contribute. Want to self-improve your skills? Just point your agent (Claude Code, Codex, etc.) at [this Tessl guide](https://docs.tessl.io/evaluate/optimize-a-skill-using-best-practices) and ask it to optimize your skill. Ping me — [@yogesh-tessl](https://github.com/yogesh-tessl) — if you hit any snags. Thanks in advance 🙏 --- skills/generate-smoke-test/SKILL.md | 60 +++++----------- .../references/implementation-patterns.md | 72 +++++++++++++++++++ 2 files changed, 91 insertions(+), 41 deletions(-) create mode 100644 skills/generate-smoke-test/references/implementation-patterns.md diff --git a/skills/generate-smoke-test/SKILL.md b/skills/generate-smoke-test/SKILL.md index f439f50..11f3d0b 100644 --- a/skills/generate-smoke-test/SKILL.md +++ b/skills/generate-smoke-test/SKILL.md @@ -11,7 +11,7 @@ Create a self-contained smoke test script for a new SDK language that captures w Each language's smoke test is a single file: `smoke/sdk-{lang}.ts` **in the emitter project**. It uses the target language's native HTTP interception to capture what the SDK actually sends over the wire, then outputs `SmokeResults` JSON. The diff tool compares this against a baseline and reports mismatches by severity. -The script is self-contained — no proxy, no subprocess protocol, no separate driver. It imports shared infrastructure from `@workos/oagen/smoke` and implements language-specific parts inline. +It imports shared infrastructure from `@workos/oagen/smoke` and implements language-specific parts inline. ## Resolve Paths @@ -50,15 +50,15 @@ Store it as `spec`. ## Step 1: Determine HTTP Interception Strategy -The interception must capture the raw request (method, path, query params, body) and raw response (status, body). Choose based on the target language's SDK: +Choose the interception mechanism for the target language. It must capture the raw request (method, path, query, body) and response (status, body), storing both in a `currentCapture` variable (~20-30 lines): -- **Node:** Patch `globalThis.fetch` -- **Ruby:** WebMock `stub_request` or monkey-patch `Net::HTTP` -- **Python:** `responses`, `respx` (for httpx), or `unittest.mock.patch` -- **Go:** Custom `http.RoundTripper` -- **Java/Kotlin:** OkHttp `Interceptor` - -The interception code is typically ~20-30 lines. It must capture the request as-sent, let the real HTTP call proceed, capture the response, and store both in a `currentCapture` variable. +| Language | Mechanism | +|----------|-----------| +| Node | Patch `globalThis.fetch` | +| Ruby | WebMock `stub_request` or monkey-patch `Net::HTTP` | +| Python | `responses`, `respx` (httpx), or `unittest.mock.patch` | +| Go | Custom `http.RoundTripper` | +| Java/Kotlin | OkHttp `Interceptor` | ## Step 2: Build the SERVICE_MAP @@ -91,48 +91,27 @@ Each language's SDK will have different accessor names — discover them by read ## Step 3: Implement SDK Method Resolution -Adapt the 4-tier resolution to the target language's naming conventions: +Adapt the 4-tier resolution to the target language's naming conventions (Ruby/Python: `snake_case`, Go: `PascalCase`, Node: `camelCase`): -0. **Manifest match** — Load the `operations` map from `.oagen-manifest.json` in the SDK output directory (emitter-generated, not hand-maintained). This is the **primary** resolution path for generated SDKs. The manifest maps every `HTTP_METHOD /path` to `{ sdkMethod, service }` and is produced by the emitter's `buildOperationsMap` hook. If the operations map is missing, warn and fall through to heuristic tiers. +0. **Manifest match** — Primary path. Uses the operations map loaded in Step 2. Fall through if unavailable. 1. **Exact match** — IR operation name converted to target convention 2. **CRUD prefix match** — standard verbs (create, list, retrieve/get, update, delete) with service name tiebreaker 3. **Keyword fuzzy match** — stem words and score overlap -Key convention differences: Ruby/Python use `snake_case`, Go uses `PascalCase`, Node uses `camelCase`. - Each resolution records provenance metadata (`ExchangeProvenance`) so findings can be traced back to the resolution path. ## Step 4: Implement Argument Construction -Build SDK call arguments from IR operations (reference `buildArgs()` in existing smoke scripts): - -- No path params + has body → `method(payload)` -- No path params + has query params → `method(queryOpts)` -- Single path param, no body/query → `method(id)` (positional) -- Complex (path params + body/query) → `method(mergedOptions)` -- Idempotent POST → append empty options object for idempotency key - -Choose the right payload convention: Node uses `generateCamelPayload()`, Ruby/Python may use `generatePayload()` directly (snake_case). +Build SDK call arguments from IR operations. Reference `buildArgs()` in existing smoke scripts and adapt to the target language's calling convention. See [references/implementation-patterns.md](references/implementation-patterns.md) for the concrete branching template covering all argument patterns (positional, payload-only, query-only, complex, idempotent POST). ## Step 5: Write `smoke/sdk-{lang}.ts` -Create the script **in the emitter project**: - -1. Imports from `@workos/oagen/smoke` -2. HTTP interception setup -3. `main()` function: - - Parse CLI args, validate API key - - Parse spec via `parseSpec()` - - Load operations map from `{sdk-path}/.oagen-manifest.json` (emitter-generated). If missing, log a warning — method resolution will rely on heuristic tiers and most operations will likely be skipped. - - Load and configure the SDK - - Iterate `planOperations()` groups - - For each operation: resolve SDK method, resolve path params, build args, call SDK, capture exchange - - Extract IDs via `ids.extractAndStore()` - - Track POST creates for cleanup - - Cleanup created entities in reverse - - Restore original HTTP behavior - - Write `smoke-results-sdk-{lang}.json` -4. Summary output (successes, errors, skipped, unexpected statuses) +Create the script **in the emitter project**. See [references/implementation-patterns.md](references/implementation-patterns.md) for the full structural template. The script follows this flow: + +1. Import shared infrastructure from `@workos/oagen/smoke` +2. Set up HTTP interception (Step 1) +3. `main()`: parse spec → load operations map → init SDK → iterate `planOperations()` groups → resolve method (Step 3) → build args (Step 4) → call SDK → capture exchange → extract IDs via `ids.extractAndStore()` → track POST creates for cleanup +4. Cleanup created entities in reverse order, restore HTTP, write `smoke-results-sdk-{lang}.json` ## Step 6: Register the Smoke Runner @@ -167,11 +146,10 @@ During initial setup, run `oagen generate` then the smoke test until skips are m ```bash oagen generate --lang {lang} --output {sdk-path} --spec {spec} --namespace {ns} -# The emitter writes the operations map into .oagen-manifest.json — the smoke test loads it automatically. oagen verify --lang {lang} --output {sdk-path} --spec {spec} ``` -If many operations are skipped with "No matching SDK method", check that the emitter's `buildOperationsMap` is implemented and that `.oagen-manifest.json` contains an `operations` field. The operations map is the primary mechanism the smoke test uses to find SDK methods. +If many operations are skipped with "No matching SDK method", verify the operations map is present (see Step 2). | Exit | Meaning | Output | Action | | ---- | ------------- | --------------------------- | --------------------------------------- | diff --git a/skills/generate-smoke-test/references/implementation-patterns.md b/skills/generate-smoke-test/references/implementation-patterns.md new file mode 100644 index 0000000..e5b4155 --- /dev/null +++ b/skills/generate-smoke-test/references/implementation-patterns.md @@ -0,0 +1,72 @@ +# Smoke Test Implementation Patterns + +Concrete templates for the language-specific parts of a smoke test script. Adapt naming conventions and HTTP interception to the target language. + +## Argument Construction (`buildArgs`) + +Build SDK call arguments from IR operations. Choose payload generator based on target language: Node uses `generateCamelPayload()`, Ruby/Python use `generatePayload()` (snake_case). + +```typescript +function buildArgs(op: OperationPlan, spec: ApiSpec): unknown[] { + const pathParams = op.parameters.filter(p => p.in === "path"); + const hasBody = !!op.requestBody; + const hasQuery = op.parameters.some(p => p.in === "query"); + + if (pathParams.length === 0 && hasBody) + return [generatePayload(op, spec)]; + if (pathParams.length === 0 && hasQuery) + return [generateQueryParams(op)]; + if (pathParams.length === 1 && !hasBody && !hasQuery) + return [ids.get(pathParams[0].schema) ?? "test_id"]; + // Complex: merge path params + body/query into options object + const opts = { ...generatePayload(op, spec), ...resolvePathParams(op) }; + if (hasQuery) Object.assign(opts, generateQueryParams(op)); + // Idempotent POST: append empty options for idempotency key + if (op.method === "post" && op.parameters.some(p => p.name === "idempotency_key")) + return [opts, {}]; + return [opts]; +} +``` + +## Smoke Script Structural Template + +The complete structure for `smoke/sdk-{lang}.ts`: + +```typescript +import { parseSpec, planOperations, generatePayload, generateQueryParams, + IdRegistry, isUnexpectedStatus, resolvePath, + type CapturedExchange, type SmokeResults } from "@workos/oagen/smoke"; + +let currentCapture: CapturedExchange | null = null; +const ids = new IdRegistry(); + +// Interception setup — see language table in SKILL.md Step 1 + +async function main() { + const spec = await parseSpec(specPath); + const opsMap = loadManifestOperations(sdkPath); // from Step 2 + const client = initSdk(apiKey); + const results: SmokeResults = { exchanges: [], errors: [], skipped: [] }; + const cleanups: Array<() => Promise> = []; + + for (const group of planOperations(spec)) { + for (const op of group.operations) { + const resolved = resolveMethod(op, opsMap, client); // Step 3 + if (!resolved) { results.skipped.push(op); continue; } + const args = buildArgs(op, spec); // Step 4 + try { + currentCapture = null; + await resolved.fn(...args); + if (currentCapture) { + ids.extractAndStore(currentCapture.response); + results.exchanges.push(currentCapture); + if (op.method === "post") cleanups.push(() => deleteEntity(client, op)); + } + } catch (e) { results.errors.push({ op, error: e }); } + } + } + for (const cleanup of cleanups.reverse()) await cleanup(); + writeFileSync(`smoke-results-sdk-{lang}.json`, JSON.stringify(results)); + printSummary(results); // successes, errors, skipped, unexpected statuses +} +``` From 5584762a8735b026f218c939f54dd075458ec6d1 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 21 May 2026 13:47:59 -0400 Subject: [PATCH 2/2] fix: format --- skills/generate-smoke-test/SKILL.md | 14 +++---- .../references/implementation-patterns.md | 41 +++++++++++++------ 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/skills/generate-smoke-test/SKILL.md b/skills/generate-smoke-test/SKILL.md index 11f3d0b..458901e 100644 --- a/skills/generate-smoke-test/SKILL.md +++ b/skills/generate-smoke-test/SKILL.md @@ -52,13 +52,13 @@ Store it as `spec`. Choose the interception mechanism for the target language. It must capture the raw request (method, path, query, body) and response (status, body), storing both in a `currentCapture` variable (~20-30 lines): -| Language | Mechanism | -|----------|-----------| -| Node | Patch `globalThis.fetch` | -| Ruby | WebMock `stub_request` or monkey-patch `Net::HTTP` | -| Python | `responses`, `respx` (httpx), or `unittest.mock.patch` | -| Go | Custom `http.RoundTripper` | -| Java/Kotlin | OkHttp `Interceptor` | +| Language | Mechanism | +| ----------- | ------------------------------------------------------ | +| Node | Patch `globalThis.fetch` | +| Ruby | WebMock `stub_request` or monkey-patch `Net::HTTP` | +| Python | `responses`, `respx` (httpx), or `unittest.mock.patch` | +| Go | Custom `http.RoundTripper` | +| Java/Kotlin | OkHttp `Interceptor` | ## Step 2: Build the SERVICE_MAP diff --git a/skills/generate-smoke-test/references/implementation-patterns.md b/skills/generate-smoke-test/references/implementation-patterns.md index e5b4155..d7555c9 100644 --- a/skills/generate-smoke-test/references/implementation-patterns.md +++ b/skills/generate-smoke-test/references/implementation-patterns.md @@ -8,21 +8,22 @@ Build SDK call arguments from IR operations. Choose payload generator based on t ```typescript function buildArgs(op: OperationPlan, spec: ApiSpec): unknown[] { - const pathParams = op.parameters.filter(p => p.in === "path"); + const pathParams = op.parameters.filter((p) => p.in === "path"); const hasBody = !!op.requestBody; - const hasQuery = op.parameters.some(p => p.in === "query"); + const hasQuery = op.parameters.some((p) => p.in === "query"); - if (pathParams.length === 0 && hasBody) - return [generatePayload(op, spec)]; - if (pathParams.length === 0 && hasQuery) - return [generateQueryParams(op)]; + if (pathParams.length === 0 && hasBody) return [generatePayload(op, spec)]; + if (pathParams.length === 0 && hasQuery) return [generateQueryParams(op)]; if (pathParams.length === 1 && !hasBody && !hasQuery) return [ids.get(pathParams[0].schema) ?? "test_id"]; // Complex: merge path params + body/query into options object const opts = { ...generatePayload(op, spec), ...resolvePathParams(op) }; if (hasQuery) Object.assign(opts, generateQueryParams(op)); // Idempotent POST: append empty options for idempotency key - if (op.method === "post" && op.parameters.some(p => p.name === "idempotency_key")) + if ( + op.method === "post" && + op.parameters.some((p) => p.name === "idempotency_key") + ) return [opts, {}]; return [opts]; } @@ -33,9 +34,17 @@ function buildArgs(op: OperationPlan, spec: ApiSpec): unknown[] { The complete structure for `smoke/sdk-{lang}.ts`: ```typescript -import { parseSpec, planOperations, generatePayload, generateQueryParams, - IdRegistry, isUnexpectedStatus, resolvePath, - type CapturedExchange, type SmokeResults } from "@workos/oagen/smoke"; +import { + parseSpec, + planOperations, + generatePayload, + generateQueryParams, + IdRegistry, + isUnexpectedStatus, + resolvePath, + type CapturedExchange, + type SmokeResults, +} from "@workos/oagen/smoke"; let currentCapture: CapturedExchange | null = null; const ids = new IdRegistry(); @@ -52,7 +61,10 @@ async function main() { for (const group of planOperations(spec)) { for (const op of group.operations) { const resolved = resolveMethod(op, opsMap, client); // Step 3 - if (!resolved) { results.skipped.push(op); continue; } + if (!resolved) { + results.skipped.push(op); + continue; + } const args = buildArgs(op, spec); // Step 4 try { currentCapture = null; @@ -60,9 +72,12 @@ async function main() { if (currentCapture) { ids.extractAndStore(currentCapture.response); results.exchanges.push(currentCapture); - if (op.method === "post") cleanups.push(() => deleteEntity(client, op)); + if (op.method === "post") + cleanups.push(() => deleteEntity(client, op)); } - } catch (e) { results.errors.push({ op, error: e }); } + } catch (e) { + results.errors.push({ op, error: e }); + } } } for (const cleanup of cleanups.reverse()) await cleanup();