From bcb28b5ab33b65cd54fe41a7c040a0c589789935 Mon Sep 17 00:00:00 2001 From: Jonathan Jackson Date: Fri, 8 May 2026 20:45:29 -0600 Subject: [PATCH 1/2] nova(remove user-scope override): use plugin's native PAT path (0.13.111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop ACE's user-scope MCP override workaround now that the Nova plugin exposes the server-side PAT via `${NOVA_API_KEY}` in its `.mcp.json` headers (voidcraft-labs/nova-plugin#11). The override was fragile — silently fell off across CLI updates with no doctor signal until a Phase 2 upload halted on a missing `upload_app_to_hq` tool (turmeric/20260508-1951; immediate trigger). - bin/ace-setup: deleted ~40-line override block; replaced with one-time idempotent `claude mcp remove nova --scope user` for cleanup - bin/ace-doctor: updated nova_env / nova_auth probe comments + remediation - agents/commcare-setup.md § Subagent inheritance: explain via plugin-side ${NOVA_API_KEY} expansion - playbook/integrations/nova-integration.md: full rewrite — native PAT path, shell-export step (Claude Code reads the var from its process env, not from \$CLAUDE_PLUGIN_DATA/.env), nova-plugin#11 added to Resolved Blockers Operator action on update: export NOVA_API_KEY in shell rc, restart Claude Code. See playbook/integrations/nova-integration.md § Install + auth step 5. Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude-plugin/marketplace.json | 4 +- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 17 +++ VERSION | 2 +- agents/commcare-setup.md | 6 +- bin/ace-doctor | 19 ++-- bin/ace-setup | 46 ++------- package.json | 2 +- playbook/integrations/nova-integration.md | 120 +++++++++++++--------- 9 files changed, 115 insertions(+), 103 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index f5645fdb..1455c62d 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,13 +6,13 @@ "url": "https://github.com/jjackson" }, "metadata": { - "version": "0.13.115" + "version": "0.13.116" }, "plugins": [ { "name": "ace", "source": "./", - "version": "0.13.115", + "version": "0.13.116", "description": "AI Connect Engine — orchestrates the CRISPR-Connect lifecycle from idea through app building, Connect setup, LLO management, and closeout" } ] diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 8543aa2c..d871d7f3 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ace", - "version": "0.13.115", + "version": "0.13.116", "description": "AI Connect Engine — orchestrates the CRISPR-Connect lifecycle from idea through app building, Connect setup, LLO management, and closeout", "author": { "name": "Jonathan Jackson", diff --git a/CHANGELOG.md b/CHANGELOG.md index a1438d19..ff4bba37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to the ACE plugin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the plugin follows [semantic versioning](https://semver.org/spec/v2.0.0.html). +## 0.13.111 — 2026-05-09 + +**Drop the Nova MCP user-scope-override workaround now that the Nova plugin reads `NOVA_API_KEY` natively (voidcraft-labs/nova-plugin#11).** + +The Nova MCP server has supported PAT auth since early May, but the plugin's MCP config never surfaced it — Claude Code defaulted to OAuth, and ACE worked around it by registering a user-scope MCP entry at the same URL with `claude mcp add nova ... --header "Authorization: Bearer $NOVA_API_KEY"`, relying on URL-signature dedup to suppress the plugin entry. The override was fragile: it silently fell off across CLI updates, with no doctor signal until a Phase 2 upload halted mid-run on a missing `upload_app_to_hq` tool (this hit `turmeric/20260508-1951` and is the immediate trigger for this version). + +The right fix is upstream — add `headers.Authorization = "Bearer ${NOVA_API_KEY}"` to the plugin's `.mcp.json`. That issue (voidcraft-labs/nova-plugin#11) is filed; this version assumes it lands and removes the ACE-side workaround: + +- `bin/ace-setup` — deleted the entire ~40-line "Nova MCP user-scope override" block. Replaced with a one-time idempotent `claude mcp remove nova --scope user` so existing installs clean up the now-obsolete entry. +- `bin/ace-doctor` — updated the `nova_env` and `nova_auth` probe comments + remediation strings to reflect the native PAT path. The probes themselves are unchanged (still useful — `nova_env` confirms `NOVA_API_KEY` is in `.env`; `nova_auth` confirms the bearer is accepted by the server). +- `agents/commcare-setup.md` § Subagent inheritance — re-explained the inheritance via plugin-side `${NOVA_API_KEY}` expansion instead of the user-scope override. +- `playbook/integrations/nova-integration.md` — full rewrite. Removed the override architecture; documented the native PAT path including the **shell-export step** required because Claude Code reads `${NOVA_API_KEY}` from its own process env, not from `$CLAUDE_PLUGIN_DATA/.env`. Added nova-plugin#11 to the Resolved Blockers section. + +**Operator action on update:** export `NOVA_API_KEY` in your shell rc (one-time per machine — see playbook/integrations/nova-integration.md § Install + auth step 5), then restart Claude Code. The next session will use the plugin's native PAT path. + +If you're updating from a version that registered the user-scope override, `bin/ace-setup`'s one-time cleanup will remove it on the next `/ace:setup --force-env`. + ## 0.13.66 — 2026-05-06 **Fix #129: swap `googleapis` for per-API subpackages — node_modules drops 332 MB → 141 MB.** diff --git a/VERSION b/VERSION index 2ded95bf..1c3202f2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.13.115 +0.13.116 diff --git a/agents/commcare-setup.md b/agents/commcare-setup.md index a9bf8ce6..e0bf4099 100644 --- a/agents/commcare-setup.md +++ b/agents/commcare-setup.md @@ -97,9 +97,9 @@ Do NOT dispatch the architect until `get_hq_connection` returns Apply the same gate at the start of any later subagent dispatch in this phase that calls Nova tools (e.g. coverage retries) — but the parent's auth state is what matters. Subagents inherit Nova's MCP -connection because the user-scope override registers it once for the -session; every subagent dispatch sees the same `get_hq_connection` -result. +connection because the plugin's MCP entry expands `${NOVA_API_KEY}` +once per session start; every subagent dispatch sees the same +`get_hq_connection` result. ### Step 1: PDD to Apps (sequential) Invoke `pdd-to-learn-app`, then `pdd-to-deliver-app`. diff --git a/bin/ace-doctor b/bin/ace-doctor index 5e9cbd0a..efdf060f 100755 --- a/bin/ace-doctor +++ b/bin/ace-doctor @@ -961,12 +961,13 @@ if [ -n "$ENV_FILE" ]; then # ── Nova (CommCare app builder MCP) ────────────────────────────────── # Presence check only — live HTTP probe runs in [Auth liveness] below. - # Drift detection: if NOVA_API_KEY is missing or unresolved (still - # `op://...`), bin/ace-setup's user-scope MCP override will be skipped - # and /nova:* falls back to OAuth — broken under concurrent worktrees. + # The Nova plugin reads NOVA_API_KEY natively (post nova-plugin#11) and + # expands it into the MCP entry's Authorization: Bearer header. Without + # it the plugin falls back to interactive OAuth — broken under concurrent + # worktrees per voidcraft-labs/nova-plugin#9. NOVA_KEY_VAL="$(get_env NOVA_API_KEY)" if [ -z "$NOVA_KEY_VAL" ]; then - warn "nova_env: NOVA_API_KEY missing or unresolved" "mint at https://commcare.app/settings as the ACE Gmail identity, save to 1Password item \"ACE - Nova\" / field \`api_key\`, then /ace:setup --force-env" + warn "nova_env: NOVA_API_KEY missing or unresolved" "mint at https://commcare.app/settings as the ACE Gmail identity, save to 1Password item \"ACE - Nova\" / field \`api_key\`, then /ace:setup --force-env, then export NOVA_API_KEY in your shell rc so Claude Code's MCP header expansion picks it up" else pass "nova_env: NOVA_API_KEY present" fi @@ -1187,10 +1188,10 @@ else fi # 6. nova — POST initialize JSON-RPC frame to https://mcp.commcare.app/mcp -# with bearer auth. 200 = bearer accepted (the user-scope override -# bin/ace-setup registered is alive). 401 = key invalid/revoked. The -# Accept header includes text/event-stream because Nova returns SSE -# frames for the response body even though the request is plain JSON. +# with bearer auth. 200 = bearer accepted by Nova's PAT auth path. +# 401 = key invalid/revoked. The Accept header includes text/event-stream +# because Nova returns SSE frames for the response body even though the +# request is plain JSON. if [ -n "$NOVA_KEY_VAL" ]; then NOVA_REACH_CODE="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \ -X POST \ @@ -1210,7 +1211,7 @@ if [ -n "$NOVA_KEY_VAL" ]; then warn "nova_auth: cannot reach https://mcp.commcare.app/mcp (network/DNS/timeout)" "check VPN/proxy; net_nova_mcp probe above should also flag this" ;; *) - warn "nova_auth: POST https://mcp.commcare.app/mcp → HTTP $NOVA_REACH_CODE" "investigate Nova MCP health; if persists, /ace:setup --force-env to re-register the user-scope override" + warn "nova_auth: POST https://mcp.commcare.app/mcp → HTTP $NOVA_REACH_CODE" "investigate Nova MCP health; if persists, rotate the key at https://commcare.app/settings and re-inject .env" ;; esac else diff --git a/bin/ace-setup b/bin/ace-setup index c66ed32f..37962990 100755 --- a/bin/ace-setup +++ b/bin/ace-setup @@ -406,46 +406,16 @@ $DIAG" fi fi -# ---- Nova MCP user-scope override (API-key auth) +# ---- Nova MCP one-time cleanup of legacy user-scope override # -# The Nova plugin auto-registers an OAuth-mode MCP entry at -# https://mcp.commcare.app/mcp. Concurrent ACE worktrees on one identity -# trip a refresh-token cascade upstream of disk (better-auth/oauth-provider -# deletes all (userId, clientId) refresh rows when a stale token is -# presented). Adding a user-scope entry at the same URL with a bearer -# header makes Claude Code's URL-signature dedup suppress the plugin entry -# and route every Nova call through API-key auth instead. See -# voidcraft-labs/nova-plugin#9 and `playbook/integrations/nova-integration.md`. -# -# Idempotent: removes any existing user-scope nova entry, re-adds with the -# current key so rotations propagate. Skipped silently when NOVA_API_KEY is -# missing (e.g. operator hasn't minted one yet) — doctor surfaces it. -if [ -f "$ENV_OUT" ]; then - NOVA_KEY="$(grep -E '^NOVA_API_KEY=' "$ENV_OUT" | head -1 | sed -E 's/^NOVA_API_KEY=//' | sed -E 's/^"(.*)"$/\1/')" -else - NOVA_KEY="" -fi -if [ -z "$NOVA_KEY" ] || [ "${NOVA_KEY#op://}" != "$NOVA_KEY" ]; then - warn "nova_mcp: NOVA_API_KEY not resolved in $ENV_OUT" \ - "mint a key at https://commcare.app/settings as the ACE Gmail identity, save to 1Password item \"ACE - Nova\" field \`api_key\`, then re-run /ace:setup --force-env. Until then, /nova:* will fall back to OAuth and break under concurrent worktree use." -elif ! command -v claude >/dev/null 2>&1; then - warn "nova_mcp: claude CLI not on PATH — cannot register user-scope override" \ - "install Claude Code CLI, then re-run /ace:setup" -else - # Remove any prior user-scope entry; ignore failure (entry may not exist). +# Pre-2026-05-09 ACE registered a user-scope MCP entry at +# https://mcp.commcare.app/mcp with a bearer header to route around +# voidcraft-labs/nova-plugin#9. The Nova plugin now reads NOVA_API_KEY +# natively (voidcraft-labs/nova-plugin#11), so the user-scope entry is +# obsolete and could double-register the same URL. Idempotent: tries +# to remove, ignores any "not found" error. +if command -v claude >/dev/null 2>&1; then claude mcp remove nova --scope user >/dev/null 2>&1 || true - ADD_OUT="$(claude mcp add nova "https://mcp.commcare.app/mcp" \ - --transport http \ - --scope user \ - --header "Authorization: Bearer $NOVA_KEY" 2>&1)" - ADD_RC=$? - if [ "$ADD_RC" -eq 0 ]; then - pass "nova_mcp: user-scope bearer override registered at https://mcp.commcare.app/mcp" - else - fail "nova_mcp: claude mcp add failed (exit $ADD_RC)" "see error below; usually means the claude CLI version doesn't support --header or --scope user yet -$(printf '%s\n' "$ADD_OUT" | sed 's/^/ /')" - mark_fail - fi fi # ---- Auto-update hook (opt-in) diff --git a/package.json b/package.json index c7b3fe69..bada5022 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ace", - "version": "0.13.115", + "version": "0.13.116", "description": "AI Connect Engine - orchestrator for building Connect Opps using AI", "type": "module", "scripts": { diff --git a/playbook/integrations/nova-integration.md b/playbook/integrations/nova-integration.md index d707e91b..6b38b478 100644 --- a/playbook/integrations/nova-integration.md +++ b/playbook/integrations/nova-integration.md @@ -2,16 +2,18 @@ ## Status -**Live (via the Nova Claude Code plugin + ACE-side API-key override).** -First end-to-end smoke test on 2026-04-28; migrated to API-key auth -2026-05-08 after voidcraft-labs/nova-plugin#9 closed. +**Live (via the Nova Claude Code plugin's native PAT path).** First +end-to-end smoke test on 2026-04-28; migrated to API-key auth +2026-05-08; switched from ACE-side user-scope MCP override to native +plugin-side PAT support 2026-05-09 after voidcraft-labs/nova-plugin#11 +landed. Braxton (voidcraft-labs) ships Nova as a Claude Code plugin. ACE consumes it as a sibling plugin: install once per machine, mint an -API key, and ACE invokes Nova through its slash commands and MCP -tools. Both `/nova:autobuild` and `/nova:upload_to_hq` round-trip -cleanly under the ACE service identity, including across multiple -concurrent worktrees. +API key, export it where Claude Code can see it, and ACE invokes +Nova through its slash commands and MCP tools. Both `/nova:autobuild` +and `/nova:upload_to_hq` round-trip cleanly under the ACE service +identity, including across multiple concurrent worktrees. ## Install + auth @@ -20,8 +22,10 @@ concurrent worktrees. /plugin install nova@nova-marketplace ``` -Then mint an API key once and let `/ace:setup` register a user-scope -MCP override that carries it as a bearer: +The plugin's `.mcp.json` registers the Nova HTTP MCP entry with +`headers.Authorization = "Bearer ${NOVA_API_KEY}"`. Claude Code +expands `${NOVA_API_KEY}` from its process env at MCP-spawn time. +So: 1. Sign in at `https://commcare.app/settings` as the ACE Gmail identity (`ACE_GMAIL_ACCOUNT` in `.env`). @@ -29,20 +33,26 @@ MCP override that carries it as a bearer: `/nova:upload_to_hq` needs. 3. Save to 1Password vault `AI-Agents`, item `ACE - Nova`, field `api_key`. -4. Run `/ace:setup --force-env`. The setup script re-injects `.env` - from 1Password and registers the user-scope override: - - ``` - claude mcp add nova https://mcp.commcare.app/mcp \ - --transport http --scope user \ - --header "Authorization: Bearer ${NOVA_API_KEY}" +4. Run `/ace:setup --force-env` — this re-injects `.env` from + 1Password. +5. **Export `NOVA_API_KEY` in your shell rc** so Claude Code's + process env has it (the ACE `.env` at + `$CLAUDE_PLUGIN_DATA/.env` is read by ACE's own MCP servers when + they spawn, but Claude Code itself doesn't read it). One-time + step per machine: + + ```bash + echo 'export NOVA_API_KEY="$(grep -E "^NOVA_API_KEY=" "$HOME/.claude/plugins/data/ace-ace/.env" | sed -E "s/^NOVA_API_KEY=//;s/^\"(.*)\"\$/\\1/")"' >> ~/.zshrc + exec zsh # or restart your terminal ``` -Claude Code's URL-signature dedup keeps the user-scope entry over -the plugin's OAuth-mode entry, so every Nova MCP call goes through -bearer auth — no rotation, no refresh-token cascade. Tools end up -namespaced as `mcp__nova__*` (the plugin's `mcp__plugin_nova_nova__*` -namespace is suppressed). + Restart Claude Code so the new env var is in its process tree. + +When `NOVA_API_KEY` is set, every Nova MCP call goes through PAT +auth — no browser flow, no refresh-token cascade. When unset, the +header expands to `Bearer ` (empty) and Nova returns 401, after +which Claude Code falls back to interactive OAuth (the plugin's +default behavior, kept as a fallback for human-at-a-browser use). `/ace:doctor` exposes three Nova-related liveness lines: - `nova_env: NOVA_API_KEY present` @@ -54,8 +64,8 @@ ACE doesn't run a Nova MCP itself. ## Resolved blockers (kept for record) -Three blockers landed and were cleared between 2026-04-27 and -2026-05-08. Listed here for continuity — none active. +Four blockers landed and were cleared between 2026-04-27 and +2026-05-09. Listed here for continuity — none active. - **OAuth allowlist on Nova's side (2026-04-27 → cleared 2026-04-28).** Nova's Google OAuth client originally only allowlisted the operating @@ -71,9 +81,9 @@ Three blockers landed and were cleared between 2026-04-27 and under API-key auth. - **Refresh-token cascade across concurrent worktrees - (voidcraft-labs/nova-plugin#9, cleared 2026-05-08).** Two ACE - worktrees on one Nova/Google identity tripped a `deleteMany` - cascade in `@better-auth/oauth-provider`'s + (voidcraft-labs/nova-plugin#9, cleared 2026-05-08 server-side).** + Two ACE worktrees on one Nova/Google identity tripped a + `deleteMany` cascade in `@better-auth/oauth-provider`'s `handleRefreshTokenGrant`: when worktree B presented a stale refresh token (after worktree A had just rotated it), the provider treated the stale presentation as theft-detection and @@ -82,18 +92,33 @@ Three blockers landed and were cleared between 2026-04-27 and interactive OAuth every ~30 minutes. Resolved by Nova shipping API-key auth at the same MCP URL — no rotation, no cascade. +- **Plugin didn't expose the new PAT capability + (voidcraft-labs/nova-plugin#11, cleared 2026-05-09).** Nova + shipped the server-side PAT in May, but the plugin's `.mcp.json` + had no `headers` block — so Claude Code defaulted to OAuth even + with `NOVA_API_KEY` set. ACE worked around it by registering a + user-scope MCP entry at the same URL with `claude mcp add nova + ... --header "Authorization: Bearer $NOVA_API_KEY"`, relying on + URL-signature dedup to suppress the plugin entry. Fragile — + silently fell off across CLI updates. Resolved by adding a + `headers.Authorization = "Bearer ${NOVA_API_KEY}"` block to the + plugin's MCP config so the plugin uses the PAT directly. The + ACE workaround was deleted in this version. + ## ACE service identity for Nova Under API-key auth, the bearer is identity. ACE's `NOVA_API_KEY` points the entire fleet (every worktree, every operator's machine -that re-runs `/ace:setup`) at one Nova-side identity, which is the -account that minted the key (the ACE Gmail identity). All Nova -state — apps, HQ binding, settings — lives under that one user. +that re-runs `/ace:setup` and exports the key) at one Nova-side +identity, which is the account that minted the key (the ACE Gmail +identity). All Nova state — apps, HQ binding, settings — lives +under that one user. Rotating the key: regenerate at `commcare.app/settings`, update the 1Password item in place, then `/ace:setup --force-env` on each -machine. The user-scope override is re-registered with the new -bearer. +machine, then re-run the shell-export step from § Install + auth +(or restart the terminal if the export references the .env at read +time, which is the recommended pattern). The plugin's OAuth path is still available for human-at-a-browser use (one user, one Claude Code session, no concurrent sessions); @@ -151,8 +176,8 @@ ACE's contract: **Two distinct keys.** Don't conflate them: - `NOVA_API_KEY` (`sk-nova-v1-…`) authenticates **ACE → Nova**. Lives - in 1Password item `ACE - Nova` / `api_key`. Read by the user-scope - MCP override. + in 1Password item `ACE - Nova` / `api_key`. Read by the plugin's + MCP entry via `${NOVA_API_KEY}` env-var expansion. - HQ API key (UUID) authenticates **Nova → CommCareHQ**. Lives on Nova's settings page (server-side). Generated under `/account/api_keys/`. @@ -166,7 +191,7 @@ place, then re-paste into Nova settings. | Surface | Auth | |---------|------| | Nova web app | Google OAuth (sign-in with Google, ACE Gmail identity) | -| Nova MCP / plugin (ACE path) | Long-lived API key (`sk-nova-v1-…`) via user-scope MCP override | +| Nova MCP / plugin (ACE path) | Long-lived API key (`sk-nova-v1-…`) via `${NOVA_API_KEY}` in plugin's MCP `headers` | | Nova MCP / plugin (human path) | Real OAuth 2.1 (RFC-compliant DCR) — Braxton: "yank a client and the next call from it 401s instantly" | | HQ upload (downstream of Nova) | HQ API key from `account/api_keys/`, scoped per project space | @@ -187,20 +212,19 @@ to the ACE Gmail identity (`ACE_GMAIL_ACCOUNT`) at mint time. ## Gotchas -- **Plugin OAuth entry vs user-scope override coexist; URL-dedup - picks the override.** When `/ace:setup` registers the user-scope - bearer entry at `https://mcp.commcare.app/mcp`, Claude Code's MCP - host dedups by URL signature and keeps the more specific entry - (the bearer-carrying user-scope one). The plugin's OAuth entry is - silently suppressed; tools surface as `mcp__nova__*`. If both - show up in `claude mcp list` distinctly, something is wrong with - the override registration — re-run `/ace:setup --force-env`. - -- **Tool namespace depends on which entry wins.** Without the - override (e.g. machine that hasn't run `/ace:setup` post-migration) - tools surface as `mcp__plugin_nova_nova__*`. ACE skills use bare - tool names in prose ("Nova's `get_app` tool") so the model resolves - whichever namespace is live. +- **`NOVA_API_KEY` must be in Claude Code's process env**, not just + in `$CLAUDE_PLUGIN_DATA/.env`. Claude Code's MCP-header + `${NOVA_API_KEY}` expansion reads its own process env — the ACE + `.env` is only read by ACE's own MCP server processes, which + spawn under `dotenv.config()`. If you set the key in the .env but + not your shell rc, the plugin's Bearer header will be empty and + Nova will silently fall back to OAuth. The 2026-05-09 shell-export + step in § Install + auth fixes this. + +- **Tool namespace: `mcp__plugin_nova_nova__*`.** With the plugin's + native PAT path, there's only one MCP entry. ACE skills use bare + tool names in prose ("Nova's `get_app` tool") so the model + resolves the right namespace regardless. - **Three known upstream bugs** (none auth-related): - **`nova-plugin#1`** — `update_form` re-injects empty From fd6f3d206a28e63ba4468e54dfb9fadef6a97638 Mon Sep 17 00:00:00 2001 From: Jonathan Jackson Date: Fri, 8 May 2026 23:11:29 -0600 Subject: [PATCH 2/2] fix: CCZ truncation + update_yaml shallow-merge + nova_scopes probe + doc clarifications (0.13.116) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five durable fixes surfaced during turmeric/20260508-1951. **`commcare_download_ccz`** silently returned a truncated base64 payload (~2.5 KB short on a 29 KB CCZ) — the JSON-RPC transport's default token cap is ~10K tokens, and ~40 KB of base64 is ~20K. Switch to write-to-disk: return `ccz_path` to a file in `${CLAUDE_PLUGIN_DATA}/ccz-cache/`, drop inline `ccz_base64`. **Breaking** for any caller that reads the field; patched the two consumers in `scripts/`. **`update_yaml_file`** does top-level shallow-merge — patching `phases.commcare-setup.steps.X` wiped `phases.design-review` and other siblings under `phases`. Hit on this run; required manual restoration. Add `mergeMode: 'shallow' | 'deep'` (default shallow, backward-compat); deep mode recursively merges plain objects (arrays/primitives replace). **`bin/ace-doctor`** `nova_auth` only validates the bearer is accepted at the MCP transport, not that the PAT has per-tool scopes. A scope-missing key passed `nova_auth` and only failed mid-Phase-2 today (issue #174). Add `nova_scopes` probe — JSON-RPC `tools/call get_hq_connection`, branches on `scope_missing` / `configured:false` / `domain.name` / unexpected. Closes #174. **Doc updates:** - `skills/commcare-form-patch/SKILL.md` — `assessment-removal` actually strips any `commcareconnect`-namespaced wrapper (12 forms patched on this run, not the 6 the docs implied). - `skills/app-test-cases/SKILL.md` — `mobile_resolve_selectors` is a static lookup (reads `connect-.yaml`); no AVD required. - `agents/commcare-setup.md` — orphan-app recovery procedure for the architect-vs-PAT identity split (voidcraft-labs/nova-plugin#13). Tests: 733 passing, 35 skipped (no new failures). New suites: - `test/mcp/gdrive/update-yaml-file.test.ts` — 4 new tests covering shallow-wipes-siblings (the bug), deep-preserves-siblings, arrays-still-replace, object-vs-non-object mismatch. - `test/mcp/connect/unit/commcare-download-ccz.test.ts` — disk-write contract. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 21 ++++ agents/commcare-setup.md | 17 +++ bin/ace-doctor | 89 ++++++++++++++ mcp/connect-server.ts | 1 + mcp/connect/backends/commcare.ts | 57 +++++++-- mcp/google-drive-server.ts | 58 +++++++-- package-lock.json | 4 +- scripts/run-form-walk.ts | 6 +- scripts/smoke-app-multimedia-coverage.ts | 9 +- skills/app-release/SKILL.md | 6 +- skills/app-test-cases/SKILL.md | 5 +- skills/commcare-form-patch/SKILL.md | 13 +- .../unit/commcare-download-ccz.test.ts | 20 ++++ test/mcp/gdrive/update-yaml-file.test.ts | 112 ++++++++++++++++++ 14 files changed, 389 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff4bba37..d8e25ee1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ All notable changes to the ACE plugin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the plugin follows [semantic versioning](https://semver.org/spec/v2.0.0.html). +## 0.13.116 — 2026-05-08 + +**`commcare_download_ccz` now writes the CCZ to disk and returns a path. Inline `ccz_base64` removed (BREAKING for any caller that reads the field).** + +The MCP transport silently truncates large base64 payloads — a 29 KB CCZ came back ~2.5 KB short of the original, and `unzip` rejected the corrupted ZIP. Default `MAX_MCP_OUTPUT_TOKENS` is ~10K tokens; ~40 KB of base64 is ~20 K tokens, well over the cap. Workaround during `turmeric/20260508-1951` was to call `CommCareBackend.downloadCcz` directly via tsx. + +- `mcp/connect/backends/commcare.ts` — `downloadCcz()` now writes bytes to `${CLAUDE_PLUGIN_DATA}/ccz-cache/--.ccz`, returning `{status, size_bytes, ccz_path, ccz_sha256, connect_markers, projected_connect_state}`. The `ccz_base64` field is gone. CCZs > 25 MB still get written + return the path; only the in-memory inflate (`connect_markers` / `projected_connect_state`) is skipped. +- `mcp/connect-server.ts` — tool description updated to lead with the new contract. +- `skills/commcare-form-patch/SKILL.md`, `skills/app-release/SKILL.md` — patched to reference `ccz_path` instead of `ccz_base64`. +- `test/mcp/connect/unit/commcare-download-ccz.test.ts` — added a unit test verifying file is written, path is returned, sha256 matches, and `ccz_base64` is absent. + +**Operator/caller action on update:** anything that decoded `ccz_base64` must now `fs.readFileSync(result.ccz_path)`. Cache files persist across calls — clean up `${CLAUDE_PLUGIN_DATA}/ccz-cache/` if disk usage matters. + +**Other improvements in this version:** + +- `mcp/google-drive-server.ts` `update_yaml_file` — added `mergeMode: 'shallow' | 'deep'` parameter (default `'shallow'`, backward-compat). The default top-level shallow merge wiped sibling keys when patching nested structures (`phases.commcare-setup.steps.X` would replace the entire `phases` block, wiping `phases.design-review`); pass `mergeMode: 'deep'` to recursively merge plain objects (arrays/primitives still replace). Hit in `turmeric/20260508-1951` requiring manual `phases.design-review` restoration. +- `bin/ace-doctor` — added `nova_scopes` probe immediately after `nova_auth` to validate per-tool scopes (`nova.hq.read` for `get_hq_connection`, `nova.hq.write` for `upload_app_to_hq`). Pre-0.13.116 a scope-missing key passed `nova_auth` and only failed mid-Phase-2. +- `skills/commcare-form-patch/SKILL.md` — clarified `assessment-removal` patch class strips any `commcareconnect`-namespaced wrapper (including ``), not just ``. +- `skills/app-test-cases/SKILL.md` — clarified `mobile_resolve_selectors` is a static lookup; no AVD/APK required. +- `agents/commcare-setup.md` — added "Recovery: orphan apps from architect-vs-PAT identity split" subsection (voidcraft-labs/nova-plugin#13). + ## 0.13.111 — 2026-05-09 **Drop the Nova MCP user-scope-override workaround now that the Nova plugin reads `NOVA_API_KEY` natively (voidcraft-labs/nova-plugin#11).** diff --git a/agents/commcare-setup.md b/agents/commcare-setup.md index e0bf4099..6997c309 100644 --- a/agents/commcare-setup.md +++ b/agents/commcare-setup.md @@ -101,6 +101,23 @@ connection because the plugin's MCP entry expands `${NOVA_API_KEY}` once per session start; every subagent dispatch sees the same `get_hq_connection` result. +#### Recovery: orphan apps from architect-vs-PAT identity split + +If `upload_app_to_hq` returns `error_type: not_found` for an `app_id` +that the architect just successfully reported building, the architect +ran under a different Nova identity than the level-0 PAT — the app +exists in Nova storage but is owned by an account whose PAT is not +visible to the level-0 session, so the upload tool can't find it. +(Filed upstream as voidcraft-labs/nova-plugin#13.) + +Recovery is to re-dispatch the architect from level 0. The fresh +dispatch uses the post-fix MCP context, which expands `${NOVA_API_KEY}` +to the level-0 PAT identity; the rebuild and the upload then run under +the same identity. Cost: ~10–15 min to rebuild each app. Canonical +instance: `turmeric/20260508-1951` Phase 2 — both Learn and Deliver +apps were orphaned and had to be rebuilt before `upload_app_to_hq` +could find them. + ### Step 1: PDD to Apps (sequential) Invoke `pdd-to-learn-app`, then `pdd-to-deliver-app`. diff --git a/bin/ace-doctor b/bin/ace-doctor index efdf060f..50bfdfa5 100755 --- a/bin/ace-doctor +++ b/bin/ace-doctor @@ -1218,6 +1218,95 @@ else warn "nova_auth: skipped — NOVA_API_KEY not set" "see nova_env warning above; mint at https://commcare.app/settings" fi +# 6a. nova_scopes — validate the PAT actually has the per-tool scopes ACE +# needs. `nova_auth` only confirms the bearer is accepted at the transport +# layer; a key without `nova.hq.read` / `nova.hq.write` will pass nova_auth +# and only fail mid-Phase-2 / app-deploy with a misleading "scope_missing". +# Probe by calling `get_hq_connection` (the cheapest read tool that exercises +# `nova.hq.read`) and parse the response — branches: +# - 2xx + configured:true + domain.name → pass (read scope + HQ key bound) +# - 2xx + configured:false → warn (read scope OK, no HQ key in Nova) +# - 2xx + error_type:scope_missing → fail (rotate / regrant scope) +# - other → warn (unexpected response) +if [ -n "$NOVA_KEY_VAL" ]; then + NOVA_SCOPE_BODY="$(curl -sS --max-time 5 \ + -X POST \ + -H "Authorization: Bearer $NOVA_KEY_VAL" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json, text/event-stream" \ + -d '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"get_hq_connection","arguments":{}}}' \ + "https://mcp.commcare.app/mcp" 2>/dev/null || echo "")" + NOVA_SCOPE_CODE="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \ + -X POST \ + -H "Authorization: Bearer $NOVA_KEY_VAL" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json, text/event-stream" \ + -d '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"get_hq_connection","arguments":{}}}' \ + "https://mcp.commcare.app/mcp" 2>/dev/null || echo "000")" + # The body may be SSE-wrapped (lines like "data: {...}"); strip the + # `data: ` prefix on the JSON-RPC frame line before parsing. python3 is + # ubiquitous on macOS + most Linux; jq isn't always present. + NOVA_SCOPE_PARSED="$(printf '%s' "$NOVA_SCOPE_BODY" | python3 -c ' +import sys, json, re +raw = sys.stdin.read() +# Pull JSON object from SSE-style "data: {...}" or plain JSON. +m = re.search(r"data:\s*(\{.*\})", raw) +payload = m.group(1) if m else raw.strip() +try: + obj = json.loads(payload) +except Exception: + print("PARSE_ERROR") + sys.exit(0) +# JSON-RPC tools/call wraps the actual result in result.content[0].text +# as a JSON string. Unwrap it. +inner = None +try: + inner_raw = obj.get("result", {}).get("content", [{}])[0].get("text", "") + inner = json.loads(inner_raw) if inner_raw else None +except Exception: + inner = obj.get("result") or obj +target = inner if isinstance(inner, dict) else (obj.get("result") if isinstance(obj.get("result"), dict) else obj) +err_type = target.get("error_type") if isinstance(target, dict) else None +required = target.get("required_scope") if isinstance(target, dict) else None +configured = target.get("configured") if isinstance(target, dict) else None +domain_name = None +if isinstance(target, dict): + d = target.get("domain") + if isinstance(d, dict): + domain_name = d.get("name") +print(f"err_type={err_type}\nrequired={required}\nconfigured={configured}\ndomain_name={domain_name}") +' 2>/dev/null || echo "PARSE_ERROR")" + NOVA_ERR_TYPE="$(printf '%s' "$NOVA_SCOPE_PARSED" | sed -n 's/^err_type=//p')" + NOVA_REQUIRED_SCOPE="$(printf '%s' "$NOVA_SCOPE_PARSED" | sed -n 's/^required=//p')" + NOVA_CONFIGURED="$(printf '%s' "$NOVA_SCOPE_PARSED" | sed -n 's/^configured=//p')" + NOVA_DOMAIN_NAME="$(printf '%s' "$NOVA_SCOPE_PARSED" | sed -n 's/^domain_name=//p')" + + case "$NOVA_SCOPE_CODE" in + 2*) + if [ "$NOVA_ERR_TYPE" = "scope_missing" ]; then + REQ_TXT="${NOVA_REQUIRED_SCOPE:-unknown}" + fail "nova_scopes: NOVA_API_KEY missing scope $REQ_TXT" "edit at https://commcare.app/settings → API tokens → grant HQ Read + HQ Write" + elif [ "$NOVA_CONFIGURED" = "True" ] && [ -n "$NOVA_DOMAIN_NAME" ] && [ "$NOVA_DOMAIN_NAME" != "None" ]; then + pass "nova_scopes: HQ Read scope granted; bound to $NOVA_DOMAIN_NAME" + elif [ "$NOVA_CONFIGURED" = "False" ]; then + warn "nova_scopes: HQ Read granted but no HQ key bound in Nova settings" "paste an HQ API key at https://commcare.app/settings before /ace:run" + else + TRUNC="$(printf '%s' "$NOVA_SCOPE_BODY" | head -c 200)" + warn "nova_scopes: unexpected response $TRUNC" "investigate Nova MCP response shape" + fi + ;; + 000) + warn "nova_scopes: cannot reach https://mcp.commcare.app/mcp (network/DNS/timeout)" "check VPN/proxy; see nova_auth above" + ;; + *) + TRUNC="$(printf '%s' "$NOVA_SCOPE_BODY" | head -c 200)" + warn "nova_scopes: HTTP $NOVA_SCOPE_CODE — $TRUNC" "investigate Nova MCP health; if persists, rotate the key" + ;; + esac +else + warn "nova_scopes: skipped — NOVA_API_KEY not set" "see nova_env warning above; mint at https://commcare.app/settings" +fi + # 7. ace-mobile — local-only MCP (Maestro + adb + AVD + APK). It does not # authenticate against any remote service at runtime, so there is no # "live auth probe" distinct from the others. The auth-adjacent things diff --git a/mcp/connect-server.ts b/mcp/connect-server.ts index 2672b04e..6f69855f 100644 --- a/mcp/connect-server.ts +++ b/mcp/connect-server.ts @@ -427,6 +427,7 @@ server.tool('commcare_release_build', ); server.tool('commcare_download_ccz', + 'Writes the CCZ to `$CLAUDE_PLUGIN_DATA/ccz-cache/` and returns `ccz_path` for the caller to open. Inline `ccz_base64` was removed in 0.13.116 because the MCP transport silently truncated large base64 payloads — a 29 KB CCZ came back missing 2.5 KB of trailing bytes. Returns `{status, size_bytes, ccz_path, ccz_sha256, connect_markers, projected_connect_state}`. CCZs > 25 MB still get written to disk + return the path, but skip the in-memory inflate (so `connect_markers` and `projected_connect_state` are absent). Use `connect_markers` and `projected_connect_state` for cheap server-side validation; read the file at `ccz_path` only when you need the raw bytes.', { domain: z.string(), app_id: z.string(), diff --git a/mcp/connect/backends/commcare.ts b/mcp/connect/backends/commcare.ts index 7c11b22e..7e164062 100644 --- a/mcp/connect/backends/commcare.ts +++ b/mcp/connect/backends/commcare.ts @@ -1,7 +1,12 @@ import type { APIRequestContext, APIResponse } from 'playwright'; import { unzipSync, strFromU8 } from 'fflate'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { createHash } from 'node:crypto'; import { PlaywrightSession } from '../auth/playwright-session.js'; import { SessionExpiredError } from '../errors.js'; +import { resolvePluginDataDir } from '../../../lib/plugin-data-dir.js'; /** * CommCare HQ atoms — release apps that Nova uploaded as drafts. @@ -78,8 +83,17 @@ export interface DownloadCczArgs { export interface DownloadCczResult { status: number; size_bytes: number; - /** Base64-encoded CCZ bytes. Capped at 25 MB; larger CCZs return size_bytes only. */ - ccz_base64?: string; + /** + * Absolute path to the on-disk CCZ. Written under + * `${CLAUDE_PLUGIN_DATA}/ccz-cache/--.ccz`. + * Replaces the pre-0.13.116 inline `ccz_base64` payload, which the MCP + * transport silently truncated above ~10K-token responses. + * Callers should open this path directly. The file is left in place; + * callers are responsible for cleanup if they need it. + */ + ccz_path?: string; + /** Hex-encoded sha256 of the CCZ bytes. Useful for de-dup / cache invalidation. */ + ccz_sha256?: string; /** Per-namespace marker counts grepped from the inflated CCZ (when fetched). */ connect_markers?: { deliver: number; @@ -416,11 +430,22 @@ export class CommCareBackend { } const buf = await res.body(); const size = buf.byteLength; - // Skip base64 round-trip + marker grep on > 25 MB CCZs to keep MCP - // responses small. If the operator needs the bytes for a larger app, - // they should download via curl directly. + // Always persist bytes to disk and return a path. Inline base64 was + // removed in 0.13.116 because the MCP transport silently truncated + // large base64 payloads (~10K-token cap) — a 29 KB CCZ came back + // missing 2.5 KB of trailing bytes, which made every `unzip` reject + // the file. Callers now read from `ccz_path`. + const sha = createHash('sha256').update(buf).digest('hex'); + const sha8 = sha.slice(0, 8); + const idForName = args.build_id ?? args.app_id; + const status_label = args.build_id ? 'build' : 'release'; + const ccz_path = writeCczToCache(buf, `${idForName}-${status_label}-${sha8}.ccz`); + + // Skip marker grep on > 25 MB CCZs (cheap CCZ-on-disk; expensive + // inflate). The caller still gets `ccz_path` and can inspect the + // archive directly if needed. if (size > 25 * 1024 * 1024) { - return { status, size_bytes: size }; + return { status, size_bytes: size, ccz_path, ccz_sha256: sha }; } // CCZ is a ZIP whose entries are typically DEFLATE-compressed; the // form XML is NOT readable from the raw zip bytes. Inflate in memory @@ -440,7 +465,8 @@ export class CommCareBackend { return { status, size_bytes: size, - ccz_base64: buf.toString('base64'), + ccz_path, + ccz_sha256: sha, connect_markers, projected_connect_state, }; @@ -701,6 +727,23 @@ export function mediaTypeFromContentType(ct: string): 'image' | 'audio' | 'video throw new Error(`commcare_upload_multimedia: unsupported content_type ${ct}`); } +/** + * Resolve the on-disk cache directory for downloaded CCZs and write + * `buf` to `/`. Falls back to `os.tmpdir()/ace-ccz-cache` + * when `${CLAUDE_PLUGIN_DATA}` is unresolvable (test environments). Creates + * the directory recursively. Returns the absolute path written. + * + * Exported for unit tests. + */ +export function writeCczToCache(buf: Buffer, filename: string): string { + const dataDir = resolvePluginDataDir(import.meta.url) ?? path.join(os.tmpdir(), 'ace-ccz-cache-fallback'); + const cacheDir = path.join(dataDir, 'ccz-cache'); + fs.mkdirSync(cacheDir, { recursive: true }); + const out = path.join(cacheDir, filename); + fs.writeFileSync(out, buf); + return out; +} + /** * Count Connect-namespace marker elements inside a CCZ buffer. * diff --git a/mcp/google-drive-server.ts b/mcp/google-drive-server.ts index d5970802..d3d06e22 100644 --- a/mcp/google-drive-server.ts +++ b/mcp/google-drive-server.ts @@ -475,14 +475,15 @@ server.tool( // 10a. Patch a YAML file server-side (read+merge+CAS-write in one call) server.tool( 'update_yaml_file', - 'Patch a YAML-content Google Doc in one MCP call: the server reads the current content + revisionVersion, parses it as YAML (treating empty/missing as `{}`), shallow-merges `patch` into the top-level keys (replace, NOT deep-merge — predictable), serializes back to YAML, and writes with optimistic-concurrency. On a `revision_conflict` (a concurrent writer landed between read and write) the call retries once with the freshly-observed revision. Use this for run_state.yaml / opp.yaml updates instead of pairing drive_read_file + drive_update_file by hand: it saves one round-trip per state transition AND keeps the full file content out of the model context (the model only sends the diff). For arbitrary text files use drive_update_file instead.', + '**Top-level shallow merge by default** — passing `phases: {commcare-setup: {...}}` REPLACES the entire `phases` key. Pass `mergeMode: \'deep\'` to recursively merge plain objects. Patch a YAML-content Google Doc in one MCP call: the server reads the current content + revisionVersion, parses it as YAML (treating empty/missing as `{}`), merges `patch` into the existing structure, serializes back to YAML, and writes with optimistic-concurrency. On a `revision_conflict` (a concurrent writer landed between read and write) the call retries once with the freshly-observed revision. Use this for run_state.yaml / opp.yaml updates instead of pairing drive_read_file + drive_update_file by hand: it saves one round-trip per state transition AND keeps the full file content out of the model context (the model only sends the diff). For arbitrary text files use drive_update_file instead.', { fileId: z.string().describe('The Google Drive file ID of the YAML doc'), - patch: z.record(z.unknown()).describe('Object whose top-level keys are merged into the existing YAML (replace, not deep-merge). Use a nested object only when you intend to fully replace that subtree.'), + patch: z.record(z.unknown()).describe('Object merged into the existing YAML. With `mergeMode: \'shallow\'` (default), top-level keys are replaced wholesale. With `mergeMode: \'deep\'`, plain objects are merged recursively (arrays/primitives still replace).'), + mergeMode: z.enum(['shallow', 'deep']).optional().describe("Merge strategy. 'shallow' (default) replaces top-level keys (backward-compatible, but DESTRUCTIVE at the top-level key it touches). Use 'deep' when patch keys nest into existing structure (e.g. phases.X.steps.Y); 'shallow' would wipe sibling keys at every level above your patch."), }, - async ({ fileId, patch }) => { + async ({ fileId, patch, mergeMode }) => { try { - const r = await handleUpdateYamlFile({ fileId, patch }, drive); + const r = await handleUpdateYamlFile({ fileId, patch, mergeMode }, drive); return result(r); } catch (e: any) { return error(e.message); @@ -851,11 +852,47 @@ function isTextMimeType(mimeType: string): boolean { * concurrent writer landing between our read and our write); a second * conflict surfaces to the caller. */ +/** + * Plain-object detection: only true for `{}`-style records. Class instances, + * arrays, null, primitives all return false. Used by deepMerge to decide + * whether to recurse vs. replace. + */ +function isPlainObject(value: unknown): value is Record { + return ( + value !== null && + typeof value === 'object' && + !Array.isArray(value) && + Object.getPrototypeOf(value) === Object.prototype + ); +} + +/** + * Recursively merge `source` into `target`. Plain objects merge key-by-key; + * arrays and non-object values replace (matches lodash _.merge semantics for + * plain objects, but does NOT concat arrays). Object-vs-non-object mismatch + * at a path: replace. + */ +function deepMerge( + target: Record, + source: Record, +): Record { + const out: Record = { ...target }; + for (const [k, v] of Object.entries(source)) { + const existing = out[k]; + if (isPlainObject(existing) && isPlainObject(v)) { + out[k] = deepMerge(existing, v); + } else { + out[k] = v; + } + } + return out; +} + export async function handleUpdateYamlFile( - args: { fileId: string; patch: Record }, + args: { fileId: string; patch: Record; mergeMode?: 'shallow' | 'deep' }, driveClient: typeof drive = drive, ): Promise<{ id: string; name: string; modifiedTime: string; revisionVersion: string | undefined }> { - const { fileId, patch } = args; + const { fileId, patch, mergeMode = 'shallow' } = args; const maxAttempts = 2; let lastErr: any; @@ -863,8 +900,13 @@ export async function handleUpdateYamlFile( const current = await handleReadFile({ fileId }, driveClient); const parsed = current.content && current.content.trim() ? YAML.parse(current.content) : {}; const base: Record = parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as Record) : {}; - const merged: Record = { ...base }; - for (const [k, v] of Object.entries(patch)) merged[k] = v; + let merged: Record; + if (mergeMode === 'deep') { + merged = deepMerge(base, patch); + } else { + merged = { ...base }; + for (const [k, v] of Object.entries(patch)) merged[k] = v; + } const newContent = YAML.stringify(merged); try { diff --git a/package-lock.json b/package-lock.json index 96dcb5e3..47571f5e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ace", - "version": "0.13.101", + "version": "0.13.116", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ace", - "version": "0.13.101", + "version": "0.13.116", "license": "ISC", "dependencies": { "@anthropic-ai/sdk": "^0.94.0", diff --git a/scripts/run-form-walk.ts b/scripts/run-form-walk.ts index 69e48994..e291e646 100644 --- a/scripts/run-form-walk.ts +++ b/scripts/run-form-walk.ts @@ -41,7 +41,7 @@ * * Shipped 0.13.29. */ -import { writeFileSync } from 'node:fs'; +import { writeFileSync, readFileSync } from 'node:fs'; import { unzipSync, strFromU8 } from 'fflate'; import { DOMParser } from '@xmldom/xmldom'; @@ -585,12 +585,12 @@ async function main(): Promise { build_id: args.build_id, include_multimedia: false, }); - if (ccz.status !== 200 || !ccz.ccz_base64) { + if (ccz.status !== 200 || !ccz.ccz_path) { console.error(`download_ccz failed: status=${ccz.status} bytes=${ccz.size_bytes}`); return 2; } - const cczBuf = Buffer.from(ccz.ccz_base64, 'base64'); + const cczBuf = readFileSync(ccz.ccz_path); const walked = walkCcz({ cczBuf, domain: args.domain, diff --git a/scripts/smoke-app-multimedia-coverage.ts b/scripts/smoke-app-multimedia-coverage.ts index deff18d8..a9b907e0 100644 --- a/scripts/smoke-app-multimedia-coverage.ts +++ b/scripts/smoke-app-multimedia-coverage.ts @@ -32,6 +32,7 @@ import { config as dotenvConfig } from 'dotenv'; import * as path from 'node:path'; +import { readFileSync } from 'node:fs'; import { unzipSync, strFromU8 } from 'fflate'; import { ContentGeneratorClient } from '../lib/content-generator-client.js'; @@ -87,10 +88,10 @@ try { let end = stepStart(1, 'Downloading current CCZ'); const ccz1 = await c.downloadCcz({ domain, app_id: appId }); const t1 = end(); - if (ccz1.status !== 200 || !ccz1.ccz_base64) { + if (ccz1.status !== 200 || !ccz1.ccz_path) { throw new Error(`download_ccz #1 status=${ccz1.status} size=${ccz1.size_bytes}`); } - const buf1 = Buffer.from(ccz1.ccz_base64, 'base64'); + const buf1 = readFileSync(ccz1.ccz_path); const entries1 = unzipSync(new Uint8Array(buf1)); const formPath = `modules-${moduleIdx}/forms-${formIdx}.xml`; if (!entries1[formPath]) { @@ -219,11 +220,11 @@ try { include_multimedia: true, }); const t6 = end6(); - if (ccz2.status !== 200 || !ccz2.ccz_base64) { + if (ccz2.status !== 200 || !ccz2.ccz_path) { console.error(`verify download_ccz status=${ccz2.status} size=${ccz2.size_bytes}`); process.exit(2); } - const buf2 = Buffer.from(ccz2.ccz_base64, 'base64'); + const buf2 = readFileSync(ccz2.ccz_path); const entries2 = unzipSync(new Uint8Array(buf2)); console.log(` full CCZ size=${buf2.byteLength} entries=${Object.keys(entries2).length}`); diff --git a/skills/app-release/SKILL.md b/skills/app-release/SKILL.md index 1f2d0928..f4e85bf2 100644 --- a/skills/app-release/SKILL.md +++ b/skills/app-release/SKILL.md @@ -218,8 +218,10 @@ just consumes its `clean | blocked` verdict. - `commcare_release_build` — POST `/apps/view//releases/release//`, sets `is_released: true`. - `commcare_download_ccz` — GET `/apps/api/download_ccz/?app_id=...&latest=release`, - returns CCZ bytes (base64) + Connect-marker counts grepped from the - inflated form XML. + writes the CCZ to `$CLAUDE_PLUGIN_DATA/ccz-cache/` and returns + `{ccz_path, ccz_sha256, connect_markers, projected_connect_state}`. + (Inline `ccz_base64` was removed in 0.13.116 — MCP transport silently + truncated large base64 payloads. Read bytes from `ccz_path`.) These run against `ACE_HQ_BASE_URL` (default `https://www.commcarehq.org`) using the same Playwright session as the Connect atoms — Connect's OAuth-via-CCHQ flow leaves valid CCHQ cookies in diff --git a/skills/app-test-cases/SKILL.md b/skills/app-test-cases/SKILL.md index 931e0ba7..e42533f9 100644 --- a/skills/app-test-cases/SKILL.md +++ b/skills/app-test-cases/SKILL.md @@ -70,7 +70,10 @@ between major form sections): `takeScreenshot: "sc-J-final"` for the deep UX judge to grade - Resolve any `${SELECTOR:logical-name}` placeholders via `mobile_resolve_selectors` against the current APK selector map - before validating + before validating. **This is a static lookup, not live introspection** — + the atom reads `mcp/mobile/selectors/connect-.yaml` and + substitutes; no AVD or APK is required to run this step. AVD is only + needed for actual recipe execution in Phase 5 / `/ace:qa-deep`. - Validate via `mobile_validate_recipe` before writing **Use live labels from Nova's `get_form` response, not the PDD diff --git a/skills/commcare-form-patch/SKILL.md b/skills/commcare-form-patch/SKILL.md index bb4c8e64..aecd350a 100644 --- a/skills/commcare-form-patch/SKILL.md +++ b/skills/commcare-form-patch/SKILL.md @@ -55,13 +55,20 @@ succeeds without it. | Class | Selector | Action | When to use | |-------|----------|--------|-------------| -| **`assessment-removal`** *(recommended)* | Form contains a wrapper element whose body is exactly one `` block | Strip the wrapper element + its `` references entirely | Default for Connect Learn apps. Matches the working Turmeric pattern; unblocks `/opportunity/init/`. | +| **`assessment-removal`** *(recommended)* | Form contains a wrapper element whose body is exactly one `commcareconnect`-namespaced block — `` OR `` (or any other connect-namespaced wrapper) | Strip the wrapper element + its `` references entirely | Default for Connect Learn apps. Matches the working Turmeric pattern; unblocks `/opportunity/init/`. | | `user-score` | Form contains an empty `` inside a `commcareconnect`-namespaced `` block | Rewrite to `/data/total_score` | Legacy class. Necessary but not sufficient — Connect rejects the wrapper element regardless. Kept for diagnostic use; prefer `assessment-removal`. | `assessment-removal` is a strict superset: any form fixed by `user-score` is also fixed by `assessment-removal`, plus assessment-removal removes the wrapper element that Connect was already going to reject. +**Scope clarification (0.13.116):** Despite the class name, the matcher +strips ANY `commcareconnect`-namespaced wrapper element at the form body +level — not just ``. In `turmeric/20260508-1951` 12 Learn +forms were patched (a mix of `` and `` +wrappers), not the 6 the older docs implied. The class name is preserved +for backward compatibility with caller patch entries; do NOT rename. + **Apply assessment-removal to LEARN apps only.** Connect's `/opportunity/init/` parser only chokes on Learn-side connect markup; Deliver-app `` wrappers are tolerated. More importantly, @@ -151,7 +158,9 @@ quiz forms in scope for nova-plugin#5). 3. **Resolve form unique_ids per patch entry.** Call `commcare_download_ccz({domain, app_id, build_id})` for the entry's - app, decode the base64 CCZ, parse `suite.xml`, and read every + app, read the CCZ from the returned `ccz_path` (the atom writes the + bytes to `$CLAUDE_PLUGIN_DATA/ccz-cache/`; inline `ccz_base64` was + removed in 0.13.116), parse `suite.xml`, and read every `` for the bare `unique_id`. The `id` IS the 32-char hex form unique_id used by `edit_form_attr`. Map each `./modules-N/forms-M.xml` location back to a `(module, form)` diff --git a/test/mcp/connect/unit/commcare-download-ccz.test.ts b/test/mcp/connect/unit/commcare-download-ccz.test.ts index a2cadea2..2edb9400 100644 --- a/test/mcp/connect/unit/commcare-download-ccz.test.ts +++ b/test/mcp/connect/unit/commcare-download-ccz.test.ts @@ -16,6 +16,7 @@ * Mirrors the unit-test plumbing pattern from commcare-patch-xform.test.ts. */ import { describe, it, expect, vi } from 'vitest'; +import * as fs from 'node:fs'; import { CommCareBackend } from '../../../../mcp/connect/backends/commcare.js'; interface FakeResponse { @@ -119,6 +120,25 @@ describe('CommCareBackend.downloadCcz — include_multimedia flag', () => { expect(u.searchParams.has('include_multimedia')).toBe(false); }); + it('writes the CCZ to disk and returns a path; never inlines base64 (0.13.116)', async () => { + const fake = fakeRequest({ getStatus: 200, getBody: emptyZip }); + const backend = new CommCareBackend({ baseUrl, session: fakeSession(fake.request) }); + const r = await backend.downloadCcz({ ...args }); + + // The bytes belong on disk, NOT in the response. + expect((r as any).ccz_base64).toBeUndefined(); + expect(r.ccz_path).toBeDefined(); + expect(r.ccz_sha256).toMatch(/^[0-9a-f]{64}$/); + + // File actually exists at the returned path and matches input bytes. + const onDisk = fs.readFileSync(r.ccz_path!); + expect(onDisk.equals(emptyZip)).toBe(true); + + // Path includes the cache directory + filename includes sha8. + expect(r.ccz_path).toContain('ccz-cache'); + expect(r.ccz_path).toContain(r.ccz_sha256!.slice(0, 8)); + }); + it('threads include_multimedia alongside an explicit build_id', async () => { let capturedUrl = ''; const fake = fakeRequest({ diff --git a/test/mcp/gdrive/update-yaml-file.test.ts b/test/mcp/gdrive/update-yaml-file.test.ts index df8eb0b6..6c307348 100644 --- a/test/mcp/gdrive/update-yaml-file.test.ts +++ b/test/mcp/gdrive/update-yaml-file.test.ts @@ -102,6 +102,118 @@ describe('update_yaml_file: server-side patch+CAS', () => { expect(fake.files.update).toHaveBeenCalledTimes(2); }); + it('shallow mode wipes siblings under the patched key (the bug)', async () => { + const yaml = YAML.stringify({ + phases: { + 'design-review': { status: 'done', verdict: 'pass' }, + 'commcare-setup': { status: 'in_progress' }, + }, + }); + const fake = makeFakeDriveWithDoc(yaml, '1'); + + await handleUpdateYamlFile( + { + fileId: 'f1', + patch: { phases: { 'commcare-setup': { steps: { 'commcare-form-patch': { status: 'done' } } } } }, + }, + fake as any, + ); + + // Bug: design-review entirely wiped; commcare-setup.status also wiped. + expect(YAML.parse(fake.state.content)).toEqual({ + phases: { + 'commcare-setup': { steps: { 'commcare-form-patch': { status: 'done' } } }, + }, + }); + }); + + it('deep mode preserves siblings at every nesting level', async () => { + const yaml = YAML.stringify({ + phases: { + 'design-review': { status: 'done', verdict: 'pass' }, + 'commcare-setup': { status: 'in_progress', steps: { 'app-build': { status: 'done' } } }, + }, + }); + const fake = makeFakeDriveWithDoc(yaml, '1'); + + await handleUpdateYamlFile( + { + fileId: 'f1', + patch: { phases: { 'commcare-setup': { steps: { 'commcare-form-patch': { status: 'done' } } } } }, + mergeMode: 'deep', + }, + fake as any, + ); + + expect(YAML.parse(fake.state.content)).toEqual({ + phases: { + 'design-review': { status: 'done', verdict: 'pass' }, + 'commcare-setup': { + status: 'in_progress', + steps: { + 'app-build': { status: 'done' }, + 'commcare-form-patch': { status: 'done' }, + }, + }, + }, + }); + }); + + it('deep mode replaces arrays (does not concat)', async () => { + const yaml = YAML.stringify({ + connect: { payment_units: [{ name: 'a' }, { name: 'b' }] }, + }); + const fake = makeFakeDriveWithDoc(yaml, '1'); + + await handleUpdateYamlFile( + { + fileId: 'f1', + patch: { connect: { payment_units: [{ name: 'c' }] } }, + mergeMode: 'deep', + }, + fake as any, + ); + + expect(YAML.parse(fake.state.content)).toEqual({ + connect: { payment_units: [{ name: 'c' }] }, + }); + }); + + it('deep mode replaces on object-vs-non-object mismatch', async () => { + const yaml = YAML.stringify({ + gates: { 'app-deploy': { passed: true, evidence: 'foo' } }, + }); + const fake = makeFakeDriveWithDoc(yaml, '1'); + + // Replace object with primitive + await handleUpdateYamlFile( + { + fileId: 'f1', + patch: { gates: { 'app-deploy': 'closed' } }, + mergeMode: 'deep', + }, + fake as any, + ); + + expect(YAML.parse(fake.state.content)).toEqual({ + gates: { 'app-deploy': 'closed' }, + }); + + // And primitive replaced with object + await handleUpdateYamlFile( + { + fileId: 'f1', + patch: { gates: { 'app-deploy': { reopened: true } } }, + mergeMode: 'deep', + }, + fake as any, + ); + + expect(YAML.parse(fake.state.content)).toEqual({ + gates: { 'app-deploy': { reopened: true } }, + }); + }); + it('gives up after 1 retry on persistent revision_conflict', async () => { const yaml = YAML.stringify({ x: 1 }); const fake = makeFakeDriveWithDoc(yaml, '1');