From ff3ebe05da4592670029f9bfec142a631244757f Mon Sep 17 00:00:00 2001 From: Jonathan Jackson Date: Thu, 21 May 2026 18:46:00 -0600 Subject: [PATCH] feat(commcare): lookup_table rows + user atoms (5 of remaining REST) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Continues the REST commcare atom batch. All verified end-to-end against connect-ace-prod. - commcare_get_lookup_table_rows: GETs all rows in domain and filters by data_type_id. Flattens fields to {col: first-value}. - commcare_lookup_table_append_rows: POSTs one row per call (Tastypie doesn't support list POST for this resource). Round-trip verified by appending 3 interview_schedule rows. - commcare_list_users: GET /a//api/v0.5/user/ with pagination and optional group filter. Returns each user's id, username, basic profile, and full user_data. - commcare_get_user: GET single user by couch id. - commcare_update_user_field: GET → mutate user_data → PUT. V0_5 resource doesn't expose PATCH so we PUT the merged user_data dict. Pass value=null to clear. All five atoms use the new API-key auth pattern (introduced in PR #390). Co-Authored-By: Claude Opus 4.7 --- .claude-plugin/marketplace.json | 4 +- .claude-plugin/plugin.json | 2 +- VERSION | 2 +- mcp/connect-server.ts | 48 +++++ mcp/connect/backends/commcare.ts | 271 ++++++++++++++++++++++++++ package.json | 2 +- scripts/probe-commcare-lookup-rows.ts | 39 ++++ 7 files changed, 363 insertions(+), 5 deletions(-) create mode 100644 scripts/probe-commcare-lookup-rows.ts diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 7b3255f4..e4eceb83 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,13 +6,13 @@ "url": "https://github.com/jjackson" }, "metadata": { - "version": "0.13.318" + "version": "0.13.319" }, "plugins": [ { "name": "ace", "source": "./", - "version": "0.13.318", + "version": "0.13.319", "description": "AI Connect Engine — orchestrates the CRISPR-Connect lifecycle from idea through app building, Connect setup, LLO management, and closeout" } ] diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 8b878ddd..04128f70 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ace", - "version": "0.13.318", + "version": "0.13.319", "description": "AI Connect Engine — orchestrates the CRISPR-Connect lifecycle from idea through app building, Connect setup, LLO management, and closeout", "author": { "name": "Jonathan Jackson", diff --git a/VERSION b/VERSION index 5c79fb8a..2601050f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.13.318 +0.13.319 diff --git a/mcp/connect-server.ts b/mcp/connect-server.ts index 4b9b408c..50dd016b 100644 --- a/mcp/connect-server.ts +++ b/mcp/connect-server.ts @@ -506,6 +506,54 @@ server.tool('commcare_create_lookup_table', async (args) => runAtom(async () => (await commcareClient()).createLookupTable(args)) ); +server.tool('commcare_list_users', + 'List mobile workers (CommCareUser) in a CommCare HQ domain. GET /a//api/v0.5/user/ via Tastypie (API key auth). Supports standard Tastypie pagination (limit/offset) and group filter. Returns each user\'s id, username, basic profile, and the full user_data dict (including custom fields like cohort_id). Used by verifier to confirm cohort_id is set on the right FLWs.', + { + domain: z.string(), + limit: z.number().int().positive().optional(), + offset: z.number().int().nonnegative().optional(), + group: z.string().optional(), + }, + async (args) => runAtom(async () => (await commcareClient()).listUsers(args)) +); + +server.tool('commcare_get_user', + 'Fetch a single CommCare HQ mobile worker by id. GET /a//api/v0.5/user//. Returns the full record including user_data.', + { domain: z.string(), user_id: z.string() }, + async (args) => runAtom(async () => (await commcareClient()).getUser(args)) +); + +server.tool('commcare_update_user_field', + 'Set a single custom-user-data field on a mobile worker. Implemented as GET → mutate user_data → PUT (v0_5 CommCareUserResource exposes PUT but not PATCH, so we PUT the merged user_data). Pass value=null to clear the field. Used by per-FLW cohort_id assignment after Learn completion.', + { + domain: z.string(), + user_id: z.string(), + field_slug: z.string().describe('User-data field slug (e.g. "cohort_id").'), + value: z.union([z.string(), z.null()]).describe('New value, or null to clear.'), + }, + async (args) => runAtom(async () => (await commcareClient()).updateUserField(args)) +); + +server.tool('commcare_get_lookup_table_rows', + 'Get rows of a CommCare HQ lookup table. GET /a//api/v0.5/lookup_table_item/ via Tastypie (API key auth). Tastypie returns ALL rows in the domain (no querystring filter); this atom client-side filters by data_type_id resolved from the supplied tag or UUID. Returns each row\'s fields as a flat map (column → first field_value).', + { + domain: z.string(), + table_id_or_tag: z.string().describe('Either a 32-hex table UUID or the human-readable tag (e.g. "interview_schedule").'), + }, + async (args) => runAtom(async () => (await commcareClient()).getLookupTableRows(args)) +); + +server.tool('commcare_lookup_table_append_rows', + 'Append rows to a CommCare HQ lookup table. POST /a//api/v0.5/lookup_table_item/ once per row (Tastypie doesn\'t support list POST for this resource). Each row is a flat field_name→string-value map; HQ wraps it into its field_list shape internally. Used by the cohort-create skill to populate interview_schedule rows for a new cohort.', + { + domain: z.string(), + table_id_or_tag: z.string(), + rows: z.array(z.record(z.string())).describe('List of flat row maps: each {field_name: value}.'), + item_attributes: z.record(z.string()).optional(), + }, + async (args) => runAtom(async () => (await commcareClient()).appendLookupTableRows(args)) +); + server.tool('commcare_link_domains', 'Set up a linked-project-spaces relationship: upstream (master) → downstream. Required before linked-app push / linked content sync. POST /a//linked_domain/service/ via the jQuery-RMI protocol (corehq/util/jqueryrmi.py + corehq/apps/linked_domain/views.py:DomainLinkRMIView.create_domain_link). Caller must have access in both domains. Pro Edition is required for the LITE_RELEASE_MANAGEMENT privilege that backs linked spaces — without it, the call may succeed structurally but content-push operations downstream will fail.', { diff --git a/mcp/connect/backends/commcare.ts b/mcp/connect/backends/commcare.ts index 54110f2a..442ef815 100644 --- a/mcp/connect/backends/commcare.ts +++ b/mcp/connect/backends/commcare.ts @@ -111,6 +111,71 @@ export interface CreateLookupTableArgs { item_attributes?: string[]; } +export interface CommCareUser { + id: string; + /** Full username with @domain.commcarehq.org suffix. */ + username: string; + /** Just the local part of username. */ + first_name?: string; + last_name?: string; + email?: string; + phone_numbers?: string[]; + groups?: string[]; + user_data: Record; +} + +export interface ListUsersArgs { + domain: string; + /** Tastypie list pagination — defaults to HQ's max-per-page (usually 20). */ + limit?: number; + offset?: number; + /** Filter by group_id. */ + group?: string; +} + +export interface GetUserArgs { + domain: string; + /** User couch id (the `id` field in list responses). */ + user_id: string; +} + +export interface UpdateUserFieldArgs { + domain: string; + user_id: string; + /** Field slug to set (e.g. "cohort_id"). */ + field_slug: string; + /** Value to set; pass null to clear the field. */ + value: string | null; +} + +/** A single row in a lookup table. Flat-string values per column. */ +export interface LookupTableRow { + id: string; + data_type_id: string; + /** Flat map: field_name → string value. (Extracts the first field_list entry's field_value per column.) */ + fields: Record; + item_attributes: Record; +} + +export interface GetLookupTableRowsArgs { + domain: string; + /** Either the table UUID (data_type_id) or the table tag (name) — atom looks up by tag if no UUID syntax. */ + table_id_or_tag: string; +} + +export interface AppendLookupTableRowsArgs { + domain: string; + /** Either the table UUID (data_type_id) or the table tag (name). */ + table_id_or_tag: string; + /** + * Rows to append. Each row is `field_name → string value` (one value + * per column, no sub-properties). HQ auto-assigns sort_key. + */ + rows: Array>; + /** Optional per-row item_attributes (string-only map). */ + item_attributes?: Record; +} + export interface MakeBuildResult { build_id: string; version: number | null; @@ -721,6 +786,212 @@ export class CommCareBackend { }); } + /** + * Resolve a table UUID from either a UUID hex or a tag (name). + * UUIDs are 32 hex chars; anything else is treated as a tag. + */ + private async resolveTableId(domain: string, idOrTag: string): Promise { + if (/^[0-9a-f]{32}$/i.test(idOrTag)) return idOrTag; + const got = await this.getLookupTable({ domain, tag: idOrTag }); + if (!got.table) { + throw new Error( + `Lookup table tag "${idOrTag}" not found in domain ${domain}. Create it first via commcare_create_lookup_table.`, + ); + } + return got.table.id; + } + + /** + * List mobile workers (CommCareUser) in a domain. + * GET /a//api/v0.5/user/ with API-key auth. + */ + async listUsers(args: ListUsersArgs): Promise<{ users: CommCareUser[]; total: number }> { + return this.runWithSessionRetry(async (request) => { + const params = new URLSearchParams(); + if (args.limit !== undefined) params.set('limit', String(args.limit)); + if (args.offset !== undefined) params.set('offset', String(args.offset)); + if (args.group) params.set('group', args.group); + const qs = params.toString() ? `?${params.toString()}` : ''; + const path = `/a/${encodeURIComponent(args.domain)}/api/v0.5/user/${qs}`; + const res = await request.get(`${this.opts.baseUrl}${path}`, { + maxRedirects: 0, + headers: { Authorization: this.apiKeyAuthHeader('commcare_list_users') }, + }); + if (res.status() !== 200) { + throw new Error(`commcare_list_users GET ${path} returned ${res.status()}: ${(await res.text()).slice(0, 300)}`); + } + const parsed = JSON.parse(await res.text()) as { objects?: any[]; meta?: { total_count?: number } }; + const users = (parsed.objects ?? []).map((u) => ({ + id: u.id ?? u._id ?? '', + username: u.username ?? '', + first_name: u.first_name, + last_name: u.last_name, + email: u.email, + phone_numbers: u.phone_numbers, + groups: u.groups, + user_data: u.user_data ?? {}, + })); + return { users, total: parsed.meta?.total_count ?? users.length }; + }); + } + + /** + * Fetch one mobile worker by id. GET /a//api/v0.5/user//. + */ + async getUser(args: GetUserArgs): Promise<{ user: CommCareUser }> { + return this.runWithSessionRetry(async (request) => { + const path = `/a/${encodeURIComponent(args.domain)}/api/v0.5/user/${encodeURIComponent(args.user_id)}/`; + const res = await request.get(`${this.opts.baseUrl}${path}`, { + maxRedirects: 0, + headers: { Authorization: this.apiKeyAuthHeader('commcare_get_user') }, + }); + if (res.status() !== 200) { + throw new Error(`commcare_get_user GET ${path} returned ${res.status()}: ${(await res.text()).slice(0, 300)}`); + } + const u = JSON.parse(await res.text()) as any; + return { + user: { + id: u.id ?? u._id ?? args.user_id, + username: u.username ?? '', + first_name: u.first_name, + last_name: u.last_name, + email: u.email, + phone_numbers: u.phone_numbers, + groups: u.groups, + user_data: u.user_data ?? {}, + }, + }; + }); + } + + /** + * Set a single user_data field on a mobile worker. Implemented as GET + * → mutate user_data → PUT (v0_5 CommCareUserResource has PUT but not + * PATCH). To clear a field, pass value=null. + */ + async updateUserField(args: UpdateUserFieldArgs): Promise<{ user_id: string; field_slug: string; value: string | null }> { + return this.runWithSessionRetry(async (request) => { + const { user } = await this.getUser({ domain: args.domain, user_id: args.user_id }); + const path = `/a/${encodeURIComponent(args.domain)}/api/v0.5/user/${encodeURIComponent(args.user_id)}/`; + const next = { ...(user.user_data ?? {}) }; + if (args.value === null) delete next[args.field_slug]; + else next[args.field_slug] = args.value; + const res = await request.fetch(`${this.opts.baseUrl}${path}`, { + method: 'PUT', + data: JSON.stringify({ user_data: next }), + headers: { + 'Content-Type': 'application/json', + Authorization: this.apiKeyAuthHeader('commcare_update_user_field'), + }, + maxRedirects: 0, + }); + if (res.status() !== 200 && res.status() !== 202 && res.status() !== 204) { + throw new Error( + `commcare_update_user_field PUT ${path} returned ${res.status()}: ${(await res.text()).slice(0, 300)}`, + ); + } + return { user_id: args.user_id, field_slug: args.field_slug, value: args.value }; + }); + } + + /** + * Get rows of a lookup table. Tastypie LookupTableItemResource returns + * ALL rows in the domain (no querystring filter), so this atom + * client-side filters by `data_type_id`. + * + * Each row's `fields` is flattened: `{column_name: first_field_value}` + * (the team's Connect Interviews lookup tables don't use sub-properties + * or multi-value field_lists). + * + * Endpoint: GET /a//api/v0.5/lookup_table_item/ + * Auth: API key (Tastypie default for this resource). + */ + async getLookupTableRows(args: GetLookupTableRowsArgs): Promise<{ rows: LookupTableRow[] }> { + return this.runWithSessionRetry(async (request) => { + const tableId = await this.resolveTableId(args.domain, args.table_id_or_tag); + const path = `/a/${encodeURIComponent(args.domain)}/api/v0.5/lookup_table_item/`; + const res = await request.get(`${this.opts.baseUrl}${path}`, { + maxRedirects: 0, + headers: { Authorization: this.apiKeyAuthHeader('commcare_get_lookup_table_rows') }, + }); + if (res.status() !== 200) { + throw new Error( + `commcare_get_lookup_table_rows GET ${path} returned ${res.status()}: ${(await res.text()).slice(0, 300)}`, + ); + } + const parsed = JSON.parse(await res.text()) as { + objects?: Array<{ id?: string; data_type_id?: string; fields?: any; item_attributes?: any }>; + }; + const rows = (parsed.objects ?? []) + .filter((r) => r.data_type_id === tableId) + .map((r) => { + const flatFields: Record = {}; + for (const [col, container] of Object.entries(r.fields ?? {})) { + const list = (container as any)?.field_list ?? []; + flatFields[col] = list[0]?.field_value ?? list[0]?.value ?? ''; + } + return { + id: r.id ?? '', + data_type_id: r.data_type_id ?? tableId, + fields: flatFields, + item_attributes: (r.item_attributes ?? {}) as Record, + }; + }); + return { rows }; + }); + } + + /** + * Append rows to a lookup table. POSTs one row at a time to + * /a//api/v0.5/lookup_table_item/ — the Tastypie resource + * doesn't accept a list payload, only single-row POST. Returns the + * array of created row ids. + * + * For Connect Interviews `interview_schedule`, each row is a flat + * `{cohort_id, previous_interview, next_interview, frequency_days}` + * map. Empty string values are passed as-is (HQ stores them). + */ + async appendLookupTableRows(args: AppendLookupTableRowsArgs): Promise<{ row_ids: string[] }> { + return this.runWithSessionRetry(async (request) => { + const tableId = await this.resolveTableId(args.domain, args.table_id_or_tag); + const path = `/a/${encodeURIComponent(args.domain)}/api/v0.5/lookup_table_item/`; + const authHeader = this.apiKeyAuthHeader('commcare_lookup_table_append_rows'); + const ids: string[] = []; + for (const flatRow of args.rows) { + const fields: Record }> }> = {}; + for (const [col, val] of Object.entries(flatRow)) { + fields[col] = { field_list: [{ field_value: String(val), properties: {} }] }; + } + const body = { + data_type_id: tableId, + fields, + item_attributes: args.item_attributes ?? {}, + }; + const res = await request.post(`${this.opts.baseUrl}${path}`, { + data: JSON.stringify(body), + headers: { 'Content-Type': 'application/json', Authorization: authHeader }, + maxRedirects: 0, + }); + if (res.status() !== 201 && res.status() !== 200) { + throw new Error( + `commcare_lookup_table_append_rows POST ${path} for row ${JSON.stringify(flatRow)} returned ${res.status()}: ${(await res.text()).slice(0, 300)}`, + ); + } + // Tastypie returns the created row with id on 201 + Location header. + const location = res.headers()['location'] || ''; + const id = location.match(/\/lookup_table_item\/([0-9a-f]+)\/?$/i)?.[1]; + if (id) ids.push(id); + else { + try { + const parsed = JSON.parse(await res.text()); + if (parsed?.id) ids.push(parsed.id); + } catch { /* drop — caller can re-list */ } + } + } + return { row_ids: ids }; + }); + } + /** * POST /a//apps/save// with an empty body — CCHQ creates * a new versioned build doc and returns its `_id`. The CSRF token is read diff --git a/package.json b/package.json index 6bff9c19..c9d1cc06 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ace", - "version": "0.13.318", + "version": "0.13.319", "description": "AI Connect Engine - orchestrator for building Connect Opps using AI", "type": "module", "scripts": { diff --git a/scripts/probe-commcare-lookup-rows.ts b/scripts/probe-commcare-lookup-rows.ts new file mode 100644 index 00000000..483919eb --- /dev/null +++ b/scripts/probe-commcare-lookup-rows.ts @@ -0,0 +1,39 @@ +import 'dotenv/config'; +import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { PlaywrightSession } from '../mcp/connect/auth/playwright-session.js'; +import { CommCareBackend } from '../mcp/connect/backends/commcare.js'; +if (!process.env.ACE_HQ_USERNAME) { + for (const line of fs.readFileSync(path.join(os.homedir(), '.claude/plugins/data/ace-ace/.env'), 'utf8').split('\n')) { + const m = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/); + if (m) process.env[m[1]] = m[2].replace(/^"(.*)"$/, '$1'); + } +} +const DOMAIN = 'connect-ace-prod'; +const TAG = 'ace-interviews-probe-table'; +const COMMIT = process.argv.includes('--commit'); +const s = new PlaywrightSession({ + baseUrl: process.env.CONNECT_BASE_URL ?? 'https://connect.dimagi.com', + cchqBaseUrl: process.env.ACE_HQ_BASE_URL ?? 'https://www.commcarehq.org', + hqUsername: process.env.ACE_HQ_USERNAME!, hqPassword: process.env.ACE_HQ_PASSWORD!, +}); +try { + const backend = new CommCareBackend({ + baseUrl: process.env.ACE_HQ_BASE_URL ?? 'https://www.commcarehq.org', + session: s, hqUsername: process.env.ACE_HQ_USERNAME, hqApiKey: process.env.ACE_HQ_API_KEY, + }); + const before = await backend.getLookupTableRows({ domain: DOMAIN, table_id_or_tag: TAG }); + console.log(`Rows before: ${before.rows.length}`); + if (!COMMIT) { console.log('Dry-run; re-run with --commit'); process.exit(0); } + const created = await backend.appendLookupTableRows({ + domain: DOMAIN, table_id_or_tag: TAG, + rows: [ + { cohort_id: '1A', previous_interview: '', next_interview: 'te001', frequency_days: '2' }, + { cohort_id: '1A', previous_interview: 'te001', next_interview: 'te002', frequency_days: '2' }, + { cohort_id: '1A', previous_interview: 'te002', next_interview: 'te003', frequency_days: '9999' }, + ], + }); + console.log(`Created row_ids: ${JSON.stringify(created.row_ids)}`); + const after = await backend.getLookupTableRows({ domain: DOMAIN, table_id_or_tag: TAG }); + console.log(`Rows after: ${after.rows.length}`); + for (const r of after.rows) console.log(` ${r.id}: ${JSON.stringify(r.fields)}`); +} finally { await s.close().catch(()=>{}); }