feat(commcare): lookup_table get + create atoms

.claude-plugin/marketplace.json

@@ -6,13 +6,13 @@
     "url": "https://github.com/jjackson"
   },
   "metadata": {
-    "version": "0.13.317"
+    "version": "0.13.318"
   },
   "plugins": [
     {
       "name": "ace",
       "source": "./",
-      "version": "0.13.317",
+      "version": "0.13.318",
       "description": "AI Connect Engine — orchestrates the CRISPR-Connect lifecycle from idea through app building, Connect setup, LLO management, and closeout"
     }
   ]

.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "ace",
-  "version": "0.13.317",
+  "version": "0.13.318",
   "description": "AI Connect Engine — orchestrates the CRISPR-Connect lifecycle from idea through app building, Connect setup, LLO management, and closeout",
   "author": {
     "name": "Jonathan Jackson",

VERSION

@@ -1 +1 @@
-0.13.317
+0.13.318

mcp/connect-server.ts

@@ -90,7 +90,12 @@ async function getBackends(): Promise<{ rest: RestBackend; playwright: Playwrigh
     // CommCareBackend takes the session itself (not a bare APIRequestContext)
     // so each atom can pull a fresh request and recover from CCHQ-side
     // session expiry that the boot-time probe missed (0.13.8).
-    commcare = new CommCareBackend({ baseUrl: cchqBaseUrl, session });
+    commcare = new CommCareBackend({
+      baseUrl: cchqBaseUrl,
+      session,
+      hqUsername: process.env.ACE_HQ_USERNAME,
+      hqApiKey: process.env.ACE_HQ_API_KEY,
+    });
     return { rest, playwright };
   })();
   try { return await initPromise; }
@@ -477,6 +482,30 @@ server.tool('commcare_create_domain',
   async (args) => runAtom(async () => (await commcareClient()).createDomain(args))
 );
 
+server.tool('commcare_get_lookup_table',
+  'Fetch a CommCare HQ lookup table by tag (name). GET /a/<domain>/api/v0.5/lookup_table/ via Tastypie (session auth OK). Lists all tables in the domain and returns the one whose `tag` matches; returns `{table: null}` if not found. Use this to verify a lookup table exists before appending rows (see also commcare_lookup_table_append_rows, planned).',
+  {
+    domain: z.string(),
+    tag: z.string().describe('Lookup table name as the team uses it (e.g. "interview_schedule").'),
+  },
+  async (args) => runAtom(async () => (await commcareClient()).getLookupTable(args))
+);
+
+server.tool('commcare_create_lookup_table',
+  'Create a new CommCare HQ lookup table. POST /a/<domain>/api/v0.5/lookup_table/ via Tastypie. Body: {tag, is_global, fields: [{field_name, properties}], item_attributes}. Returns the new table\'s UUID hex id. Rejects with 400 if a table with the same tag already exists in the domain.',
+  {
+    domain: z.string(),
+    tag: z.string().describe('Name for the new table (e.g. "interview_schedule").'),
+    fields: z.array(z.object({
+      field_name: z.string(),
+      properties: z.array(z.string()).optional().describe('Sub-properties for this column. Empty/omitted for plain string columns.'),
+    })),
+    is_global: z.boolean().optional().describe('If true, table is shared across the domain (default false).'),
+    item_attributes: z.array(z.string()).optional(),
+  },
+  async (args) => runAtom(async () => (await commcareClient()).createLookupTable(args))
+);
+
 server.tool('commcare_link_domains',
   'Set up a linked-project-spaces relationship: upstream (master) → downstream. Required before linked-app push / linked content sync. POST /a/<upstream>/linked_domain/service/ via the jQuery-RMI protocol (corehq/util/jqueryrmi.py + corehq/apps/linked_domain/views.py:DomainLinkRMIView.create_domain_link). Caller must have access in both domains. Pro Edition is required for the LITE_RELEASE_MANAGEMENT privilege that backs linked spaces — without it, the call may succeed structurally but content-push operations downstream will fail.',
   {

mcp/connect/backends/commcare.ts

@@ -29,6 +29,15 @@ import { SessionExpiredError, summarizeServerErrorBody } from '../errors.js';
 export interface CommCareBackendOptions {
   baseUrl: string;
   session: PlaywrightSession;
+  /**
+   * HQ username + API key for endpoints that require `Authorization: ApiKey`
+   * auth (Tastypie resources without `allow_session_auth=True` — e.g.
+   * `lookup_table`, `lookup_table_item`, `user`). Optional; atoms that
+   * need it throw a typed error if these are missing at call time.
+   * Read from .env (ACE_HQ_USERNAME / ACE_HQ_API_KEY) by the server bootstrap.
+   */
+  hqUsername?: string;
+  hqApiKey?: string;
 }
 
 export interface CreateDomainArgs {
@@ -69,6 +78,39 @@ export interface MakeBuildArgs {
   comment?: string;
 }
 
+export interface LookupTableField {
+  /** Column name (e.g. "cohort_id", "next_interview"). */
+  field_name: string;
+  /** Sub-property names. Empty list [] for plain string columns. */
+  properties?: string[];
+}
+
+export interface LookupTable {
+  /** UUID hex (no dashes). */
+  id: string;
+  /** "Name" the team uses (e.g. "interview_schedule"). */
+  tag: string;
+  is_global: boolean;
+  fields: LookupTableField[];
+  item_attributes: string[];
+}
+
+export interface GetLookupTableArgs {
+  domain: string;
+  /** Lookup table name (e.g. "interview_schedule"). */
+  tag: string;
+}
+
+export interface CreateLookupTableArgs {
+  domain: string;
+  tag: string;
+  fields: LookupTableField[];
+  /** Default false. */
+  is_global?: boolean;
+  /** Default []. */
+  item_attributes?: string[];
+}
+
 export interface MakeBuildResult {
   build_id: string;
   version: number | null;
@@ -559,6 +601,126 @@ export class CommCareBackend {
     });
   }
 
+  /**
+   * Fetch a lookup table by tag (name). HQ's Tastypie LookupTableResource
+   * supports list-with-filters but not directly by tag, so this atom
+   * lists all tables in the domain and finds the one whose `tag` matches.
+   *
+   * Endpoint: GET /a/<domain>/api/v0.5/lookup_table/
+   * Auth: session via RequirePermissionAuthentication(edit_apps,
+   *   allow_session_auth=True) per corehq/apps/fixtures/resources/v0_1.py.
+   * Returns null if no table with that tag exists.
+   *
+   * Verified against /tmp/ace-refs/hq/corehq/apps/fixtures/resources/v0_1.py:111-244.
+   */
+  async getLookupTable(args: GetLookupTableArgs): Promise<{ table: LookupTable | null }> {
+    return this.runWithSessionRetry(async (request) => {
+      const path = `/a/${encodeURIComponent(args.domain)}/api/v0.5/lookup_table/`;
+      const res = await request.get(`${this.opts.baseUrl}${path}`, {
+        maxRedirects: 0,
+        headers: { Authorization: this.apiKeyAuthHeader('commcare_get_lookup_table') },
+      });
+      if (res.status() === 302) {
+        CommCareBackend.assertNotLoginRedirect(res, `commcare_get_lookup_table GET ${path}`);
+      }
+      if (res.status() !== 200) {
+        throw new Error(
+          `commcare_get_lookup_table GET ${path} returned ${res.status()}: ${(await res.text()).slice(0, 300)}`,
+        );
+      }
+      const parsed = JSON.parse(await res.text()) as {
+        objects?: Array<{ id?: string; tag?: string; is_global?: boolean; fields?: any[]; item_attributes?: string[] }>;
+      };
+      const match = (parsed.objects ?? []).find((t) => t.tag === args.tag);
+      if (!match || !match.id) {
+        return { table: null };
+      }
+      return {
+        table: {
+          id: match.id,
+          tag: match.tag ?? args.tag,
+          is_global: match.is_global ?? false,
+          fields: (match.fields ?? []).map((f) => ({
+            field_name: f.field_name ?? f.name,
+            properties: f.properties ?? [],
+          })),
+          item_attributes: match.item_attributes ?? [],
+        },
+      };
+    });
+  }
+
+  /**
+   * Create a new lookup table.
+   * POST /a/<domain>/api/v0.5/lookup_table/ with body
+   *   {tag, is_global, fields: [{field_name, properties}], item_attributes}
+   * Returns the new table's UUID (hex, no dashes).
+   *
+   * Raises if the tag is already taken (HQ's obj_create rejects with 400
+   * via `LookupTable.objects.domain_tag_exists`).
+   *
+   * Verified against
+   * /tmp/ace-refs/hq/corehq/apps/fixtures/resources/v0_1.py:192-204.
+   */
+  async createLookupTable(args: CreateLookupTableArgs): Promise<{ id: string; tag: string }> {
+    return this.runWithSessionRetry(async (request) => {
+      const path = `/a/${encodeURIComponent(args.domain)}/api/v0.5/lookup_table/`;
+      const authHeader = this.apiKeyAuthHeader('commcare_create_lookup_table');
+      const body = {
+        tag: args.tag,
+        is_global: args.is_global ?? false,
+        fields: args.fields.map((f) => ({
+          field_name: f.field_name,
+          properties: f.properties ?? [],
+        })),
+        item_attributes: args.item_attributes ?? [],
+      };
+      // API-key-authenticated Tastypie endpoints don't need CSRF — the
+      // CsrfViewMiddleware skips Authorization-header requests.
+      const res = await request.post(`${this.opts.baseUrl}${path}`, {
+        data: JSON.stringify(body),
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: authHeader,
+        },
+        maxRedirects: 0,
+      });
+      if (res.status() === 302) {
+        CommCareBackend.assertNotLoginRedirect(res, `commcare_create_lookup_table POST ${path}`);
+      }
+      if (res.status() !== 201 && res.status() !== 200) {
+        const text = await res.text();
+        // HQ's obj_create raises BadRequest("A lookup table with name <tag> already exists")
+        if (/already exists/i.test(text)) {
+          throw new Error(
+            `commcare_create_lookup_table: a lookup table with tag "${args.tag}" already exists in domain ${args.domain}.`,
+          );
+        }
+        throw new Error(
+          `commcare_create_lookup_table POST ${path} returned ${res.status()}: ${text.slice(0, 400)}`,
+        );
+      }
+      // Tastypie returns the created object on 201 with Location header
+      // pointing at the detail URI. Parse Location for the id.
+      const location = res.headers()['location'] || '';
+      const idFromLocation = location.match(/\/lookup_table\/([0-9a-f]+)\/?$/i)?.[1];
+      if (idFromLocation) {
+        return { id: idFromLocation, tag: args.tag };
+      }
+      // Fallback: parse response body
+      try {
+        const parsed = JSON.parse(await res.text());
+        if (parsed?.id) return { id: parsed.id, tag: args.tag };
+      } catch { /* fall through */ }
+      // Worst case: re-list and find by tag
+      const got = await this.getLookupTable({ domain: args.domain, tag: args.tag });
+      if (got.table) return { id: got.table.id, tag: args.tag };
+      throw new Error(
+        `commcare_create_lookup_table POST ${path} returned ${res.status()} but neither Location header nor body nor list-readback exposed the new table's id.`,
+      );
+    });
+  }
+
   /**
    * POST /a/<domain>/apps/save/<app_id>/ with an empty body — CCHQ creates
    * a new versioned build doc and returns its `_id`. The CSRF token is read
@@ -984,6 +1146,25 @@ export class CommCareBackend {
     return cookie?.value;
   }
 
+  /**
+   * Build the `Authorization: ApiKey <username>:<key>` header value used by
+   * Tastypie endpoints that require API key auth (LookupTableResource,
+   * LookupTableItemResource, and most CommCareUserResource calls).
+   * Throws a typed error if the credentials aren't configured.
+   */
+  private apiKeyAuthHeader(atomLabel: string): string {
+    const u = this.opts.hqUsername;
+    const k = this.opts.hqApiKey;
+    if (!u || !k) {
+      throw new Error(
+        `${atomLabel}: HQ API-key credentials not configured. ` +
+          `ACE_HQ_USERNAME + ACE_HQ_API_KEY must be set in the plugin .env. ` +
+          `Run /ace:doctor to verify.`,
+      );
+    }
+    return `ApiKey ${u}:${k}`;
+  }
+
   /**
    * List CommCare HQ applications in a domain.
    *

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ace",
-  "version": "0.13.317",
+  "version": "0.13.318",
   "description": "AI Connect Engine - orchestrator for building Connect Opps using AI",
   "type": "module",
   "scripts": {

scripts/probe-commcare-lookup-table.ts

@@ -0,0 +1,95 @@
+/**
+ * Probe: round-trip the new commcare_get_lookup_table +
+ * commcare_create_lookup_table atoms against the ACE-owned HQ master
+ * domain (ace-interviews-master). Creates the `interview_schedule`
+ * table that the Connect Interviews team uses to drive the bot's
+ * routing logic.
+ *
+ * Run from worktree root:
+ *   npx tsx scripts/probe-commcare-lookup-table.ts            # dry-run (peek)
+ *   npx tsx scripts/probe-commcare-lookup-table.ts --commit   # create the table
+ *
+ * Optional --table <name>     override table name (default interview_schedule)
+ * Optional --domain <slug>    override target domain (default ace-interviews-master)
+ */
+import 'dotenv/config';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { PlaywrightSession } from '../mcp/connect/auth/playwright-session.js';
+import { CommCareBackend } from '../mcp/connect/backends/commcare.js';
+
+const BASE_URL = process.env.CONNECT_BASE_URL ?? 'https://connect.dimagi.com';
+const CCHQ_BASE_URL = process.env.ACE_HQ_BASE_URL ?? 'https://www.commcarehq.org';
+const COMMIT = process.argv.includes('--commit');
+const tableIdx = process.argv.indexOf('--table');
+const TABLE = tableIdx > -1 ? process.argv[tableIdx + 1] : 'interview_schedule';
+const domainIdx = process.argv.indexOf('--domain');
+const DOMAIN = domainIdx > -1 ? process.argv[domainIdx + 1] : 'ace-interviews-master';
+
+if (!process.env.ACE_HQ_USERNAME) {
+  const envPath = path.join(os.homedir(), '.claude/plugins/data/ace-ace/.env');
+  if (fs.existsSync(envPath)) {
+    for (const line of fs.readFileSync(envPath, 'utf8').split('\n')) {
+      const m = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);
+      if (m) process.env[m[1]] = m[2].replace(/^"(.*)"$/, '$1');
+    }
+  }
+}
+
+console.log(`[probe-commcare-lookup-table]`);
+console.log(`  Domain:   ${DOMAIN}`);
+console.log(`  Table:    ${TABLE}`);
+console.log(`  Commit:   ${COMMIT ? 'YES — will create if missing' : 'no (read-only)'}`);
+console.log('');
+
+const session = new PlaywrightSession({
+  baseUrl: BASE_URL,
+  cchqBaseUrl: CCHQ_BASE_URL,
+  hqUsername: process.env.ACE_HQ_USERNAME!,
+  hqPassword: process.env.ACE_HQ_PASSWORD!,
+});
+
+try {
+  const backend = new CommCareBackend({
+    baseUrl: CCHQ_BASE_URL,
+    session,
+    hqUsername: process.env.ACE_HQ_USERNAME,
+    hqApiKey: process.env.ACE_HQ_API_KEY,
+  });
+
+  const before = await backend.getLookupTable({ domain: DOMAIN, tag: TABLE });
+  console.log('Before:', before.table ? `exists (id=${before.table.id})` : 'not found');
+
+  if (!COMMIT) {
+    console.log('[dry-run] Re-run with --commit to create if missing.');
+    process.exit(0);
+  }
+
+  if (before.table) {
+    console.log('Table already exists; nothing to create.');
+    process.exit(0);
+  }
+
+  // Standard Connect Interviews schedule shape from the team's tech doc:
+  //   cohort_id | previous_interview | next_interview | frequency_days
+  const result = await backend.createLookupTable({
+    domain: DOMAIN,
+    tag: TABLE,
+    fields: [
+      { field_name: 'cohort_id', properties: [] },
+      { field_name: 'previous_interview', properties: [] },
+      { field_name: 'next_interview', properties: [] },
+      { field_name: 'frequency_days', properties: [] },
+    ],
+    is_global: false,
+  });
+  console.log(`  ✓ created — id=${result.id}, tag=${result.tag}`);
+
+  const after = await backend.getLookupTable({ domain: DOMAIN, tag: TABLE });
+  console.log('After read-back:', after.table
+    ? `tag=${after.table.tag} fields=${after.table.fields.map((f) => f.field_name).join(',')}`
+    : 'NOT FOUND — round-trip broken');
+} finally {
+  await session.close().catch(() => {});
+}