feat(ui-automation): Add rs/1 runtime automation parity

[cameroncooke]

Adds an rs/1-oriented runtime UI automation contract on top of AXe. This introduces runtime snapshot refs with screen hashes, batch execution, wait predicates, stricter actionable-target handling, compact structured output schemas, and updated snapshots so agents can drive simulator UI with fewer unnecessary calls.

The branch was validated locally with npm run format:check, npm run lint, npm run typecheck, npm run build, and npm run test with 186 files and 2025 tests passing. During implementation the UI automation flows were also smoke-tested against Settings, Safari, Contacts, and AXe Playground components.

[pkg-pr-new]

Open in StackBlitz

npm i https://pkg.pr.new/xcodebuildmcp@415

commit: 04b0552

[cursor]

Cursor Bugbot has reviewed your changes and found 1 potential issue.

Bugbot Autofix prepared a fix for the issue found in the latest run.

• ✅ Fixed: Duplicated display-priority label sets and ordering logic
  • Extracted shared label sets (HIDDEN_LABELS, LOW_PRIORITY_LABELS) and common priority helper functions (isHiddenLabel, isLowPriorityLabel, isContentRichElement, isAlreadySelectedElement, compactText) into a new shared module (element-priority.ts) and updated both snapshot_ui.ts and structured-output-envelope.ts to use them, eliminating the duplication risk.

Or push these changes by commenting:

@cursor push b0521df217

Preview (b0521df217)

diff --git a/src/mcp/tools/ui-automation/shared/element-priority.ts b/src/mcp/tools/ui-automation/shared/element-priority.ts
new file mode 100644
--- /dev/null
+++ b/src/mcp/tools/ui-automation/shared/element-priority.ts
@@ -1,0 +1,45 @@
+export const HIDDEN_LABELS = new Set(['sheet grabber']);
+
+export const LOW_PRIORITY_LABELS = new Set([
+  'close',
+  'clear search',
+  'remove',
+  'delete',
+  'clear',
+  'c',
+  'ac',
+  '±',
+  '%',
+  '÷',
+  '×',
+  '-',
+  '+',
+  '=',
+]);
+
+export function compactText(value: string | undefined): string {
+  return (value ?? '').replace(/\s+/g, ' ').trim();
+}
+
+export function isHiddenLabel(label: string | undefined): boolean {
+  return HIDDEN_LABELS.has(compactText(label).toLowerCase());
+}
+
+export function isLowPriorityLabel(label: string | undefined): boolean {
+  return LOW_PRIORITY_LABELS.has(compactText(label).toLowerCase());
+}
+
+export function isContentRichElement(element: { label?: string; identifier?: string }): boolean {
+  const label = compactText(element.label);
+  const identifier = compactText(element.identifier);
+  return label.includes(',') || label.length >= 24 || /card$/i.test(identifier);
+}
+
+export function isAlreadySelectedElement(element: {
+  state?: { selected?: boolean };
+  value?: string;
+}): boolean {
+  return (
+    element.state?.selected === true || compactText(element.value).toLowerCase() === 'selected'
+  );
+}

diff --git a/src/mcp/tools/ui-automation/snapshot_ui.ts b/src/mcp/tools/ui-automation/snapshot_ui.ts
--- a/src/mcp/tools/ui-automation/snapshot_ui.ts
+++ b/src/mcp/tools/ui-automation/snapshot_ui.ts
@@ -27,6 +27,12 @@
   parseRuntimeSnapshotResponse,
   RuntimeSnapshotParseError,
 } from './shared/runtime-snapshot.ts';
+import {
+  isHiddenLabel,
+  isLowPriorityLabel,
+  isContentRichElement,
+  isAlreadySelectedElement,
+} from './shared/element-priority.ts';
 
 const snapshotUiSchema = z.object({
   simulatorId: z.uuid({ message: 'Invalid Simulator UUID format' }),
@@ -42,69 +48,19 @@
 
 const LOG_PREFIX = '[AXe]';
 
-const HIDDEN_TAP_NEXT_STEP_LABELS = new Set(['sheet grabber']);
-
-const LOW_PRIORITY_TAP_NEXT_STEP_LABELS = new Set([
-  'close',
-  'clear search',
-  'remove',
-  'delete',
-  'clear',
-  'c',
-  'ac',
-  '±',
-  '%',
-  '÷',
-  '×',
-  '-',
-  '+',
-  '=',
-]);
-
-function compactTapNextStepText(value: string | undefined): string {
-  return (value ?? '').replace(/\s+/g, ' ').trim();
-}
-
-function isHiddenTapNextStepElement(label: string | undefined): boolean {
-  return HIDDEN_TAP_NEXT_STEP_LABELS.has(compactTapNextStepText(label).toLowerCase());
-}
-
-function isLowPriorityTapNextStepElement(label: string | undefined): boolean {
-  return LOW_PRIORITY_TAP_NEXT_STEP_LABELS.has(compactTapNextStepText(label).toLowerCase());
-}
-
-function isContentRichTapNextStepElement(element: {
-  label?: string;
-  identifier?: string;
-}): boolean {
-  const label = compactTapNextStepText(element.label);
-  const identifier = compactTapNextStepText(element.identifier);
-  return label.includes(',') || label.length >= 24 || /card$/i.test(identifier);
-}
-
-function isAlreadySelectedTapNextStepElement(element: {
-  state?: { selected?: boolean };
-  value?: string;
-}): boolean {
-  return (
-    element.state?.selected === true ||
-    compactTapNextStepText(element.value).toLowerCase() === 'selected'
-  );
-}
-
 function getTapNextStepElementPriority(element: {
   label?: string;
   identifier?: string;
   state?: { selected?: boolean };
   value?: string;
 }): number {
-  if (isLowPriorityTapNextStepElement(element.label)) {
+  if (isLowPriorityLabel(element.label)) {
     return 90;
   }
-  if (isAlreadySelectedTapNextStepElement(element)) {
+  if (isAlreadySelectedElement(element)) {
     return 70;
   }
-  if (isContentRichTapNextStepElement(element)) {
+  if (isContentRichElement(element)) {
     return 0;
   }
   return 20;
@@ -212,7 +168,7 @@
           ({ element }) =>
             element.actions.includes('tap') &&
             !element.actions.includes('typeText') &&
-            !isHiddenTapNextStepElement(element.label),
+            !isHiddenLabel(element.label),
         )
         .sort((left, right) => {
           const priorityDelta =

diff --git a/src/utils/structured-output-envelope.ts b/src/utils/structured-output-envelope.ts
--- a/src/utils/structured-output-envelope.ts
+++ b/src/utils/structured-output-envelope.ts
@@ -6,6 +6,13 @@
   RuntimeSnapshotUnchangedV1,
   RuntimeSnapshotV1,
 } from '../types/ui-snapshot.ts';
+import {
+  isHiddenLabel,
+  isLowPriorityLabel,
+  isContentRichElement,
+  isAlreadySelectedElement,
+  compactText,
+} from '../mcp/tools/ui-automation/shared/element-priority.ts';
 
 type DomainResultData<TResult extends ToolDomainResult> = Omit<
   TResult,
@@ -38,56 +45,27 @@
   udid: string;
 };
 
-const HIDDEN_RUNTIME_TARGET_LABELS = new Set(['sheet grabber']);
-
-const LOW_PRIORITY_RUNTIME_TARGET_LABELS = new Set([
-  'sheet grabber',
-  'close',
-  'clear search',
-  'remove',
-  'delete',
-  'clear',
-  'c',
-  'ac',
-  '±',
-  '%',
-  '÷',
-  '×',
-  '-',
-  '+',
-  '=',
-]);
-
 function compactRuntimeSnapshotText(value: string | undefined): string {
-  return (value ?? '').replace(/\s+/g, ' ').replace(/\|/g, '/').trim();
+  return compactText(value).replace(/\|/g, '/');
 }
 
-function normalizedRuntimeSnapshotText(value: string | undefined): string {
-  return compactRuntimeSnapshotText(value).toLocaleLowerCase();
-}
-
 function isHiddenRuntimeTarget(element: RuntimeElementV1): boolean {
-  return HIDDEN_RUNTIME_TARGET_LABELS.has(normalizedRuntimeSnapshotText(element.label));
+  return isHiddenLabel(element.label);
 }
 
 function isLowPriorityRuntimeTarget(element: RuntimeElementV1): boolean {
-  return LOW_PRIORITY_RUNTIME_TARGET_LABELS.has(normalizedRuntimeSnapshotText(element.label));
+  return isLowPriorityLabel(element.label);
 }
 
 function isContentRichTapTarget(element: RuntimeElementV1): boolean {
   if (!element.actions.includes('tap')) {
     return false;
   }
-
-  const label = compactRuntimeSnapshotText(element.label);
-  const identifier = compactRuntimeSnapshotText(element.identifier);
-  return label.includes(',') || label.length >= 24 || /card$/i.test(identifier);
+  return isContentRichElement(element);
 }
 
 function isAlreadySelectedRuntimeTarget(element: RuntimeElementV1): boolean {
-  return (
-    element.state?.selected === true || normalizedRuntimeSnapshotText(element.value) === 'selected'
-  );
+  return isAlreadySelectedElement(element);
 }
 
 function getRuntimeTargetDisplayPriority(element: RuntimeElementV1): number {

_{You can send follow-ups to the cloud agent here.}

^{Reviewed by Cursor Bugbot for commit 04b0552. Configure here.}

[cursor]

Duplicated display-priority label sets and ordering logic

Low Severity

snapshot_ui.ts and structured-output-envelope.ts independently maintain identical hidden-label and low-priority-label Set constants (HIDDEN_TAP_NEXT_STEP_LABELS / HIDDEN_RUNTIME_TARGET_LABELS, LOW_PRIORITY_TAP_NEXT_STEP_LABELS / LOW_PRIORITY_RUNTIME_TARGET_LABELS) along with nearly identical priority-ranking, content-richness, and selected-state detection functions. If one copy is updated without the other, the next-step suggestion ordering and compact JSON target ordering will silently diverge.

Additional Locations (1)

• src/utils/structured-output-envelope.ts#L40-L121

 

^{Reviewed by Cursor Bugbot for commit 04b0552. Configure here.}