From eec8e268eb32c417e9c975d4f374f1ddedf8dbdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 21 May 2026 21:34:05 +0200 Subject: [PATCH 1/8] feat: add Maestro replay compatibility --- .../RunnerTests+CommandExecution.swift | 18 +- .../RunnerTests+Interaction.swift | 41 ++- .../RunnerTests+Models.swift | 1 + src/compat/__tests__/replay-input.test.ts | 4 +- .../maestro/__tests__/replay-flow.test.ts | 242 +++++++------ src/compat/maestro/command-mapper.ts | 157 ++------- src/compat/maestro/device-actions.ts | 233 ++----------- src/compat/maestro/flow-control.ts | 203 +++++++++++ src/compat/maestro/interactions.ts | 138 ++++---- src/compat/maestro/points.ts | 57 ++++ src/compat/maestro/run-script.ts | 136 ++++++++ src/compat/maestro/runtime-commands.ts | 7 + src/compat/maestro/support.ts | 2 +- src/compat/maestro/types.ts | 2 - src/core/dispatch-context.ts | 6 + src/core/dispatch.ts | 2 + src/core/interactor-types.ts | 9 +- src/core/interactors/apple.ts | 1 + src/daemon/__tests__/context.test.ts | 6 + src/daemon/context.ts | 2 + src/daemon/direct-ios-selector.ts | 1 + .../handlers/__tests__/interaction.test.ts | 37 ++ .../__tests__/session-replay-vars.test.ts | 260 ++++++++++++++ src/daemon/handlers/interaction-touch.ts | 7 +- .../session-replay-maestro-runtime.ts | 318 ++++++++++++++++++ src/daemon/handlers/session-replay-runtime.ts | 37 +- src/daemon/types.ts | 1 + src/platforms/ios/apps.ts | 12 +- src/platforms/ios/interactions.ts | 3 +- src/platforms/ios/runner-contract.ts | 1 + src/utils/__tests__/args.test.ts | 3 +- src/utils/command-schema.ts | 2 +- website/docs/docs/replay-e2e.md | 4 +- 33 files changed, 1408 insertions(+), 545 deletions(-) create mode 100644 src/compat/maestro/flow-control.ts create mode 100644 src/compat/maestro/points.ts create mode 100644 src/compat/maestro/run-script.ts create mode 100644 src/compat/maestro/runtime-commands.ts create mode 100644 src/daemon/handlers/session-replay-maestro-runtime.ts diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 17c1323da..b71f570cb 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -252,7 +252,12 @@ extension RunnerTests { ) case .tap: if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue { - let match = findElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue) + let match = findElement( + app: activeApp, + selectorKey: selectorKey, + selectorValue: selectorValue, + allowNonHittableFallback: command.allowNonHittableSelectorTap == true + ) if match.isAmbiguous { return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) } @@ -264,7 +269,14 @@ extension RunnerTests { var outcome = RunnerInteractionOutcome.performed let timing = measureGesture { withTemporaryScrollIdleTimeoutIfSupported(activeApp) { - outcome = activateElement(app: activeApp, element: element, action: "tap by selector") + if match.usedNonHittableFallback { + // Maestro compatibility: RN E2E backdoor controls can be 1x1 and + // reported non-hittable by XCTest, while Maestro still taps their + // resolved bounds. Keep this behind the explicit replay-only flag. + outcome = tapAt(app: activeApp, x: frame.midX, y: frame.midY) + } else { + outcome = activateElement(app: activeApp, element: element, action: "tap by selector") + } } } if let response = unsupportedResponse(for: outcome) { @@ -273,7 +285,7 @@ extension RunnerTests { return Response( ok: true, data: DataPayload( - message: "tapped", + message: match.usedNonHittableFallback ? "tapped via non-hittable coordinate fallback" : "tapped", gestureStartUptimeMs: timing.gestureStartUptimeMs, gestureEndUptimeMs: timing.gestureEndUptimeMs, x: touchFrame?.x, diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index efeae75a9..b912d6981 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -27,6 +27,7 @@ extension RunnerTests { struct SelectorElementMatch { let element: XCUIElement? let isAmbiguous: Bool + let usedNonHittableFallback: Bool } enum TextTypingRepairMode { @@ -177,10 +178,15 @@ extension RunnerTests { return element.exists ? element : nil } - func findElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> SelectorElementMatch { + func findElement( + app: XCUIApplication, + selectorKey: String, + selectorValue: String, + allowNonHittableFallback: Bool = false + ) -> SelectorElementMatch { let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines) guard !value.isEmpty else { - return SelectorElementMatch(element: nil, isAmbiguous: false) + return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } let predicate: NSPredicate switch selectorKey { @@ -193,21 +199,44 @@ extension RunnerTests { case "text": predicate = NSPredicate(format: "label ==[c] %@ OR identifier ==[c] %@ OR value ==[c] %@", value, value, value) default: - return SelectorElementMatch(element: nil, isAmbiguous: false) + return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } var matchedElement: XCUIElement? + var nonHittableElement: XCUIElement? let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex for element in matches where element.exists { - guard element.isHittable else { + if !element.isHittable { + if allowNonHittableFallback && hasTappableFrame(app: app, element: element) { + guard nonHittableElement == nil else { + return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false) + } + nonHittableElement = element + } continue } guard matchedElement == nil else { - return SelectorElementMatch(element: nil, isAmbiguous: true) + return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false) } matchedElement = element } - return SelectorElementMatch(element: matchedElement, isAmbiguous: false) + if let matchedElement { + return SelectorElementMatch(element: matchedElement, isAmbiguous: false, usedNonHittableFallback: false) + } + return SelectorElementMatch( + element: nonHittableElement, + isAmbiguous: false, + usedNonHittableFallback: nonHittableElement != nil + ) + } + + private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) -> Bool { + let frame = element.frame + if frame.isEmpty { + return false + } + let appFrame = app.frame + return appFrame.isEmpty || appFrame.intersects(frame) } func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response { diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index de5fa632f..13a295e69 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -39,6 +39,7 @@ struct Command: Codable { let text: String? let selectorKey: String? let selectorValue: String? + let allowNonHittableSelectorTap: Bool? let delayMs: Int? let textEntryMode: String? let clearFirst: Bool? diff --git a/src/compat/__tests__/replay-input.test.ts b/src/compat/__tests__/replay-input.test.ts index 3b69320d5..d7d31fdfb 100644 --- a/src/compat/__tests__/replay-input.test.ts +++ b/src/compat/__tests__/replay-input.test.ts @@ -19,7 +19,7 @@ test('parseReplayInput routes compat replay scripts through the selected parser' parsed.actions.map((action) => [action.command, action.positionals]), [ ['open', ['com.callstack.agentdevicelab']], - ['click', ['id="submit-order"']], + ['__maestroTapOn', ['id="submit-order"']], ], ); }); @@ -47,7 +47,7 @@ env: parsed.actions.map((action) => [action.command, action.positionals]), [ ['open', ['cli-app']], - ['click', ['id="shell-button"']], + ['__maestroTapOn', ['id="shell-button"']], ], ); }); diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index cf17308d5..f50fb6c3f 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -14,6 +14,8 @@ env: - launchApp - tapOn: id: home-open-form +- tapOn: + point: 20%,20% - doubleTapOn: id: release-notice delay: 150 @@ -37,6 +39,11 @@ env: start: 50%, 75% end: 50%, 35% duration: 300 +- swipe: + direction: LEFT +- scrollUntilVisible: + element: Discover + direction: UP - takeScreenshot: ./screens/form.png - hideKeyboard - stopApp @@ -47,34 +54,87 @@ env: parsed.actions.map((entry) => [entry.command, entry.positionals]), [ ['open', ['com.callstack.agentdevicelab']], - ['click', ['id="home-open-form"']], + ['__maestroTapOn', ['id="home-open-form"']], + ['__maestroTapPointPercent', ['20', '20']], ['click', ['id="release-notice"']], ['click', ['label="Agent Device Tester"']], ['open', ['exp://localhost:8082']], - ['click', ['label="Full name" || text="Full name" || id="Full name"']], + ['__maestroTapOn', ['label="Full name" || text="Full name" || id="Full name"']], ['type', ['Ada Lovelace']], ['wait', ['label="Checkout form"', '5000']], ['is', ['hidden', 'label="Missing banner"']], ['wait', ['id="submit-order"', '7000']], ['scroll', ['down']], ['scroll', ['down', '0.4']], + ['scroll', ['right']], + [ + '__maestroScrollUntilVisible', + ['label="Discover" || text="Discover" || id="Discover"', '5000', 'down'], + ], ['screenshot', ['./screens/form.png']], ['keyboard', ['dismiss']], ['close', ['com.callstack.agentdevicelab']], ], ); - assert.equal(parsed.actions[2]?.flags.doubleTap, true); - assert.equal(parsed.actions[2]?.flags.intervalMs, 150); - assert.equal(parsed.actions[3]?.flags.holdMs, 3000); + assert.equal(parsed.actions[3]?.flags.doubleTap, true); + assert.equal(parsed.actions[3]?.flags.intervalMs, 150); + assert.equal(parsed.actions[4]?.flags.holdMs, 3000); + assert.equal(parsed.actions[1]?.flags.allowNonHittableSelectorTap, true); + assert.equal(parsed.actions[6]?.flags?.allowNonHittableSelectorTap, undefined); +}); + +test('parseMaestroReplayFlow maps iOS openLink through the app id when available', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- openLink: exp://localhost:8082 +`, + { platform: 'ios' }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [['open', ['com.callstack.agentdevicelab', 'exp://localhost:8082']]], + ); +}); + +test('parseMaestroReplayFlow executes runScript and exposes output variables', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-runscript-')); + const scriptPath = path.join(root, 'setup.js'); + const flowPath = path.join(root, 'flow.yml'); + fs.writeFileSync( + scriptPath, + ` +var res = {body: '{"appviewDid":"did:plc:test"}'} +output.result = SERVER_PATH + ':' + json(res.body).appviewDid +`, + ); + + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runScript: + file: ./setup.js + env: + SERVER_PATH: local +- inputText: \${output.result} +`, + { sourcePath: flowPath }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [['type', ['local:did:plc:test']]], + ); }); test('parseMaestroReplayFlow rejects unsupported Maestro commands', () => { assert.throws( - () => parseMaestroReplayFlow('---\n- scrollUntilVisible: Save\n'), + () => parseMaestroReplayFlow('---\n- travelThroughTime: Save\n'), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /scrollUntilVisible/.test(error.message) && + /travelThroughTime/.test(error.message) && /issues\/558/.test(error.message) && /issues\/new/.test(error.message) && /line 2/.test(error.message), @@ -103,52 +163,7 @@ test('parseMaestroReplayFlow preserves selector state and absolute swipe command assert.deepEqual(parsed.actionLines, [3, 6]); }); -test('parseMaestroReplayFlow maps easy Maestro device and utility commands', () => { - const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab -env: - VIDEO_PATH: ./recordings/checkout.mp4 ---- -- setAirplaneMode: true -- setAirplaneMode: false -- setLocation: - latitude: 52.2297 - longitude: 21.0122 -- setOrientation: landscapeLeft -- setPermissions: - camera: allow - microphone: deny - photos: unset - location: always -- killApp -- killApp: com.callstack.other -- pasteText: hello there -- startRecording: - path: \${VIDEO_PATH} -- stopRecording -- assertTrue: true -`); - - assert.deepEqual( - parsed.actions.map((entry) => [entry.command, entry.positionals]), - [ - ['settings', ['airplane', 'on']], - ['settings', ['airplane', 'off']], - ['settings', ['location', 'set', '52.2297', '21.0122']], - ['rotate', ['landscape-left']], - ['settings', ['permission', 'grant', 'camera']], - ['settings', ['permission', 'deny', 'microphone']], - ['settings', ['permission', 'reset', 'photos']], - ['settings', ['permission', 'grant', 'location-always']], - ['close', ['com.callstack.agentdevicelab']], - ['close', ['com.callstack.other']], - ['type', ['hello there']], - ['record', ['start', './recordings/checkout.mp4']], - ['record', ['stop']], - ], - ); -}); - -test('parseMaestroReplayFlow rejects unsupported easy-mapping variants loudly', () => { +test('parseMaestroReplayFlow rejects deferred Maestro utility commands loudly', () => { assert.throws( () => parseMaestroReplayFlow('---\n- assertTrue: "${READY}"\n'), (error) => @@ -160,11 +175,11 @@ test('parseMaestroReplayFlow rejects unsupported easy-mapping variants loudly', ); assert.throws( - () => parseMaestroReplayFlow('---\n- setPermissions:\n camera: always\n'), + () => parseMaestroReplayFlow('---\n- setPermissions:\n camera: allow\n'), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /setPermissions state "always"/.test(error.message) && + /setPermissions/.test(error.message) && /issues\/558/.test(error.message) && /line 2/.test(error.message), ); @@ -196,12 +211,12 @@ test('parseMaestroReplayFlow reports top-level command lines around nested lists - runFlow: commands: - tapOn: Nested -- scrollUntilVisible: Save +- travelThroughTime: Save `), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /scrollUntilVisible/.test(error.message) && + /travelThroughTime/.test(error.message) && /line 6/.test(error.message), ); }); @@ -251,14 +266,14 @@ onFlowComplete: assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), [ - ['click', ['label="Before" || text="Before" || id="Before"']], - ['click', ['label="Nested" || text="Nested" || id="Nested"']], - ['click', ['id="child-repeat"']], - ['click', ['id="child-repeat"']], - ['click', ['label="iOS only" || text="iOS only" || id="iOS only"']], - ['click', ['label="Again" || text="Again" || id="Again"']], - ['click', ['label="Again" || text="Again" || id="Again"']], - ['click', ['label="After" || text="After" || id="After"']], + ['__maestroTapOn', ['label="Before" || text="Before" || id="Before"']], + ['__maestroTapOn', ['label="Nested" || text="Nested" || id="Nested"']], + ['__maestroTapOn', ['id="child-repeat"']], + ['__maestroTapOn', ['id="child-repeat"']], + ['__maestroTapOn', ['label="iOS only" || text="iOS only" || id="iOS only"']], + ['__maestroTapOn', ['label="Again" || text="Again" || id="Again"']], + ['__maestroTapOn', ['label="Again" || text="Again" || id="Again"']], + ['__maestroTapOn', ['label="After" || text="After" || id="After"']], ], ); }); @@ -279,57 +294,66 @@ test('parseMaestroReplayFlow skips platform-gated runFlow commands for other pla assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), - [['click', ['label="Shared" || text="Shared" || id="Shared"']]], + [['__maestroTapOn', ['label="Shared" || text="Shared" || id="Shared"']]], + ); +}); + +test('parseMaestroReplayFlow keeps visible-gated runFlow commands for runtime evaluation', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runFlow: + when: + visible: Continue + commands: + - tapOn: Continue +`, + { platform: 'ios' }, ); + + assert.equal(parsed.actions[0]?.command, '__maestroRunFlowWhen'); + assert.deepEqual(parsed.actions[0]?.positionals, [ + 'visible', + 'label="Continue" || text="Continue" || id="Continue"', + ]); + assert.deepEqual(parsed.actions[0]?.flags.batchSteps, [ + { + command: '__maestroTapOn', + positionals: ['label="Continue" || text="Continue" || id="Continue"'], + flags: {}, + }, + ]); }); -test('parseMaestroReplayFlow tolerates false launchApp reset options and rejects reset side effects', () => { +test('parseMaestroReplayFlow accepts launchApp reset options without state-reset side effects', () => { const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- - launchApp: - clearState: false - clearKeychain: false + clearState: true + clearKeychain: true + arguments: + "-EXDevMenuIsOnboardingFinished": true + launchArguments: + "-Example": "ignored" stopApp: true `); assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals, entry.flags]), - [['open', ['com.callstack.agentdevicelab'], { relaunch: true }]], - ); - - assert.throws( - () => - parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab ---- -- launchApp: - clearState: true -`), - (error) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /clearState: true/.test(error.message) && - /line 3/.test(error.message), + [ + [ + 'open', + ['com.callstack.agentdevicelab'], + { + relaunch: true, + launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true', '-Example', 'ignored'], + }, + ], + ], ); }); -test('parseMaestroReplayFlow rejects runtime-dependent flow control for now', () => { - assert.throws( - () => - parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab ---- -- runFlow: - when: - visible: Continue - commands: - - tapOn: Continue -`), - (error) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /when.visible/.test(error.message) && - /line 3/.test(error.message), - ); - +test('parseMaestroReplayFlow rejects unsupported runtime-dependent flow control', () => { assert.throws( () => parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab @@ -360,21 +384,21 @@ test('parseMaestroReplayFlow parses the test-app Maestro suite fixture', () => { parsed.actions.map((entry) => entry.command), [ 'wait', - 'click', + '__maestroTapOn', 'wait', - 'click', + '__maestroTapOn', 'type', - 'click', + '__maestroTapOn', 'type', - 'click', + '__maestroTapOn', 'wait', 'wait', 'scroll', - 'click', + '__maestroTapOn', 'wait', - 'click', + '__maestroTapOn', 'wait', - 'click', + '__maestroTapOn', 'wait', 'wait', ], diff --git a/src/compat/maestro/command-mapper.ts b/src/compat/maestro/command-mapper.ts index 76a3bc1b7..0d1f12c81 100644 --- a/src/compat/maestro/command-mapper.ts +++ b/src/compat/maestro/command-mapper.ts @@ -1,23 +1,13 @@ import type { SessionAction } from '../../daemon/types.ts'; import { AppError } from '../../utils/errors.ts'; -import { - convertAssertTrue, - convertKillApp, - convertLaunchApp, - convertSetAirplaneMode, - convertSetLocation, - convertSetOrientation, - convertSetPermissions, - convertStartRecording, - convertStopApp, - convertStopRecording, -} from './device-actions.ts'; +import { convertLaunchApp, convertStopApp } from './device-actions.ts'; import { convertDoubleTapOn, convertExtendedWaitUntil, convertLongPressOn, convertPressKey, convertScroll, + convertScrollUntilVisible, convertSwipe, convertTapOn, maestroSelector, @@ -25,17 +15,14 @@ import { } from './interactions.ts'; import { action, - assertOnlyKeys, - isPlainRecord, - normalizeCommandList, - normalizePlatformValue, - readEnvMap, readTimeoutMs, + requireAppId, requireStringValue, resolveMaestroString, unsupportedCommand, - unsupportedMaestroSyntax, } from './support.ts'; +import { convertRepeat, convertRunFlow } from './flow-control.ts'; +import { executeRunScript } from './run-script.ts'; import type { MaestroCommand, MaestroCommandMapperDeps, @@ -43,7 +30,6 @@ import type { MaestroParseContext, } from './types.ts'; -const MAX_REPEAT_EXPANSIONS = 100; type MaestroCommandHandler = (params: { value: unknown; config: MaestroFlowConfig; @@ -63,36 +49,33 @@ const MAP_COMMAND_HANDLERS: Record = { pasteText: ({ value, context, name }) => [ action('type', [resolveMaestroString(requireStringValue(name, value), context)]), ], - openLink: ({ value, context, name }) => [ - action('open', [resolveMaestroString(requireStringValue(name, value), context)]), - ], + openLink: ({ value, config, context, name }) => [convertOpenLink(value, config, context, name)], assertVisible: ({ value, context, name }) => [ action('wait', [maestroSelector(value, name, [], context), '5000']), ], assertNotVisible: ({ value, context, name }) => [ action('is', ['hidden', maestroSelector(value, name, [], context)]), ], - assertTrue: ({ value, context }) => convertAssertTrue(value, context), extendedWaitUntil: ({ value, context }) => convertExtendedWaitUntil(value, context), takeScreenshot: ({ value, context, name }) => [ action('screenshot', [resolveMaestroString(requireStringValue(name, value), context)]), ], scroll: ({ value }) => [convertScroll(value)], + scrollUntilVisible: ({ value, context }) => convertScrollUntilVisible(value, context), swipe: ({ value }) => [convertSwipe(value)], hideKeyboard: () => [action('keyboard', ['dismiss'])], pressKey: ({ value }) => [convertPressKey(value)], back: () => [action('back')], waitForAnimationToEnd: ({ value }) => [action('wait', [String(readTimeoutMs(value, 250))])], stopApp: ({ value, config, context }) => [convertStopApp(value, config, context)], - killApp: ({ value, config, context }) => [convertKillApp(value, config, context)], - setAirplaneMode: ({ value, context }) => [convertSetAirplaneMode(value, context)], - setLocation: ({ value, context }) => [convertSetLocation(value, context)], - setOrientation: ({ value, context }) => [convertSetOrientation(value, context)], - setPermissions: ({ value, context }) => convertSetPermissions(value, context), - startRecording: ({ value, context }) => [convertStartRecording(value, context)], - stopRecording: ({ value }) => [convertStopRecording(value)], - runFlow: ({ value, config, context, deps }) => convertRunFlow(value, config, context, deps), - repeat: ({ value, config, context, deps }) => convertRepeat(value, config, context, deps), + runScript: ({ value, context }) => { + executeRunScript(value, context); + return []; + }, + runFlow: ({ value, config, context, deps }) => + convertRunFlow(value, config, context, deps, convertCommandList), + repeat: ({ value, config, context, deps }) => + convertRepeat(value, config, context, deps, convertCommandList), }; const SCALAR_COMMAND_HANDLERS: Record< @@ -105,9 +88,6 @@ const SCALAR_COMMAND_HANDLERS: Record< back: () => [action('back')], waitForAnimationToEnd: () => [action('wait', ['250'])], stopApp: (config, context) => [convertStopApp(undefined, config, context)], - killApp: (config, context) => [convertKillApp(undefined, config, context)], - startRecording: () => [action('record', ['start'])], - stopRecording: () => [action('record', ['stop'])], }; export function convertMaestroCommandWithLine( @@ -156,63 +136,17 @@ function convertScalarCommand( return handler(config, context); } -function convertRunFlow( +function convertOpenLink( value: unknown, config: MaestroFlowConfig, context: MaestroParseContext, - deps: MaestroCommandMapperDeps, -): SessionAction[] { - if (typeof value === 'string') { - return deps.parseRunFlowFile(resolveMaestroString(value, context), context).actions; - } - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'runFlow expects a file path string or map.'); - } - assertOnlyKeys(value, 'runFlow', ['file', 'commands', 'env', 'when', 'label']); - if (!shouldRunFlow(value.when, context)) return []; - - const runContext = { - ...context, - env: { ...context.env, ...readEnvMap(value.env, 'runFlow.env'), ...context.envOverrides }, - }; - if (typeof value.file === 'string') { - return deps.parseRunFlowFile(resolveMaestroString(value.file, runContext), runContext).actions; - } - if (Array.isArray(value.commands)) { - return convertCommandList(normalizeCommandList(value.commands), config, runContext, deps); - } - throw new AppError('INVALID_ARGS', 'runFlow map requires either file or commands.'); -} - -function convertRepeat( - value: unknown, - config: MaestroFlowConfig, - context: MaestroParseContext, - deps: MaestroCommandMapperDeps, -): SessionAction[] { - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'repeat expects a map.'); - } - assertOnlyKeys(value, 'repeat', ['times', 'commands', 'while']); - if (value.while !== undefined) { - throw unsupportedMaestroSyntax( - 'Maestro repeat.while is not supported yet. Only deterministic repeat.times is supported.', - ); - } - const times = readRepeatTimes(value.times, context); - if (!Array.isArray(value.commands)) { - throw new AppError('INVALID_ARGS', 'repeat requires a commands list.'); - } - if (times > MAX_REPEAT_EXPANSIONS) { - throw new AppError( - 'INVALID_ARGS', - `repeat.times must be <= ${MAX_REPEAT_EXPANSIONS} for deterministic replay expansion.`, - ); + name: string, +): SessionAction { + const url = resolveMaestroString(requireStringValue(name, value), context); + if (context.platform === 'ios' && config.appId) { + return action('open', [resolveMaestroString(requireAppId(config, name), context), url]); } - const commands = normalizeCommandList(value.commands); - return Array.from({ length: times }).flatMap(() => - convertCommandList(commands, config, context, deps), - ); + return action('open', [url]); } function convertCommandList( @@ -225,50 +159,3 @@ function convertCommandList( convertMaestroCommandWithLine(command, config, index + 1, context, deps), ); } - -function shouldRunFlow(value: unknown, context: MaestroParseContext): boolean { - if (value === undefined || value === null) return true; - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'runFlow.when expects a map.'); - } - assertOnlyKeys(value, 'runFlow.when', ['platform', 'visible', 'notVisible', 'true']); - rejectUnsupportedCondition(value, 'visible', 'when.visible'); - rejectUnsupportedCondition(value, 'notVisible', 'when.notVisible'); - rejectUnsupportedCondition(value, 'true', 'when.true'); - if (value.platform === undefined) return true; - const platform = normalizePlatformValue(value.platform, 'runFlow.when.platform'); - if (!context.platform) { - throw new AppError( - 'INVALID_ARGS', - 'Maestro runFlow.when.platform requires replay to be run with --platform ios|android.', - ); - } - return platform === context.platform; -} - -function readRepeatTimes(value: unknown, context: MaestroParseContext): number { - const resolved = typeof value === 'string' ? resolveMaestroString(value, context) : value; - const numeric = - typeof resolved === 'number' - ? resolved - : typeof resolved === 'string' && /^\d+$/.test(resolved) - ? Number(resolved) - : undefined; - if (numeric === undefined || !Number.isInteger(numeric) || numeric < 0) { - throw new AppError( - 'INVALID_ARGS', - 'repeat.times must be a non-negative integer or ${VAR} resolving to one.', - ); - } - return numeric; -} - -function rejectUnsupportedCondition( - value: Record, - key: string, - label: string, -): void { - if (value[key] !== undefined) { - throw unsupportedMaestroSyntax(`Maestro ${label} is not supported yet.`); - } -} diff --git a/src/compat/maestro/device-actions.ts b/src/compat/maestro/device-actions.ts index 95e33db4d..8069b2fab 100644 --- a/src/compat/maestro/device-actions.ts +++ b/src/compat/maestro/device-actions.ts @@ -4,50 +4,11 @@ import { action, assertOnlyKeys, isPlainRecord, - normalizeToken, - readBooleanLiteral, requireAppId, resolveMaestroString, - resolveMaybeMaestroString, unsupportedMaestroSyntax, } from './support.ts'; -import type { MaestroFlowConfig, MaestroParseContext, PermissionCommand } from './types.ts'; - -const SUPPORTED_PERMISSION_TARGETS = new Set([ - 'accessibility', - 'calendar', - 'camera', - 'contacts', - 'contacts-limited', - 'input-monitoring', - 'location', - 'location-always', - 'media-library', - 'microphone', - 'motion', - 'notifications', - 'photos', - 'reminders', - 'screen-recording', - 'siri', -]); - -const BASIC_PERMISSION_STATES: Record = { - allow: 'grant', - grant: 'grant', - granted: 'grant', - deny: 'deny', - denied: 'deny', - reset: 'reset', - unset: 'reset', - revoke: 'reset', - revoked: 'reset', -}; - -const MODE_PERMISSION_STATES: Record = { - limited: { command: 'grant', mode: 'limited' }, - full: { command: 'grant', mode: 'full' }, -}; +import type { MaestroFlowConfig, MaestroParseContext } from './types.ts'; export function convertLaunchApp( value: unknown, @@ -70,16 +31,17 @@ export function convertLaunchApp( 'permissions', 'launchArguments', ]); - rejectTruthyLaunchOption(value, 'clearState'); - rejectTruthyLaunchOption(value, 'clearKeychain'); - rejectUnsupportedLaunchOption(value, 'arguments'); rejectUnsupportedLaunchOption(value, 'permissions'); - rejectUnsupportedLaunchOption(value, 'launchArguments'); const appId = resolveMaestroString( typeof value.appId === 'string' ? value.appId : requireAppId(config, 'launchApp'), context, ); - return action('open', [appId], { relaunch: value.stopApp === true }); + const launchArgs = readLaunchArgs(value, context); + const shouldRelaunch = value.stopApp === true || launchArgs.length > 0; + return action('open', [appId], { + relaunch: shouldRelaunch, + ...(launchArgs.length > 0 ? { launchArgs } : {}), + }); } export function convertStopApp( @@ -94,173 +56,32 @@ export function convertStopApp( throw new AppError('INVALID_ARGS', 'stopApp expects a string appId or no value.'); } -export function convertSetAirplaneMode( - value: unknown, - context: MaestroParseContext, -): SessionAction { - const enabled = readBooleanLiteral(resolveMaybeMaestroString(value, context), 'setAirplaneMode'); - return action('settings', ['airplane', enabled ? 'on' : 'off']); -} - -export function convertSetLocation(value: unknown, context: MaestroParseContext): SessionAction { - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'setLocation expects a map.'); - } - assertOnlyKeys(value, 'setLocation', ['latitude', 'longitude', 'lat', 'lon', 'lng']); - const latitude = readCoordinate(value.latitude ?? value.lat, 'setLocation.latitude', context); - const longitude = readCoordinate( - value.longitude ?? value.lon ?? value.lng, - 'setLocation.longitude', - context, - ); - return action('settings', ['location', 'set', latitude, longitude]); -} - -export function convertSetOrientation(value: unknown, context: MaestroParseContext): SessionAction { - const raw = resolveMaybeMaestroString(value, context); - if (typeof raw !== 'string') { - throw new AppError('INVALID_ARGS', 'setOrientation expects a string value.'); - } - const orientation = normalizeToken(raw); - switch (orientation) { - case 'portrait': - case 'landscape-left': - case 'landscape-right': - return action('rotate', [orientation]); - case 'portrait-upside-down': - case 'upside-down': - return action('rotate', ['portrait-upside-down']); - default: - throw unsupportedMaestroSyntax( - `Maestro setOrientation "${raw}" cannot be mapped to a supported rotate orientation.`, - ); - } +function readLaunchArgs(value: Record, context: MaestroParseContext): string[] { + return [ + ...readLaunchArgValue(value.arguments, 'launchApp.arguments', context), + ...readLaunchArgValue(value.launchArguments, 'launchApp.launchArguments', context), + ]; } -export function convertSetPermissions( - value: unknown, - context: MaestroParseContext, -): SessionAction[] { - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'setPermissions expects a map.'); +function readLaunchArgValue(value: unknown, name: string, context: MaestroParseContext): string[] { + if (value === undefined || value === null) return []; + if (typeof value === 'string') return [resolveMaestroString(value, context)]; + if (Array.isArray(value)) { + return value.map((entry, index) => readLaunchArgScalar(entry, `${name}[${index}]`, context)); } - return Object.entries(value).map(([rawTarget, rawState]) => { - const { target, command, mode } = readPermissionMapping(rawTarget, rawState, context); - return action('settings', ['permission', command, target, ...(mode ? [mode] : [])]); - }); -} - -export function convertKillApp( - value: unknown, - config: MaestroFlowConfig, - context: MaestroParseContext, -): SessionAction { - if (value === null || value === undefined) { - return action('close', [resolveMaestroString(requireAppId(config, 'killApp'), context)]); + if (isPlainRecord(value)) { + return Object.entries(value).flatMap(([key, entry]) => [ + resolveMaestroString(key, context), + readLaunchArgScalar(entry, `${name}.${key}`, context), + ]); } - if (typeof value === 'string') return action('close', [resolveMaestroString(value, context)]); - throw new AppError('INVALID_ARGS', 'killApp expects a string appId or no value.'); + throw new AppError('INVALID_ARGS', `${name} expects a string, list, or map.`); } -export function convertStartRecording(value: unknown, context: MaestroParseContext): SessionAction { - if (value === null || value === undefined) return action('record', ['start']); - if (typeof value === 'string') - return action('record', ['start', resolveMaestroString(value, context)]); - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'startRecording expects a string path, map, or no value.'); - } - assertOnlyKeys(value, 'startRecording', ['path', 'file']); - const rawPath = value.path ?? value.file; - if (rawPath === undefined) return action('record', ['start']); - if (typeof rawPath !== 'string') { - throw new AppError('INVALID_ARGS', 'startRecording path must be a string.'); - } - return action('record', ['start', resolveMaestroString(rawPath, context)]); -} - -export function convertStopRecording(value: unknown): SessionAction { - if (value !== null && value !== undefined) { - throw new AppError('INVALID_ARGS', 'stopRecording expects no value.'); - } - return action('record', ['stop']); -} - -export function convertAssertTrue(value: unknown, context: MaestroParseContext): SessionAction[] { - const resolved = resolveMaybeMaestroString(value, context); - if (resolved === true || (typeof resolved === 'string' && normalizeToken(resolved) === 'true')) { - return []; - } - if ( - resolved === false || - (typeof resolved === 'string' && normalizeToken(resolved) === 'false') - ) { - throw new AppError('INVALID_ARGS', 'Maestro assertTrue literal evaluated to false.'); - } - throw unsupportedMaestroSyntax('Only literal Maestro assertTrue true/false is supported.'); -} - -function readCoordinate(value: unknown, name: string, context: MaestroParseContext): string { - const resolved = resolveMaybeMaestroString(value, context); - const numeric = - typeof resolved === 'number' - ? resolved - : typeof resolved === 'string' && resolved.trim().length > 0 - ? Number(resolved) - : Number.NaN; - if (!Number.isFinite(numeric)) { - throw new AppError('INVALID_ARGS', `${name} must be a finite number.`); - } - return String(numeric); -} - -function readPermissionMapping( - rawTarget: string, - rawState: unknown, - context: MaestroParseContext, -): { target: string; command: PermissionCommand; mode?: string } { - let target = normalizeToken(rawTarget); - const resolvedState = resolveMaybeMaestroString(rawState, context); - if (typeof resolvedState !== 'string') { - throw new AppError('INVALID_ARGS', `setPermissions.${rawTarget} expects a string state.`); - } - const state = normalizeToken(resolvedState); - if (target === 'location' && state === 'always') target = 'location-always'; - - if (!SUPPORTED_PERMISSION_TARGETS.has(target)) { - throw unsupportedMaestroSyntax( - `Maestro setPermissions target "${rawTarget}" cannot be mapped to a supported settings permission target.`, - ); - } - - const basicCommand = BASIC_PERMISSION_STATES[state]; - if (basicCommand) return { target, command: basicCommand }; - - const modeMapping = MODE_PERMISSION_STATES[state]; - if (modeMapping) return { target, ...modeMapping }; - - const locationCommand = readLocationPermissionCommand(target, state); - if (locationCommand) return { target, command: locationCommand }; - - throw unsupportedMaestroSyntax( - `Maestro setPermissions state "${resolvedState}" cannot be mapped to grant, deny, or reset.`, - ); -} - -function readLocationPermissionCommand( - target: string, - state: string, -): PermissionCommand | undefined { - if (target === 'location-always' && state === 'always') return 'grant'; - if (target === 'location' && (state === 'while-in-use' || state === 'when-in-use')) { - return 'grant'; - } - return undefined; -} - -function rejectTruthyLaunchOption(value: Record, key: string): void { - if (value[key] === true) { - throw unsupportedMaestroSyntax(`Maestro launchApp ${key}: true is not supported yet.`); - } +function readLaunchArgScalar(value: unknown, name: string, context: MaestroParseContext): string { + if (typeof value === 'string') return resolveMaestroString(value, context); + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + throw new AppError('INVALID_ARGS', `${name} must be a string, number, or boolean.`); } function rejectUnsupportedLaunchOption(value: Record, key: string): void { diff --git a/src/compat/maestro/flow-control.ts b/src/compat/maestro/flow-control.ts new file mode 100644 index 000000000..d11e8af87 --- /dev/null +++ b/src/compat/maestro/flow-control.ts @@ -0,0 +1,203 @@ +import type { CommandFlags } from '../../core/dispatch.ts'; +import type { SessionAction } from '../../daemon/types.ts'; +import { AppError } from '../../utils/errors.ts'; +import { maestroSelector } from './interactions.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; +import { + action, + assertOnlyKeys, + isPlainRecord, + normalizeCommandList, + normalizePlatformValue, + readEnvMap, + resolveMaestroString, + unsupportedMaestroSyntax, +} from './support.ts'; +import type { + MaestroCommand, + MaestroCommandMapperDeps, + MaestroFlowConfig, + MaestroParseContext, +} from './types.ts'; + +const MAX_REPEAT_EXPANSIONS = 100; + +type ConvertCommandList = ( + commands: MaestroCommand[], + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, +) => SessionAction[]; + +export function convertRunFlow( + value: unknown, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (typeof value === 'string') { + return deps.parseRunFlowFile(resolveMaestroString(value, context), context).actions; + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'runFlow expects a file path string or map.'); + } + assertOnlyKeys(value, 'runFlow', ['file', 'commands', 'env', 'when', 'label']); + const condition = readRunFlowCondition(value.when, context); + if (!condition.shouldRun) return []; + + const runContext = { + ...context, + env: { ...context.env, ...readEnvMap(value.env, 'runFlow.env'), ...context.envOverrides }, + }; + const actions = readRunFlowActions(value, config, runContext, deps, convertCommandList); + return wrapRunFlowCondition(actions, condition); +} + +export function convertRepeat( + value: unknown, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'repeat expects a map.'); + } + assertOnlyKeys(value, 'repeat', ['times', 'commands', 'while']); + if (value.while !== undefined) { + throw unsupportedMaestroSyntax( + 'Maestro repeat.while is not supported yet. Only deterministic repeat.times is supported.', + ); + } + const times = readRepeatTimes(value.times, context); + if (!Array.isArray(value.commands)) { + throw new AppError('INVALID_ARGS', 'repeat requires a commands list.'); + } + if (times > MAX_REPEAT_EXPANSIONS) { + throw new AppError( + 'INVALID_ARGS', + `repeat.times must be <= ${MAX_REPEAT_EXPANSIONS} for deterministic replay expansion.`, + ); + } + const commands = normalizeCommandList(value.commands); + return Array.from({ length: times }).flatMap(() => + convertCommandList(commands, config, context, deps), + ); +} + +function readRunFlowActions( + value: Record, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (typeof value.file === 'string') { + return deps.parseRunFlowFile(resolveMaestroString(value.file, context), context).actions; + } + if (Array.isArray(value.commands)) { + return convertCommandList(normalizeCommandList(value.commands), config, context, deps); + } + throw new AppError('INVALID_ARGS', 'runFlow map requires either file or commands.'); +} + +type RunFlowCondition = { + shouldRun: boolean; + visibleSelector?: string; + notVisibleSelector?: string; +}; + +function readRunFlowCondition(value: unknown, context: MaestroParseContext): RunFlowCondition { + if (value === undefined || value === null) return { shouldRun: true }; + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'runFlow.when expects a map.'); + } + assertOnlyKeys(value, 'runFlow.when', ['platform', 'visible', 'notVisible', 'true']); + rejectUnsupportedCondition(value, 'true', 'when.true'); + if (value.platform !== undefined) { + const platform = normalizePlatformValue(value.platform, 'runFlow.when.platform'); + if (!context.platform) { + throw new AppError( + 'INVALID_ARGS', + 'Maestro runFlow.when.platform requires replay to be run with --platform ios|android.', + ); + } + if (platform !== context.platform) return { shouldRun: false }; + } + return { + shouldRun: true, + ...(value.visible !== undefined + ? { visibleSelector: maestroSelector(value.visible, 'runFlow.when.visible', [], context) } + : {}), + ...(value.notVisible !== undefined + ? { + notVisibleSelector: maestroSelector( + value.notVisible, + 'runFlow.when.notVisible', + [], + context, + ), + } + : {}), + }; +} + +function wrapRunFlowCondition( + actions: SessionAction[], + condition: RunFlowCondition, +): SessionAction[] { + if (!condition.visibleSelector && !condition.notVisibleSelector) return actions; + if (condition.visibleSelector && condition.notVisibleSelector) { + throw unsupportedMaestroSyntax( + 'Maestro runFlow.when cannot combine visible and notVisible yet.', + ); + } + return [ + action( + MAESTRO_RUNTIME_COMMAND.runFlowWhen, + condition.visibleSelector + ? ['visible', condition.visibleSelector] + : ['notVisible', condition.notVisibleSelector ?? ''], + { batchSteps: actions.map(sessionActionToBatchStep) }, + ), + ]; +} + +function sessionActionToBatchStep( + entry: SessionAction, +): NonNullable[number] { + return { + command: entry.command, + positionals: entry.positionals, + flags: entry.flags, + ...(entry.runtime !== undefined ? { runtime: entry.runtime } : {}), + }; +} + +function readRepeatTimes(value: unknown, context: MaestroParseContext): number { + const resolved = typeof value === 'string' ? resolveMaestroString(value, context) : value; + const numeric = + typeof resolved === 'number' + ? resolved + : typeof resolved === 'string' && /^\d+$/.test(resolved) + ? Number(resolved) + : undefined; + if (numeric === undefined || !Number.isInteger(numeric) || numeric < 0) { + throw new AppError( + 'INVALID_ARGS', + 'repeat.times must be a non-negative integer or ${VAR} resolving to one.', + ); + } + return numeric; +} + +function rejectUnsupportedCondition( + value: Record, + key: string, + label: string, +): void { + if (value[key] !== undefined) { + throw unsupportedMaestroSyntax(`Maestro ${label} is not supported yet.`); + } +} diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index c61271606..8fa5b1b13 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -9,12 +9,30 @@ import { resolveMaestroString, unsupportedMaestroSyntax, } from './support.ts'; +import { + parseAbsolutePoint, + parseMaestroPoint, + readScrollPositionalsFromPercentSwipe, +} from './points.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import type { MaestroParseContext } from './types.ts'; export function convertTapOn(value: unknown, context: MaestroParseContext): SessionAction { + if (typeof value === 'string') { + return action(MAESTRO_RUNTIME_COMMAND.tapOn, [ + visibleTextSelector(resolveMaestroString(value, context)), + ]); + } if (isPlainRecord(value) && typeof value.point === 'string') { assertOnlyKeys(value, 'tapOn', ['point', 'repeat', 'delay']); - const point = parsePoint(value.point); + const point = parseMaestroPoint(value.point); + if (point.kind === 'percent') { + return action( + MAESTRO_RUNTIME_COMMAND.tapPointPercent, + [String(point.x), String(point.y)], + tapFlags(value), + ); + } return action('click', [String(point.x), String(point.y)], tapFlags(value)); } if (isPlainRecord(value)) { @@ -30,16 +48,16 @@ export function convertTapOn(value: unknown, context: MaestroParseContext): Sess ]); } return action( - 'click', + MAESTRO_RUNTIME_COMMAND.tapOn, [maestroSelector(value, 'tapOn', ['repeat', 'delay', 'optional', 'label'], context)], - tapFlags(value), + { ...tapFlags(value), allowNonHittableSelectorTap: true }, ); } export function convertDoubleTapOn(value: unknown, context: MaestroParseContext): SessionAction { if (isPlainRecord(value) && typeof value.point === 'string') { assertOnlyKeys(value, 'doubleTapOn', ['point', 'delay']); - const point = parsePoint(value.point); + const point = parseAbsolutePoint(value.point); return action('click', [String(point.x), String(point.y)], doubleTapFlags(value)); } if (isPlainRecord(value)) { @@ -55,7 +73,7 @@ export function convertDoubleTapOn(value: unknown, context: MaestroParseContext) export function convertLongPressOn(value: unknown, context: MaestroParseContext): SessionAction { if (isPlainRecord(value) && typeof value.point === 'string') { assertOnlyKeys(value, 'longPressOn', ['point']); - const point = parsePoint(value.point); + const point = parseAbsolutePoint(value.point); return action('longpress', [String(point.x), String(point.y), '3000']); } if (isPlainRecord(value)) { @@ -105,16 +123,45 @@ export function convertScroll(value: unknown): SessionAction { return action('scroll', ['down']); } +export function convertScrollUntilVisible( + value: unknown, + context: MaestroParseContext, +): SessionAction[] { + if (typeof value === 'string') { + return [ + action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [ + visibleTextSelector(resolveMaestroString(value, context)), + '5000', + 'down', + ]), + ]; + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'scrollUntilVisible expects a string or map.'); + } + assertOnlyKeys(value, 'scrollUntilVisible', ['element', 'direction', 'timeout']); + const selector = maestroSelector(value.element, 'scrollUntilVisible.element', [], context); + const direction = + typeof value.direction === 'string' + ? readScrollPositionalsFromDirectionSwipe(value.direction)[0] + : 'down'; + const timeoutMs = String(readTimeoutMs(value, 5000)); + return [action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [selector, timeoutMs, direction])]; +} + export function convertSwipe(value: unknown): SessionAction { if (!isPlainRecord(value)) { throw new AppError('INVALID_ARGS', 'swipe expects a map.'); } - assertOnlyKeys(value, 'swipe', ['start', 'end', 'duration']); + assertOnlyKeys(value, 'swipe', ['start', 'end', 'direction', 'duration']); + if (typeof value.direction === 'string') { + return action('scroll', readScrollPositionalsFromDirectionSwipe(value.direction)); + } if (typeof value.start !== 'string' || typeof value.end !== 'string') { throw unsupportedMaestroSyntax('Only Maestro swipe start/end coordinates are supported.'); } - const start = parseSwipePoint(value.start); - const end = parseSwipePoint(value.end); + const start = parseMaestroPoint(value.start); + const end = parseMaestroPoint(value.end); const durationMs = typeof value.duration === 'number' && Number.isFinite(value.duration) ? String(Math.max(16, Math.floor(value.duration))) @@ -136,10 +183,25 @@ export function convertSwipe(value: unknown): SessionAction { ); } +function readScrollPositionalsFromDirectionSwipe(direction: string): string[] { + switch (direction.toLowerCase()) { + case 'up': + return ['down']; + case 'down': + return ['up']; + case 'left': + return ['right']; + case 'right': + return ['left']; + default: + throw unsupportedMaestroSyntax('Maestro swipe direction must be UP, DOWN, LEFT, or RIGHT.'); + } +} + export function convertPressKey(value: unknown): SessionAction { const key = requireStringValue('pressKey', value).toLowerCase(); if (key === 'back') return action('back'); - if (key === 'enter' || key === 'return') return action('press', ['return']); + if (key === 'enter' || key === 'return') return action(MAESTRO_RUNTIME_COMMAND.pressEnter); if (key === 'home') return action('home'); throw unsupportedMaestroSyntax(`Maestro pressKey "${key}" is not supported yet.`); } @@ -195,6 +257,9 @@ function tapFlags(value: unknown): SessionAction['flags'] | undefined { if (typeof value.delay === 'number' && Number.isInteger(value.delay) && value.delay >= 0) { flags.intervalMs = value.delay; } + if (value.optional === true) { + flags.maestroOptional = true; + } return Object.keys(flags).length > 0 ? flags : undefined; } @@ -205,58 +270,3 @@ function doubleTapFlags(value: unknown): SessionAction['flags'] { } return flags; } - -function parsePoint(value: string): { x: number; y: number } { - const match = value.match(/^(\d+),(\d+)$/); - if (!match) { - throw unsupportedMaestroSyntax( - 'Only absolute Maestro point selectors like "100,200" are supported.', - ); - } - return { x: Number(match[1]), y: Number(match[2]) }; -} - -type SwipePoint = - | { - kind: 'absolute'; - x: number; - y: number; - } - | { - kind: 'percent'; - x: number; - y: number; - }; - -function parseSwipePoint(value: string): SwipePoint { - const absolute = value.match(/^\s*(\d+)\s*,\s*(\d+)\s*$/); - if (absolute) { - return { kind: 'absolute', x: Number(absolute[1]), y: Number(absolute[2]) }; - } - const percent = value.match(/^\s*(\d+(?:\.\d+)?)%\s*,\s*(\d+(?:\.\d+)?)%\s*$/); - if (percent) { - return { kind: 'percent', x: Number(percent[1]), y: Number(percent[2]) }; - } - throw unsupportedMaestroSyntax( - 'Only Maestro swipe coordinates like "100,200" or "50%,75%" are supported.', - ); -} - -function readScrollPositionalsFromPercentSwipe( - start: Extract, - end: Extract, -): string[] { - const deltaX = end.x - start.x; - const deltaY = end.y - start.y; - if (Math.abs(deltaX) === 0 && Math.abs(deltaY) === 0) { - throw new AppError('INVALID_ARGS', 'swipe start and end cannot be the same point.'); - } - const vertical = Math.abs(deltaY) >= Math.abs(deltaX); - const direction = vertical ? (deltaY < 0 ? 'down' : 'up') : deltaX < 0 ? 'right' : 'left'; - const amount = Math.min(1, Math.max(0.01, Math.abs(vertical ? deltaY : deltaX) / 100)); - return [direction, formatAmount(amount)]; -} - -function formatAmount(value: number): string { - return value.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); -} diff --git a/src/compat/maestro/points.ts b/src/compat/maestro/points.ts new file mode 100644 index 000000000..c0e25ffb4 --- /dev/null +++ b/src/compat/maestro/points.ts @@ -0,0 +1,57 @@ +import { AppError } from '../../utils/errors.ts'; +import { unsupportedMaestroSyntax } from './support.ts'; + +export type MaestroPoint = + | { + kind: 'absolute'; + x: number; + y: number; + } + | { + kind: 'percent'; + x: number; + y: number; + }; + +export function parseAbsolutePoint(value: string): { x: number; y: number } { + const match = value.match(/^(\d+),(\d+)$/); + if (!match) { + throw unsupportedMaestroSyntax( + 'Only absolute Maestro point selectors like "100,200" are supported.', + ); + } + return { x: Number(match[1]), y: Number(match[2]) }; +} + +export function parseMaestroPoint(value: string): MaestroPoint { + const absolute = value.match(/^\s*(\d+)\s*,\s*(\d+)\s*$/); + if (absolute) { + return { kind: 'absolute', x: Number(absolute[1]), y: Number(absolute[2]) }; + } + const percent = value.match(/^\s*(\d+(?:\.\d+)?)%\s*,\s*(\d+(?:\.\d+)?)%\s*$/); + if (percent) { + return { kind: 'percent', x: Number(percent[1]), y: Number(percent[2]) }; + } + throw unsupportedMaestroSyntax( + 'Only Maestro swipe coordinates like "100,200" or "50%,75%" are supported.', + ); +} + +export function readScrollPositionalsFromPercentSwipe( + start: Extract, + end: Extract, +): string[] { + const deltaX = end.x - start.x; + const deltaY = end.y - start.y; + if (Math.abs(deltaX) === 0 && Math.abs(deltaY) === 0) { + throw new AppError('INVALID_ARGS', 'swipe start and end cannot be the same point.'); + } + const vertical = Math.abs(deltaY) >= Math.abs(deltaX); + const direction = vertical ? (deltaY < 0 ? 'down' : 'up') : deltaX < 0 ? 'right' : 'left'; + const amount = Math.min(1, Math.max(0.01, Math.abs(vertical ? deltaY : deltaX) / 100)); + return [direction, formatAmount(amount)]; +} + +function formatAmount(value: number): string { + return value.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); +} diff --git a/src/compat/maestro/run-script.ts b/src/compat/maestro/run-script.ts new file mode 100644 index 000000000..8f810aa2b --- /dev/null +++ b/src/compat/maestro/run-script.ts @@ -0,0 +1,136 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import vm from 'node:vm'; +import { AppError } from '../../utils/errors.ts'; +import { runCmdSync } from '../../utils/exec.ts'; +import { + assertOnlyKeys, + isPlainRecord, + readEnvMap, + requireStringValue, + resolveMaestroString, +} from './support.ts'; +import type { MaestroParseContext } from './types.ts'; + +type HttpResponse = { + status: number; + body: string; + headers: Record; +}; + +const HTTP_REQUEST_SCRIPT = ` +const fs = require('node:fs'); +const input = JSON.parse(fs.readFileSync(0, 'utf8')); +fetch(input.url, { + method: input.method, + headers: input.headers, + body: input.body, +}).then(async response => { + process.stdout.write(JSON.stringify({ + status: response.status, + body: await response.text(), + headers: Object.fromEntries(response.headers.entries()), + })); +}).catch(error => { + console.error(error && error.stack ? error.stack : String(error)); + process.exit(1); +}); +`; + +export function executeRunScript(value: unknown, context: MaestroParseContext): void { + const scriptConfig = readRunScriptConfig(value, context); + const scriptPath = resolveRunScriptPath(scriptConfig.file, context); + const script = fs.readFileSync(scriptPath, 'utf8'); + const output: Record = {}; + const scriptEnv = { + ...context.env, + ...scriptConfig.env, + ...context.envOverrides, + }; + + try { + vm.runInNewContext(script, buildScriptGlobals(scriptEnv, output), { + filename: scriptPath, + timeout: 30_000, + }); + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript failed for ${scriptPath}: ${error instanceof Error ? error.message : String(error)}`, + { scriptPath }, + error instanceof Error ? error : undefined, + ); + } + + for (const [key, rawValue] of Object.entries(output)) { + context.env[`output.${key}`] = stringifyOutputValue(rawValue); + } +} + +function readRunScriptConfig( + value: unknown, + context: MaestroParseContext, +): { file: string; env: Record } { + if (typeof value === 'string') { + return { file: resolveMaestroString(value, context), env: {} }; + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'runScript expects a file path string or map.'); + } + assertOnlyKeys(value, 'runScript', ['file', 'env']); + const file = resolveMaestroString(requireStringValue('runScript.file', value.file), context); + const rawEnv = readEnvMap(value.env, 'runScript.env'); + const env = Object.fromEntries( + Object.entries(rawEnv).map(([key, envValue]) => [key, resolveMaestroString(envValue, context)]), + ); + return { file, env }; +} + +function resolveRunScriptPath(filePath: string, context: MaestroParseContext): string { + if (path.isAbsolute(filePath)) return filePath; + if (!context.baseDir) { + throw new AppError( + 'INVALID_ARGS', + 'runScript file paths require replay input to have a source path.', + ); + } + return path.resolve(context.baseDir, filePath); +} + +function buildScriptGlobals( + env: Record, + output: Record, +): vm.Context { + return { + ...env, + output, + json: (value: string) => JSON.parse(value) as unknown, + http: { + post: (url: string, options?: { headers?: Record; body?: string }) => + runHttpRequestSync('POST', url, options), + }, + }; +} + +function runHttpRequestSync( + method: string, + url: string, + options?: { headers?: Record; body?: string }, +): HttpResponse { + const result = runCmdSync(process.execPath, ['-e', HTTP_REQUEST_SCRIPT], { + stdin: JSON.stringify({ + method, + url, + headers: options?.headers ?? {}, + body: options?.body ?? '', + }), + timeoutMs: 30_000, + }); + return JSON.parse(result.stdout) as HttpResponse; +} + +function stringifyOutputValue(value: unknown): string { + if (typeof value === 'string') return value; + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + return JSON.stringify(value); +} diff --git a/src/compat/maestro/runtime-commands.ts b/src/compat/maestro/runtime-commands.ts new file mode 100644 index 000000000..02b816a4c --- /dev/null +++ b/src/compat/maestro/runtime-commands.ts @@ -0,0 +1,7 @@ +export const MAESTRO_RUNTIME_COMMAND = { + pressEnter: '__maestroPressEnter', + runFlowWhen: '__maestroRunFlowWhen', + scrollUntilVisible: '__maestroScrollUntilVisible', + tapOn: '__maestroTapOn', + tapPointPercent: '__maestroTapPointPercent', +} as const; diff --git a/src/compat/maestro/support.ts b/src/compat/maestro/support.ts index 3bd998faf..591e23ecb 100644 --- a/src/compat/maestro/support.ts +++ b/src/compat/maestro/support.ts @@ -113,7 +113,7 @@ export function requireStringValue(command: string, value: unknown): string { } export function resolveMaestroString(value: string, context: MaestroParseContext): string { - return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (match, key: string) => { + return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_.]*)\}/g, (match, key: string) => { return Object.prototype.hasOwnProperty.call(context.env, key) ? context.env[key] : match; }); } diff --git a/src/compat/maestro/types.ts b/src/compat/maestro/types.ts index 81012d5e7..8ea39be92 100644 --- a/src/compat/maestro/types.ts +++ b/src/compat/maestro/types.ts @@ -31,5 +31,3 @@ export type MaestroParseContext = { export type MaestroCommandMapperDeps = { parseRunFlowFile(filePath: string, context: MaestroParseContext): MaestroReplayFlow; }; - -export type PermissionCommand = 'grant' | 'deny' | 'reset'; diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index 37387f42b..b6e5712ec 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -12,7 +12,10 @@ export type BatchStep = { export type CommandFlags = Omit & { batchSteps?: BatchStep[]; + launchArgs?: string[]; replayBackend?: string; + allowNonHittableSelectorTap?: boolean; + maestroOptional?: boolean; }; export type DispatchContext = ScreenshotDispatchFlags & { @@ -20,6 +23,7 @@ export type DispatchContext = ScreenshotDispatchFlags & { appBundleId?: string; activity?: string; launchConsole?: string; + launchArgs?: string[]; verbose?: boolean; logPath?: string; traceLogPath?: string; @@ -40,9 +44,11 @@ export type DispatchContext = ScreenshotDispatchFlags & { pauseMs?: number; pattern?: 'one-way' | 'ping-pong'; surface?: SessionSurface; + allowNonHittableSelectorTap?: boolean; directElementSelector?: { key: 'id' | 'label' | 'text' | 'value'; value: string; raw: string; + allowNonHittableTap?: boolean; }; }; diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index f28a3c6e8..b5b84b3ec 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -217,6 +217,7 @@ async function handleOpenCommand( await interactor.open(app, { activity: context?.activity, appBundleId: context?.appBundleId, + launchArgs: context?.launchArgs, url, }); return { app, url, ...successText(`Opened: ${app}`) }; @@ -228,6 +229,7 @@ async function handleOpenCommand( activity: context?.activity, appBundleId: context?.appBundleId, launchConsole, + launchArgs: context?.launchArgs, }); return { app, ...(launchConsole ? { launchConsole } : {}), ...successText(`Opened: ${app}`) }; } diff --git a/src/core/interactor-types.ts b/src/core/interactor-types.ts index 01653a5e9..c87d19ab8 100644 --- a/src/core/interactor-types.ts +++ b/src/core/interactor-types.ts @@ -29,6 +29,7 @@ export type ScreenshotOptions = { export type ElementSelectorTapOptions = { key: 'id' | 'label' | 'text' | 'value'; value: string; + allowNonHittableTap?: boolean; }; export type SnapshotOptions = BaseSnapshotOptions & { @@ -44,7 +45,13 @@ export type SnapshotResult = Omit & export type Interactor = { open( app: string, - options?: { activity?: string; appBundleId?: string; launchConsole?: string; url?: string }, + options?: { + activity?: string; + appBundleId?: string; + launchConsole?: string; + launchArgs?: string[]; + url?: string; + }, ): Promise; openDevice(): Promise; close(app: string): Promise; diff --git a/src/core/interactors/apple.ts b/src/core/interactors/apple.ts index a841b8c99..744adcb01 100644 --- a/src/core/interactors/apple.ts +++ b/src/core/interactors/apple.ts @@ -30,6 +30,7 @@ export function createAppleInteractor( openIosApp(device, app, { appBundleId: options?.appBundleId, launchConsole: options?.launchConsole, + launchArgs: options?.launchArgs, url: options?.url, }), openDevice: () => openIosDevice(device), diff --git a/src/daemon/__tests__/context.test.ts b/src/daemon/__tests__/context.test.ts index 090314199..9909e83de 100644 --- a/src/daemon/__tests__/context.test.ts +++ b/src/daemon/__tests__/context.test.ts @@ -14,6 +14,12 @@ test('contextFromFlags forwards scroll pixels from CLI flags', () => { assert.equal(context.pixels, 240); }); +test('contextFromFlags forwards internal non-hittable selector tap flag', () => { + const flags: CommandFlags = { allowNonHittableSelectorTap: true }; + const context = contextFromFlags('/tmp/agent-device.log', flags); + assert.equal(context.allowNonHittableSelectorTap, true); +}); + test('contextFromFlags forwards screenshot flags from CLI flags', () => { const flags: CommandFlags = { screenshotFullscreen: true, diff --git a/src/daemon/context.ts b/src/daemon/context.ts index 05c0864d4..4e858d529 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -21,6 +21,7 @@ export function contextFromFlags( appBundleId, activity: flags?.activity, launchConsole: flags?.launchConsole, + launchArgs: flags?.launchArgs, verbose: flags?.verbose, logPath, traceLogPath, @@ -41,5 +42,6 @@ export function contextFromFlags( backMode: flags?.backMode, pauseMs: flags?.pauseMs, pattern: flags?.pattern, + allowNonHittableSelectorTap: flags?.allowNonHittableSelectorTap, }; } diff --git a/src/daemon/direct-ios-selector.ts b/src/daemon/direct-ios-selector.ts index 5063f6632..aa0e82058 100644 --- a/src/daemon/direct-ios-selector.ts +++ b/src/daemon/direct-ios-selector.ts @@ -6,6 +6,7 @@ export type DirectIosSelectorTarget = { key: 'id' | 'label' | 'text' | 'value'; value: string; raw: string; + allowNonHittableTap?: boolean; }; export function readSimpleIosSelectorTarget(params: { diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 8d37ad9ad..7a4531eb0 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -417,6 +417,43 @@ test('click simple iOS id selector uses direct runner selector tap without snaps } }); +test('click simple iOS selector forwards Maestro non-hittable tap backdoor', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-maestro-selector-fallback'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + + mockDispatch.mockResolvedValue({ + message: 'tapped via non-hittable coordinate fallback', + x: 439.5, + y: 101.5, + referenceWidth: 440, + referenceHeight: 956, + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'click', + positionals: ['id="e2eSignInAlice"'], + flags: { allowNonHittableSelectorTap: true }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + const pressCalls = mockDispatch.mock.calls.filter((call) => call[1] === 'press'); + expect(pressCalls.length).toBe(1); + expect((pressCalls[0]?.[4] as Record)?.directElementSelector).toEqual({ + key: 'id', + value: 'e2eSignInAlice', + raw: 'id="e2eSignInAlice"', + allowNonHittableTap: true, + }); +}); + test('click simple iOS id selector falls back to snapshot coordinates when direct tap fails', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-direct-selector-fallback'; diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index b50584763..980c748f4 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -408,6 +408,266 @@ test('runReplayScriptFile applies CLI env overrides before Maestro compat mappin assert.deepEqual(calls[1]?.positionals, ['id="shell-button"']); }); +test('runReplayScriptFile retries Maestro scrollUntilVisible with scroll probes', async () => { + const calls: CapturedInvocation[] = []; + let waitAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-scroll-until-visible', + script: [ + 'appId: demo.app', + '---', + '- scrollUntilVisible:', + ' element: Discover', + ' direction: UP', + ' timeout: 1200', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'scroll') return { ok: true, data: {} }; + if (req.command === 'find') { + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'find wait timed out' }, + }; + } + waitAttempts += 1; + if (waitAttempts === 3) return { ok: true, data: { waitedMs: 1100 } }; + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'wait timed out' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], + ['find', ['Discover', 'wait', '500']], + ['scroll', ['down']], + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], + ['find', ['Discover', 'wait', '500']], + ['scroll', ['down']], + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '200']], + ], + ); +}); + +test('runReplayScriptFile lets Maestro scrollUntilVisible use fuzzy visible text matching', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-scroll-until-visible-fuzzy-text', + script: ['appId: demo.app', '---', '- scrollUntilVisible:', ' element: Discover', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'find') return { ok: true, data: { found: true } }; + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'wait timed out' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], + ['find', ['Discover', 'wait', '500']], + ], + ); +}); + +test('runReplayScriptFile lets Maestro tapOn use fuzzy visible text matching', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-fuzzy', + script: ['appId: demo.app', '---', '- tapOn: Discover', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'find') return { ok: true, data: { found: true } }; + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'Selector did not match' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['find', ['Discover', 'click']]], + ); +}); + +test('runReplayScriptFile resolves Maestro percentage point taps from snapshot size', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-point-percent', + script: ['appId: demo.app', '---', '- tapOn:', ' point: 20%,20%', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 1000, height: 2000 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['200', '400']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile retries Maestro tapOn until the selector appears', async () => { + const calls: CapturedInvocation[] = []; + let clickAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-tap-on-retry', + script: ['appId: demo.app', '---', '- tapOn:', ' id: delayedButton', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + clickAttempts += 1; + if (clickAttempts === 3) return { ok: true, data: {} }; + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['click', ['id="delayedButton"']], + ['click', ['id="delayedButton"']], + ['click', ['id="delayedButton"']], + ], + ); +}); + +test('runReplayScriptFile recovers Maestro enter submit after iOS runner transport reset', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-press-enter-recover', + script: ['appId: demo.app', '---', '- pressKey: Enter', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') return { ok: true, data: {} }; + return { + ok: false, + error: { code: 'UNKNOWN', message: 'fetch failed' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['type', ['\n']], + ['snapshot', []], + ], + ); +}); + +test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absent', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-skip', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'not visible' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']]], + ); +}); + +test('runReplayScriptFile runs Maestro runFlow.when.visible commands when present', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-run', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'is') return { ok: true, data: { pass: true } }; + if (req.command === 'click') { + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'Selector did not match' }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']], + ['find', ['Continue', 'click']], + ], + ); +}); + test('runReplayScriptFile reads shell env from request (client-collected), not daemon process.env', async () => { // Ensure the daemon's own process.env does NOT contain AD_VAR_APP. assert.equal(process.env.AD_VAR_APP, undefined); diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 494722171..00ce38d36 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -258,7 +258,12 @@ function readDirectIosSelectorTapTarget(params: { if (commandLabel !== 'click') return null; if (target.kind !== 'selector') return null; if (hasNonDefaultClickOptions(flags)) return null; - return readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); + const selector = readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); + if (!selector) return null; + return { + ...selector, + ...(flags?.allowNonHittableSelectorTap ? { allowNonHittableTap: true } : {}), + }; } function hasNonDefaultClickOptions(flags: CommandFlags | undefined): boolean { diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts new file mode 100644 index 000000000..93a41ae67 --- /dev/null +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -0,0 +1,318 @@ +import { type CommandFlags } from '../../core/dispatch.ts'; +import { MAESTRO_RUNTIME_COMMAND } from '../../compat/maestro/runtime-commands.ts'; +import type { SnapshotState } from '../../utils/snapshot.ts'; +import { sleep } from '../../utils/timeouts.ts'; +import { parseSelectorChain } from '../selectors.ts'; +import { getSnapshotReferenceFrame } from '../touch-reference-frame.ts'; +import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; +import { errorResponse } from './response.ts'; + +const MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS = 500; +const MAESTRO_TAP_ON_TIMEOUT_MS = 30000; +const MAESTRO_TAP_ON_RETRY_MS = 250; + +type ReplayBaseRequest = Omit; + +type MaestroReplayInvoker = (params: { + action: SessionAction; + line: number; + step: number; +}) => Promise; + +export async function invokeMaestroRuntimeCommand(params: { + command: string; + baseReq: ReplayBaseRequest; + positionals: string[]; + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: MaestroReplayInvoker; +}): Promise { + switch (params.command) { + case MAESTRO_RUNTIME_COMMAND.scrollUntilVisible: + return await invokeMaestroScrollUntilVisible(params); + case MAESTRO_RUNTIME_COMMAND.tapOn: + return await invokeMaestroTapOn(params); + case MAESTRO_RUNTIME_COMMAND.tapPointPercent: + return await invokeMaestroTapPointPercent(params); + case MAESTRO_RUNTIME_COMMAND.runFlowWhen: + return await invokeMaestroRunFlowWhen(params); + case MAESTRO_RUNTIME_COMMAND.pressEnter: + return await invokeMaestroPressEnter(params); + default: + return undefined; + } +} + +async function invokeMaestroScrollUntilVisible(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const [selector, timeoutValue = '5000', direction = 'down'] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'scrollUntilVisible requires a selector.'); + } + const timeoutMs = Number(timeoutValue); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + return errorResponse('INVALID_ARGS', 'scrollUntilVisible timeout must be a positive number.'); + } + const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + const attempts = Math.max(1, Math.ceil(timeoutMs / MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS)); + let lastWaitResponse: DaemonResponse | undefined; + + for (let index = 0; index < attempts; index += 1) { + const probeMs = Math.min( + MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS, + Math.max(1, timeoutMs - index * MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS), + ); + const waitResponse = await params.invoke({ + ...params.baseReq, + command: 'wait', + positionals: [selector, String(probeMs)], + }); + if (waitResponse.ok) return waitResponse; + lastWaitResponse = waitResponse; + + const fuzzyResponse = fuzzyTextQuery + ? await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [fuzzyTextQuery, 'wait', String(probeMs)], + }) + : undefined; + if (fuzzyResponse?.ok) return fuzzyResponse; + lastWaitResponse = fuzzyResponse ?? lastWaitResponse; + + if (index === attempts - 1) break; + + const scrollResponse = await params.invoke({ + ...params.baseReq, + command: 'scroll', + positionals: [direction], + }); + if (!scrollResponse.ok) return scrollResponse; + } + + return withMaestroScrollTimeoutContext(lastWaitResponse, selector, timeoutMs); +} + +async function invokeMaestroTapPointPercent(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const [xValue, yValue] = params.positionals; + const xPercent = Number(xValue); + const yPercent = Number(yValue); + if (!Number.isFinite(xPercent) || !Number.isFinite(yPercent)) { + return errorResponse('INVALID_ARGS', 'tapOn percentage point requires numeric x/y values.'); + } + + const snapshotResponse = await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, + }); + if (!snapshotResponse.ok) return snapshotResponse; + + const snapshot = readSnapshotState(snapshotResponse.data); + if (!snapshot) { + return errorResponse( + 'COMMAND_FAILED', + 'Unable to read snapshot data for Maestro percentage point tap.', + ); + } + + const frame = getSnapshotReferenceFrame(snapshot); + if (!frame) { + return errorResponse( + 'COMMAND_FAILED', + 'Unable to resolve screen size for Maestro percentage point tap.', + ); + } + + return await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [ + String(Math.round((frame.referenceWidth * xPercent) / 100)), + String(Math.round((frame.referenceHeight * yPercent) / 100)), + ], + }); +} + +function readSnapshotState(data: unknown): SnapshotState | undefined { + if ( + typeof data === 'object' && + data !== null && + Array.isArray((data as { nodes?: unknown }).nodes) + ) { + return data as SnapshotState; + } + return undefined; +} + +async function invokeMaestroTapOn(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const [selector] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'tapOn requires a selector.'); + } + const startedAt = Date.now(); + const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + let lastResponse: DaemonResponse | undefined; + while (Date.now() - startedAt < MAESTRO_TAP_ON_TIMEOUT_MS) { + if (fuzzyTextQuery) { + const findResponse = await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [fuzzyTextQuery, 'click'], + }); + if (findResponse.ok) return findResponse; + lastResponse = findResponse; + } + + const clickResponse = await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [selector], + }); + if (clickResponse.ok) return clickResponse; + lastResponse = clickResponse; + await sleep(MAESTRO_TAP_ON_RETRY_MS); + } + + if (params.baseReq.flags?.maestroOptional === true) { + return { ok: true, data: { skipped: true, optional: true, selector } }; + } + return ( + lastResponse ?? errorResponse('COMMAND_FAILED', `tapOn timed out for selector: ${selector}`) + ); +} + +async function invokeMaestroRunFlowWhen(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: MaestroReplayInvoker; +}): Promise { + const [mode, selector] = params.positionals; + if ((mode !== 'visible' && mode !== 'notVisible') || !selector) { + return errorResponse( + 'INVALID_ARGS', + 'runFlow.when requires visible/notVisible and a selector.', + ); + } + const predicate = mode === 'visible' ? 'visible' : 'hidden'; + const conditionResponse = await params.invoke({ + ...params.baseReq, + command: 'is', + positionals: [predicate, selector], + flags: { ...params.baseReq.flags, noRecord: true }, + }); + if (!conditionResponse.ok) { + return { ok: true, data: { skipped: true, condition: mode, selector } }; + } + + const steps = (params.batchSteps ?? []).map(batchStepToSessionAction); + for (const [index, action] of steps.entries()) { + const response = await params.invokeReplayAction({ + action, + line: params.line, + step: params.step + index / 1000, + }); + if (!response.ok) return response; + } + + return { ok: true, data: { ran: steps.length, condition: mode, selector } }; +} + +async function invokeMaestroPressEnter(params: { + baseReq: ReplayBaseRequest; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const response = await params.invoke({ + ...params.baseReq, + command: 'type', + positionals: ['\n'], + }); + if (response.ok) return response; + const message = response.error.message.toLowerCase(); + if (!message.includes('fetch failed')) return response; + + const snapshotResponse = await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { ...params.baseReq.flags, noRecord: true }, + }); + if (!snapshotResponse.ok) return response; + return { + ok: true, + data: { + recovered: true, + warning: 'Enter key submit reset the iOS runner transport; recovered after snapshot.', + }, + }; +} + +function batchStepToSessionAction( + step: NonNullable[number], +): SessionAction { + const action: SessionAction = { + ts: Date.now(), + command: step.command, + positionals: step.positionals ?? [], + flags: step.flags ?? {}, + }; + if (step.runtime && typeof step.runtime === 'object') { + action.runtime = step.runtime as SessionAction['runtime']; + } + return action; +} + +function extractMaestroVisibleTextQuery(selectorExpression: string): string | null { + const chain = parseSelectorChain(selectorExpression); + const terms = chain.selectors.flatMap((selector) => selector.terms); + if (terms.length === 0) return null; + if (!terms.some((term) => term.key === 'label' || term.key === 'text')) return null; + if (!terms.every((term) => ['label', 'text', 'id'].includes(term.key))) return null; + const values = terms.map((term) => (typeof term.value === 'string' ? term.value : '')); + const first = values[0]; + if (!first || !values.every((value) => value === first)) return null; + return first; +} + +function withMaestroScrollTimeoutContext( + response: DaemonResponse | undefined, + selector: string, + timeoutMs: number, +): DaemonResponse { + if (!response || response.ok) { + return errorResponse( + 'COMMAND_FAILED', + `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}`, + ); + } + return { + ok: false, + error: { + ...response.error, + message: `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}. Last wait: ${response.error.message}`, + }, + }; +} diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index 12ff2cee0..b2774457d 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -10,6 +10,7 @@ import { healReplayAction } from './session-replay-heal.ts'; import { formatScriptActionSummary } from '../../replay/script-utils.ts'; import { mergeParentFlags } from './handler-utils.ts'; import { errorResponse } from './response.ts'; +import { invokeMaestroRuntimeCommand } from './session-replay-maestro-runtime.ts'; import { buildReplayVarScope, collectReplayShellEnv, @@ -180,15 +181,41 @@ async function invokeReplayAction(params: { command: resolved.command, positionals: resolved.positionals ?? [], }); - const response = await invoke({ + const flags = buildReplayActionFlags(req.flags, resolved.flags); + const baseReq = { token: req.token, session: sessionName, - command: resolved.command, - positionals: resolved.positionals ?? [], - flags: buildReplayActionFlags(req.flags, resolved.flags), + flags, runtime: resolved.runtime, meta: req.meta, - }); + }; + const response = + (await invokeMaestroRuntimeCommand({ + command: resolved.command, + baseReq, + positionals: resolved.positionals ?? [], + batchSteps: resolved.flags?.batchSteps, + line, + step, + invoke, + invokeReplayAction: async (nested) => + await invokeReplayAction({ + req, + sessionName, + action: nested.action, + scope, + filePath, + line: nested.line, + step: nested.step, + tracePath, + invoke, + }), + })) ?? + (await invoke({ + ...baseReq, + command: resolved.command, + positionals: resolved.positionals ?? [], + })); const finishedAt = Date.now(); appendReplayTraceEvent(tracePath, { type: 'replay_action_stop', diff --git a/src/daemon/types.ts b/src/daemon/types.ts index d5f3720bd..aa5df7f59 100644 --- a/src/daemon/types.ts +++ b/src/daemon/types.ts @@ -228,6 +228,7 @@ export type SessionAction = { snapshotDepth?: number; snapshotScope?: string; snapshotRaw?: boolean; + launchArgs?: string[]; saveScript?: boolean | string; noRecord?: boolean; }; diff --git a/src/platforms/ios/apps.ts b/src/platforms/ios/apps.ts index 70bd66553..5eccf112c 100644 --- a/src/platforms/ios/apps.ts +++ b/src/platforms/ios/apps.ts @@ -127,7 +127,7 @@ export async function resolveIosApp(device: DeviceInfo, app: string): Promise { const launchConsole = options?.launchConsole?.trim(); if (launchConsole && (device.platform !== 'ios' || device.kind !== 'simulator')) { @@ -185,7 +185,10 @@ export async function openIosApp( const bundleId = options?.appBundleId ?? (await resolveIosApp(device, app)); if (device.kind === 'simulator') { - await launchIosSimulatorApp(device, bundleId, launchConsole ? { launchConsole } : undefined); + await launchIosSimulatorApp(device, bundleId, { + ...(launchConsole ? { launchConsole } : {}), + ...(options?.launchArgs ? { launchArgs: options.launchArgs } : {}), + }); return; } @@ -884,7 +887,7 @@ function isIosBiometricCapabilityMissing(stdout: string, stderr: string): boolea async function launchIosSimulatorApp( device: DeviceInfo, bundleId: string, - options?: { launchConsole?: string }, + options?: { launchConsole?: string; launchArgs?: string[] }, ): Promise { await ensureBootedSimulator(device); @@ -947,11 +950,12 @@ async function launchIosSimulatorApp( function buildIosSimulatorLaunchArgs( deviceId: string, bundleId: string, - options?: { launchConsole?: string }, + options?: { launchConsole?: string; launchArgs?: string[] }, ): string[] { const args = ['launch']; if (options?.launchConsole) args.push('--console-pty'); args.push(deviceId, bundleId); + if (options?.launchArgs) args.push(...options.launchArgs); return args; } diff --git a/src/platforms/ios/interactions.ts b/src/platforms/ios/interactions.ts index 50f91f0df..eadbd1798 100644 --- a/src/platforms/ios/interactions.ts +++ b/src/platforms/ios/interactions.ts @@ -81,6 +81,7 @@ export function iosRunnerOverrides( command: 'tap', selectorKey: selector.key, selectorValue: selector.value, + allowNonHittableSelectorTap: selector.allowNonHittableTap, appBundleId: ctx.appBundleId, }, runnerOpts, @@ -159,7 +160,7 @@ export function iosRunnerOverrides( command: 'type', text, delayMs, - textEntryMode: 'append', + textEntryMode: text === '\n' ? undefined : 'append', appBundleId: ctx.appBundleId, }, runnerOpts, diff --git a/src/platforms/ios/runner-contract.ts b/src/platforms/ios/runner-contract.ts index 598ed94f2..e87b6ccf9 100644 --- a/src/platforms/ios/runner-contract.ts +++ b/src/platforms/ios/runner-contract.ts @@ -44,6 +44,7 @@ export type RunnerCommand = { text?: string; selectorKey?: 'id' | 'label' | 'text' | 'value'; selectorValue?: string; + allowNonHittableSelectorTap?: boolean; delayMs?: number; textEntryMode?: 'append' | 'replace'; action?: 'get' | 'accept' | 'dismiss'; diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index a42b09922..4ac3e631f 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -917,10 +917,9 @@ test('usageForCommand includes Maestro replay flag', () => { assert.match(help, /--maestro/); assert.match(help, /doubleTapOn/); assert.match(help, /pasteText/); - assert.match(help, /setPermissions/); - assert.match(help, /startRecording\/stopRecording/); assert.match(help, /runFlow file\/inline/); assert.match(help, /repeat\.times/); + assert.match(help, /stopApp/); assert.match(help, /Unsupported syntax fails loudly/); assert.match(help, /issues\/558/); }); diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index 2480965bb..9bafc1240 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -1263,7 +1263,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--maestro', usageDescription: - 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp without state-reset side effects, runFlow file/inline with when.platform, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, assertTrue literal true/false, extendedWaitUntil, scroll, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, stopApp/killApp, setAirplaneMode, setLocation, setOrientation, supported setPermissions targets, and startRecording/stopRecording. ' + + 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with launch arguments but without state-reset side effects, runFlow file/inline with when.platform/visible/notVisible, runScript file/env with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + 'Unsupported syntax fails loudly with a link to https://github.com/callstackincubator/agent-device/issues/558', }, { diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 675266000..24b264501 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -61,11 +61,11 @@ Maestro compatibility translates supported YAML commands into Agent Device repla - Supported and unsupported capabilities: https://github.com/callstackincubator/agent-device/issues/558 - New focused compatibility request: https://github.com/callstackincubator/agent-device/issues/new -Currently supported areas include app launch without state-reset side effects, file and inline `runFlow` with `when.platform`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn`, `doubleTapOn`, `longPressOn`, `inputText`, `pasteText`, `openLink`, visibility assertions, literal `assertTrue`, `extendedWaitUntil`, `scroll`, absolute/percentage `swipe`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, `stopApp` / `killApp`, airplane mode, mock location, orientation, supported permission targets, and screen recording. +Currently supported areas include app launch with launch arguments but without state-reset side effects, file and inline `runFlow` with `when.platform`, `when.visible`, and `when.notVisible`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional` and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as a Maestro compatibility feature for file/env scripts that use `http.post`, `json`, and `output` variables; it is not a native `.ad` command. Maestro `env` values use the same replay precedence as `.ad` files: flow `env` is the default, shell `AD_VAR_*` values override it, and CLI `-e KEY=VALUE` wins over both. -Runtime-dependent Maestro features such as `scrollUntilVisible`, `repeat.while`, `runFlow.when.visible`, `runScript`, `evalScript`, text clearing, and app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. +Unsupported Maestro features such as `repeat.while`, `runFlow.when.true`, full expression predicates, `evalScript`, text clearing, selector relations such as `index` / `childOf`, device utility commands, and app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. ## Run a lightweight `.ad` suite From 1eb3a07248051a2d011002ba3e88c510cee0a02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 22 May 2026 09:50:25 +0200 Subject: [PATCH 2/8] fix: harden Maestro replay compatibility --- .../RunnerTests+CommandExecution.swift | 1 + .../RunnerTests+Interaction.swift | 29 +++ .../maestro/__tests__/replay-flow.test.ts | 1 + src/compat/maestro/device-actions.ts | 4 +- src/compat/maestro/interactions.ts | 22 +- src/compat/maestro/support.ts | 22 -- src/core/dispatch-context.ts | 2 + src/core/dispatch.ts | 17 +- src/daemon/__tests__/context.test.ts | 6 + src/daemon/context.ts | 4 + src/daemon/handlers/__tests__/find.test.ts | 36 ++++ .../__tests__/session-replay-vars.test.ts | 65 ++++++ src/daemon/handlers/find.ts | 204 +++++++++++++----- .../session-replay-maestro-runtime.ts | 122 +++++++---- src/platforms/ios/apps.ts | 47 ++++ src/utils/command-schema.ts | 2 +- 16 files changed, 455 insertions(+), 129 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index b71f570cb..c7dd5cf96 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -282,6 +282,7 @@ extension RunnerTests { if let response = unsupportedResponse(for: outcome) { return response } + waitForTextEntryReadinessAfterTap(app: activeApp, element: element) return Response( ok: true, data: DataPayload( diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index b912d6981..827379e8c 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -809,6 +809,35 @@ extension RunnerTests { #endif } + func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { +#if os(iOS) + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil { + return + } + let frame = element.frame + if !frame.isEmpty { + _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) + } + default: + return + } +#endif + } + + private func waitForFocusedTextInput(app: XCUIApplication, timeout: TimeInterval) -> XCUIElement? { + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if let focused = focusedTextInput(app: app) { + return focused + } + sleepFor(TextEntryTiming.pollInterval) + } + return focusedTextInput(app: app) + } + private func textEntryRefreshPoint(for element: XCUIElement?) -> CGPoint? { guard let element else { return nil diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index f50fb6c3f..3425baa77 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -345,6 +345,7 @@ test('parseMaestroReplayFlow accepts launchApp reset options without state-reset 'open', ['com.callstack.agentdevicelab'], { + maestroClearState: true, relaunch: true, launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true', '-Example', 'ignored'], }, diff --git a/src/compat/maestro/device-actions.ts b/src/compat/maestro/device-actions.ts index 8069b2fab..8dcaa62b0 100644 --- a/src/compat/maestro/device-actions.ts +++ b/src/compat/maestro/device-actions.ts @@ -37,9 +37,11 @@ export function convertLaunchApp( context, ); const launchArgs = readLaunchArgs(value, context); - const shouldRelaunch = value.stopApp === true || launchArgs.length > 0; + const shouldClearState = value.clearState === true; + const shouldRelaunch = value.stopApp === true || shouldClearState || launchArgs.length > 0; return action('open', [appId], { relaunch: shouldRelaunch, + ...(shouldClearState ? { maestroClearState: true } : {}), ...(launchArgs.length > 0 ? { launchArgs } : {}), }); } diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index 8fa5b1b13..81153af5e 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -251,15 +251,11 @@ function selectorTerm(key: string, value: string): string { function tapFlags(value: unknown): SessionAction['flags'] | undefined { if (!isPlainRecord(value)) return undefined; const flags: SessionAction['flags'] = {}; - if (typeof value.repeat === 'number' && Number.isInteger(value.repeat) && value.repeat > 1) { - flags.count = value.repeat; - } - if (typeof value.delay === 'number' && Number.isInteger(value.delay) && value.delay >= 0) { - flags.intervalMs = value.delay; - } - if (value.optional === true) { - flags.maestroOptional = true; - } + const repeat = positiveInteger(value.repeat); + const delay = nonNegativeInteger(value.delay); + if (repeat && repeat > 1) flags.count = repeat; + if (delay !== undefined) flags.intervalMs = delay; + if (value.optional === true) flags.maestroOptional = true; return Object.keys(flags).length > 0 ? flags : undefined; } @@ -270,3 +266,11 @@ function doubleTapFlags(value: unknown): SessionAction['flags'] { } return flags; } + +function positiveInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined; +} + +function nonNegativeInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : undefined; +} diff --git a/src/compat/maestro/support.ts b/src/compat/maestro/support.ts index 591e23ecb..997a5c70d 100644 --- a/src/compat/maestro/support.ts +++ b/src/compat/maestro/support.ts @@ -63,24 +63,6 @@ export function normalizePlatformValue(value: unknown, name: string): 'android' return platform; } -export function normalizeToken(value: string): string { - return value - .trim() - .replace(/([a-z0-9])([A-Z])/g, '$1-$2') - .replace(/[\s_]+/g, '-') - .toLowerCase(); -} - -export function readBooleanLiteral(value: unknown, command: string): boolean { - if (typeof value === 'boolean') return value; - if (typeof value === 'string') { - const normalized = normalizeToken(value); - if (normalized === 'true') return true; - if (normalized === 'false') return false; - } - throw new AppError('INVALID_ARGS', `${command} expects a boolean value.`); -} - export function readEnvMap(value: unknown, name: string): Record { if (value === undefined || value === null) return {}; if (!isPlainRecord(value)) { @@ -118,10 +100,6 @@ export function resolveMaestroString(value: string, context: MaestroParseContext }); } -export function resolveMaybeMaestroString(value: unknown, context: MaestroParseContext): unknown { - return typeof value === 'string' ? resolveMaestroString(value, context) : value; -} - export function unsupportedCommand(command: string): never { throw unsupportedMaestroSyntax(`Maestro command "${command}" is not supported yet.`); } diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index b6e5712ec..9bf959b8b 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -16,6 +16,7 @@ export type CommandFlags = Omit & { replayBackend?: string; allowNonHittableSelectorTap?: boolean; maestroOptional?: boolean; + maestroClearState?: boolean; }; export type DispatchContext = ScreenshotDispatchFlags & { @@ -24,6 +25,7 @@ export type DispatchContext = ScreenshotDispatchFlags & { activity?: string; launchConsole?: string; launchArgs?: string[]; + maestroClearState?: boolean; verbose?: boolean; logPath?: string; traceLogPath?: string; diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index b5b84b3ec..f4eea78d1 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -10,7 +10,7 @@ import { pushAndroidNotification } from '../platforms/android/notifications.ts'; import { getInteractor } from './interactors.ts'; import type { Interactor, RunnerContext } from './interactor-types.ts'; import { runIosRunnerCommand } from '../platforms/ios/runner-client.ts'; -import { pushIosNotification } from '../platforms/ios/apps.ts'; +import { clearIosSimulatorAppState, pushIosNotification } from '../platforms/ios/apps.ts'; import { isDeepLinkTarget } from './open-target.ts'; import { parseTriggerAppEventArgs, resolveAppEventUrl } from './app-events.ts'; import { @@ -225,6 +225,21 @@ async function handleOpenCommand( if (launchConsole && isDeepLinkTarget(app)) { throw new AppError('INVALID_ARGS', LAUNCH_CONSOLE_DIRECT_APP_ONLY_MESSAGE); } + if (context?.maestroClearState) { + if (isDeepLinkTarget(app)) { + throw new AppError( + 'INVALID_ARGS', + 'Maestro launchApp.clearState requires an app target, not a deep link.', + ); + } + if (device.platform !== 'ios' || device.kind !== 'simulator') { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Maestro launchApp.clearState is currently supported only on iOS simulators.', + ); + } + await clearIosSimulatorAppState(device, app); + } await interactor.open(app, { activity: context?.activity, appBundleId: context?.appBundleId, diff --git a/src/daemon/__tests__/context.test.ts b/src/daemon/__tests__/context.test.ts index 9909e83de..bf117b083 100644 --- a/src/daemon/__tests__/context.test.ts +++ b/src/daemon/__tests__/context.test.ts @@ -20,6 +20,12 @@ test('contextFromFlags forwards internal non-hittable selector tap flag', () => assert.equal(context.allowNonHittableSelectorTap, true); }); +test('contextFromFlags forwards Maestro clearState launch compatibility flag', () => { + const flags: CommandFlags = { maestroClearState: true }; + const context = contextFromFlags('/tmp/agent-device.log', flags); + assert.equal(context.maestroClearState, true); +}); + test('contextFromFlags forwards screenshot flags from CLI flags', () => { const flags: CommandFlags = { screenshotFullscreen: true, diff --git a/src/daemon/context.ts b/src/daemon/context.ts index 4e858d529..a3fd128c7 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -8,6 +8,9 @@ import { getDiagnosticsMeta } from '../utils/diagnostics.ts'; export type DaemonCommandContext = DispatchContext & ScreenshotRuntimeFlags; +// Flat compatibility mapper: keeping each CLI flag visible here makes request +// context drift easier to spot than splitting the same optional fields apart. +// fallow-ignore-next-line complexity export function contextFromFlags( logPath: string, flags: CommandFlags | undefined, @@ -22,6 +25,7 @@ export function contextFromFlags( activity: flags?.activity, launchConsole: flags?.launchConsole, launchArgs: flags?.launchArgs, + maestroClearState: flags?.maestroClearState, verbose: flags?.verbose, logPath, traceLogPath, diff --git a/src/daemon/handlers/__tests__/find.test.ts b/src/daemon/handlers/__tests__/find.test.ts index 7efbca153..9b7d165ba 100644 --- a/src/daemon/handlers/__tests__/find.test.ts +++ b/src/daemon/handlers/__tests__/find.test.ts @@ -121,6 +121,42 @@ test('handleFindCommands click returns deterministic metadata across locator var } }); +test('handleFindCommands click prefers on-screen duplicate text matches', async () => { + const { response, invokeCalls } = await runFindClickScenario({ + positionals: ['Sign in', 'click'], + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Application', + hittable: true, + rect: { x: 0, y: 0, width: 440, height: 956 }, + }, + { + index: 1, + ref: 'e2', + type: 'Button', + label: 'Sign in', + hittable: false, + rect: { x: -199, y: 186, width: 70, height: 33 }, + parentIndex: 0, + }, + { + index: 2, + ref: 'e3', + type: 'Button', + label: 'Sign in', + hittable: false, + rect: { x: 40, y: 870, width: 360, height: 44 }, + parentIndex: 0, + }, + ], + }); + + expect(response.ok).toBe(true); + expect(invokeCalls[0].positionals?.[0]).toBe('@e3'); +}); + test('handleFindCommands wait bypasses snapshot cache while Android freshness recovery is active', async () => { const sessionName = 'android-find-wait'; const session: SessionState = { diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index 980c748f4..cb12bc227 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -507,6 +507,71 @@ test('runReplayScriptFile lets Maestro tapOn use fuzzy visible text matching', a ); }); +test('runReplayScriptFile retries Maestro fuzzy tapOn without raw selector fallback', async () => { + const calls: CapturedInvocation[] = []; + let findAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-fuzzy-retry', + script: ['appId: demo.app', '---', '- tapOn: Discover', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'find') { + findAttempts += 1; + if (findAttempts === 2) return { ok: true, data: { found: true } }; + } + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['find', ['Discover', 'click']], + ['find', ['Discover', 'click']], + ], + ); +}); + +test('runReplayScriptFile lets optional Maestro fuzzy tapOn hit native alert labels', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-optional-native-label', + script: [ + 'appId: demo.app', + '---', + '- tapOn:', + ' text: Not Now', + ' optional: true', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'click' && req.positionals?.[0] === 'label="Not Now"') { + return { ok: true, data: { dismissed: true } }; + } + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['find', ['Not Now', 'click']], + ['click', ['label="Not Now"']], + ], + ); +}); + test('runReplayScriptFile resolves Maestro percentage point taps from snapshot size', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 42f2c9104..66363d85b 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -37,6 +37,10 @@ type ResolvedMatch = { actionFlags: Record; }; +type FindMatchResult = + | { ok: true; node: SnapshotState['nodes'][number] } + | { ok: false; response: DaemonResponse }; + export async function handleFindCommands(params: { req: DaemonRequest; sessionName: string; @@ -67,8 +71,7 @@ export async function handleFindCommands(params: { }); if (runtimeResponse) return runtimeResponse; const session = sessionStore.get(sessionName); - const isReadOnly = - action === 'exists' || action === 'wait' || action === 'get_text' || action === 'get_attrs'; + const isReadOnly = isReadOnlyFindAction(action); if (!session && !isReadOnly) { return errorResponse('SESSION_NOT_FOUND', 'No active session. Run open first.'); } @@ -76,9 +79,10 @@ export async function handleFindCommands(params: { if (!session) { await ensureDeviceReady(device); } - const scope = shouldScopeFind(locator) ? query : undefined; - const requiresRect = - action === 'click' || action === 'focus' || action === 'fill' || action === 'type'; + const requiresRect = findActionRequiresRect(action); + // Interaction targets need the full compact tree so duplicate labels can be + // resolved against viewport visibility before an off-screen subtree wins. + const scope = shouldScopeFind(locator) && !requiresRect ? query : undefined; const interactiveOnly = requiresRect; let lastSnapshotAt = 0; let lastNodes: SnapshotState['nodes'] | null = null; @@ -134,29 +138,16 @@ export async function handleFindCommands(params: { } const { nodes } = await fetchNodes(); - const bestMatches = findBestMatchesByLocator(nodes, locator, query, { - requireRect: requiresRect, + const matchResult = resolveFindMatch({ + nodes, + locator, + query, + requiresRect, + flags: req.flags, }); - - if (requiresRect && bestMatches.matches.length > 1) { - if (req.flags?.findFirst) { - bestMatches.matches = [bestMatches.matches[0]]; - } else if (req.flags?.findLast) { - bestMatches.matches = [bestMatches.matches[bestMatches.matches.length - 1]]; - } else { - return buildAmbiguousMatchError(bestMatches.matches, locator, query); - } - } - - const node = bestMatches.matches[0] ?? null; - if (!node) { - return errorResponse('COMMAND_FAILED', 'find did not match any element'); - } - - const resolvedNode = - action === 'click' || action === 'focus' || action === 'fill' || action === 'type' - ? (findNearestHittableAncestor(nodes, node) ?? node) - : node; + if (!matchResult.ok) return matchResult.response; + const node = matchResult.node; + const resolvedNode = requiresRect ? resolveInteractiveMatchNode(nodes, node) : node; const ref = `@${resolvedNode.ref}`; const actionFlags = { ...(req.flags ?? {}), noRecord: true }; const match: ResolvedMatch = { node, resolvedNode, ref, nodes, actionFlags }; @@ -177,6 +168,97 @@ export async function handleFindCommands(params: { // --- Per-action handlers --- +function isReadOnlyFindAction(action: string): boolean { + return ( + action === 'exists' || action === 'wait' || action === 'get_text' || action === 'get_attrs' + ); +} + +function findActionRequiresRect(action: string): boolean { + return action === 'click' || action === 'focus' || action === 'fill' || action === 'type'; +} + +function resolveFindMatch(params: { + nodes: SnapshotState['nodes']; + locator: FindLocator; + query: string; + requiresRect: boolean; + flags: DaemonRequest['flags']; +}): FindMatchResult { + const { nodes, locator, query, requiresRect, flags } = params; + const bestMatches = findBestMatchesByLocator(nodes, locator, query, { + requireRect: requiresRect, + }); + if (requiresRect) { + bestMatches.matches = preferOnscreenMatches(bestMatches.matches, nodes); + } + + if (requiresRect && bestMatches.matches.length > 1) { + if (flags?.findFirst) { + bestMatches.matches = [bestMatches.matches[0]]; + } else if (flags?.findLast) { + bestMatches.matches = [bestMatches.matches[bestMatches.matches.length - 1]]; + } else { + return { ok: false, response: buildAmbiguousMatchError(bestMatches.matches, locator, query) }; + } + } + + const node = bestMatches.matches[0] ?? null; + if (!node) { + return { + ok: false, + response: errorResponse('COMMAND_FAILED', 'find did not match any element'), + }; + } + return { ok: true, node }; +} + +function preferOnscreenMatches( + matches: SnapshotState['nodes'], + nodes: SnapshotState['nodes'], +): SnapshotState['nodes'] { + const viewport = nodes[0]?.rect; + if (!viewport) return matches; + const onscreen = matches.filter((node) => { + if (!node.rect) return false; + const center = centerOfRect(node.rect); + return ( + center.x >= viewport.x && + center.x <= viewport.x + viewport.width && + center.y >= viewport.y && + center.y <= viewport.y + viewport.height + ); + }); + return onscreen.length > 0 ? onscreen : matches; +} + +function resolveInteractiveMatchNode( + nodes: SnapshotState['nodes'], + node: SnapshotState['nodes'][number], +): SnapshotState['nodes'][number] { + const ancestor = findNearestHittableAncestor(nodes, node); + if (!ancestor) return node; + if (node.rect && isRootInteractionContainer(ancestor, nodes[0])) { + return node; + } + return ancestor; +} + +function isRootInteractionContainer( + node: SnapshotState['nodes'][number], + root: SnapshotState['nodes'][number] | undefined, +): boolean { + if (!root?.rect || !node.rect) return false; + const type = node.type?.toLowerCase() ?? ''; + if (!type.includes('application') && !type.includes('window')) return false; + return ( + node.rect.x === root.rect.x && + node.rect.y === root.rect.y && + node.rect.width === root.rect.width && + node.rect.height === root.rect.height + ); +} + async function handleFindWait( ctx: FindContext, fetchNodes: () => Promise<{ nodes: SnapshotState['nodes'] }>, @@ -266,7 +348,11 @@ async function handleFindClick(ctx: FindContext, match: ResolvedMatch): Promise< flags: match.actionFlags, }); if (!response.ok) return response; - const matchCoords = match.resolvedNode.rect ? centerOfRect(match.resolvedNode.rect) : null; + const matchCoords = match.resolvedNode.rect + ? centerOfRect(match.resolvedNode.rect) + : match.node.rect + ? centerOfRect(match.node.rect) + : null; const matchData: Record = { ref: match.ref, locator, query }; if (matchCoords) { matchData.x = matchCoords.x; @@ -312,7 +398,35 @@ async function handleFindFill( } async function handleFindFocus(ctx: FindContext, match: ResolvedMatch): Promise { - const { req, sessionStore, session, device, command, logPath } = ctx; + const response = await dispatchFocusForFindMatch(ctx, match); + if (!response.ok) return response; + recordFindAction(ctx, match, 'focus'); + return response; +} + +async function handleFindType( + ctx: FindContext, + match: ResolvedMatch, + value: string | undefined, +): Promise { + const { req, device, logPath, session } = ctx; + if (!value) { + return errorResponse('INVALID_ARGS', 'find type requires text'); + } + const focusResponse = await dispatchFocusForFindMatch(ctx, match); + if (!focusResponse.ok) return focusResponse; + const response = await dispatchCommand(device, 'type', [value], req.flags?.out, { + ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), + }); + recordFindAction(ctx, match, 'type'); + return { ok: true, data: response ?? { ref: match.ref } }; +} + +async function dispatchFocusForFindMatch( + ctx: FindContext, + match: ResolvedMatch, +): Promise { + const { req, device, logPath, session } = ctx; const coords = match.node.rect ? centerOfRect(match.node.rect) : null; if (!coords) { return errorResponse('COMMAND_FAILED', 'matched element has no bounds'); @@ -326,45 +440,19 @@ async function handleFindFocus(ctx: FindContext, match: ResolvedMatch): Promise< ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), }, ); - if (session) { - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { ref: match.ref, action: 'focus' }, - }); - } return { ok: true, data: response ?? { ref: match.ref } }; } -async function handleFindType( - ctx: FindContext, - match: ResolvedMatch, - value: string | undefined, -): Promise { - const { req, sessionStore, session, device, command, logPath } = ctx; - if (!value) { - return errorResponse('INVALID_ARGS', 'find type requires text'); - } - const coords = match.node.rect ? centerOfRect(match.node.rect) : null; - if (!coords) { - return errorResponse('COMMAND_FAILED', 'matched element has no bounds'); - } - await dispatchCommand(device, 'focus', [String(coords.x), String(coords.y)], req.flags?.out, { - ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), - }); - const response = await dispatchCommand(device, 'type', [value], req.flags?.out, { - ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), - }); +function recordFindAction(ctx: FindContext, match: ResolvedMatch, action: string): void { + const { req, sessionStore, session, command } = ctx; if (session) { sessionStore.recordAction(session, { command, positionals: req.positionals ?? [], flags: req.flags ?? {}, - result: { ref: match.ref, action: 'type' }, + result: { ref: match.ref, action }, }); } - return { ok: true, data: response ?? { ref: match.ref } }; } // --- Helpers --- diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts index 93a41ae67..10b7d1db3 100644 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -19,6 +19,20 @@ type MaestroReplayInvoker = (params: { step: number; }) => Promise; +type MaestroRuntimeInvoke = (req: DaemonRequest) => Promise; + +type MaestroScrollUntilVisibleParams = { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}; + +type MaestroTapOnParams = { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}; + export async function invokeMaestroRuntimeCommand(params: { command: string; baseReq: ReplayBaseRequest; @@ -45,11 +59,9 @@ export async function invokeMaestroRuntimeCommand(params: { } } -async function invokeMaestroScrollUntilVisible(params: { - baseReq: ReplayBaseRequest; - positionals: string[]; - invoke: (req: DaemonRequest) => Promise; -}): Promise { +async function invokeMaestroScrollUntilVisible( + params: MaestroScrollUntilVisibleParams, +): Promise { const [selector, timeoutValue = '5000', direction = 'down'] = params.positionals; if (!selector) { return errorResponse('INVALID_ARGS', 'scrollUntilVisible requires a selector.'); @@ -63,27 +75,14 @@ async function invokeMaestroScrollUntilVisible(params: { let lastWaitResponse: DaemonResponse | undefined; for (let index = 0; index < attempts; index += 1) { - const probeMs = Math.min( - MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS, - Math.max(1, timeoutMs - index * MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS), + const probe = await probeMaestroScrollVisibility( + params, + selector, + fuzzyTextQuery, + scrollProbeMs(timeoutMs, index), ); - const waitResponse = await params.invoke({ - ...params.baseReq, - command: 'wait', - positionals: [selector, String(probeMs)], - }); - if (waitResponse.ok) return waitResponse; - lastWaitResponse = waitResponse; - - const fuzzyResponse = fuzzyTextQuery - ? await params.invoke({ - ...params.baseReq, - command: 'find', - positionals: [fuzzyTextQuery, 'wait', String(probeMs)], - }) - : undefined; - if (fuzzyResponse?.ok) return fuzzyResponse; - lastWaitResponse = fuzzyResponse ?? lastWaitResponse; + if (probe.visible) return probe.response; + lastWaitResponse = probe.response; if (index === attempts - 1) break; @@ -98,6 +97,35 @@ async function invokeMaestroScrollUntilVisible(params: { return withMaestroScrollTimeoutContext(lastWaitResponse, selector, timeoutMs); } +async function probeMaestroScrollVisibility( + params: MaestroScrollUntilVisibleParams, + selector: string, + fuzzyTextQuery: string | null, + probeMs: number, +): Promise<{ visible: boolean; response: DaemonResponse }> { + const waitResponse = await params.invoke({ + ...params.baseReq, + command: 'wait', + positionals: [selector, String(probeMs)], + }); + if (waitResponse.ok) return { visible: true, response: waitResponse }; + if (!fuzzyTextQuery) return { visible: false, response: waitResponse }; + + const fuzzyResponse = await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [fuzzyTextQuery, 'wait', String(probeMs)], + }); + return { visible: fuzzyResponse.ok, response: fuzzyResponse }; +} + +function scrollProbeMs(timeoutMs: number, index: number): number { + return Math.min( + MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS, + Math.max(1, timeoutMs - index * MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS), + ); +} + async function invokeMaestroTapPointPercent(params: { baseReq: ReplayBaseRequest; positionals: string[]; @@ -160,11 +188,7 @@ function readSnapshotState(data: unknown): SnapshotState | undefined { return undefined; } -async function invokeMaestroTapOn(params: { - baseReq: ReplayBaseRequest; - positionals: string[]; - invoke: (req: DaemonRequest) => Promise; -}): Promise { +async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { const [selector] = params.positionals; if (!selector) { return errorResponse('INVALID_ARGS', 'tapOn requires a selector.'); @@ -174,13 +198,11 @@ async function invokeMaestroTapOn(params: { let lastResponse: DaemonResponse | undefined; while (Date.now() - startedAt < MAESTRO_TAP_ON_TIMEOUT_MS) { if (fuzzyTextQuery) { - const findResponse = await params.invoke({ - ...params.baseReq, - command: 'find', - positionals: [fuzzyTextQuery, 'click'], - }); - if (findResponse.ok) return findResponse; - lastResponse = findResponse; + const attempt = await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); + if (!attempt.retry) return attempt.response; + lastResponse = attempt.response; + await sleep(MAESTRO_TAP_ON_RETRY_MS); + continue; } const clickResponse = await params.invoke({ @@ -201,6 +223,28 @@ async function invokeMaestroTapOn(params: { ); } +async function invokeMaestroFuzzyTapOn( + params: MaestroTapOnParams, + query: string, +): Promise<{ retry: boolean; response: DaemonResponse }> { + const findResponse = await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [query, 'click'], + }); + if (findResponse.ok) return { retry: false, response: findResponse }; + if (params.baseReq.flags?.maestroOptional !== true) { + return { retry: true, response: findResponse }; + } + + const nativeLabelResponse = await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [simpleLabelSelector(query)], + }); + return { retry: !nativeLabelResponse.ok, response: nativeLabelResponse }; +} + async function invokeMaestroRunFlowWhen(params: { baseReq: ReplayBaseRequest; positionals: string[]; @@ -297,6 +341,10 @@ function extractMaestroVisibleTextQuery(selectorExpression: string): string | nu return first; } +function simpleLabelSelector(value: string): string { + return `label=${JSON.stringify(value)}`; +} + function withMaestroScrollTimeoutContext( response: DaemonResponse | undefined, selector: string, diff --git a/src/platforms/ios/apps.ts b/src/platforms/ios/apps.ts index 5eccf112c..6f091f1f1 100644 --- a/src/platforms/ios/apps.ts +++ b/src/platforms/ios/apps.ts @@ -238,6 +238,53 @@ export async function closeIosApp(device: DeviceInfo, app: string): Promise { + if (device.platform !== 'ios' || device.kind !== 'simulator') { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Maestro launchApp.clearState is currently supported only on iOS simulators.', + ); + } + + const bundleId = await resolveIosApp(device, app); + await ensureBootedSimulator(device); + await closeIosApp(device, bundleId); + + const result = await runSimctl(device, ['get_app_container', device.id, bundleId, 'data'], { + allowFailure: true, + }); + if (result.exitCode !== 0) { + throw new AppError('COMMAND_FAILED', `simctl get_app_container failed for ${bundleId}`, { + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.exitCode, + }); + } + + const containerPath = result.stdout.trim(); + if (!containerPath) { + throw new AppError( + 'COMMAND_FAILED', + `simctl get_app_container returned an empty data container path for ${bundleId}`, + ); + } + + const entries = await fs.readdir(containerPath); + await Promise.all( + entries.map((entry) => + fs.rm(path.join(containerPath, entry), { + recursive: true, + force: true, + }), + ), + ); + + return { bundleId, containerPath }; +} + export async function uninstallIosApp( device: DeviceInfo, app: string, diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index 9bafc1240..ff6f42315 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -1263,7 +1263,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--maestro', usageDescription: - 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with launch arguments but without state-reset side effects, runFlow file/inline with when.platform/visible/notVisible, runScript file/env with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + + 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible, runScript file/env with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + 'Unsupported syntax fails loudly with a link to https://github.com/callstackincubator/agent-device/issues/558', }, { From 0ff03dd3b0e5e019207197487f78dc39f303222c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 22 May 2026 13:54:10 +0200 Subject: [PATCH 3/8] fix: address Maestro replay review feedback --- .../RunnerTests+Interaction.swift | 5 +- .../maestro/__tests__/replay-flow.test.ts | 27 +++++- src/compat/maestro/interactions.ts | 18 +++- src/compat/maestro/run-script.ts | 36 ++++++- src/core/__tests__/dispatch-open.test.ts | 23 +++++ src/core/dispatch.ts | 6 ++ .../__tests__/session-replay-vars.test.ts | 95 ++++++++++++++++++- .../session-replay-maestro-runtime.ts | 72 +++++++++++--- src/utils/command-schema.ts | 2 +- website/docs/docs/replay-e2e.md | 4 +- 10 files changed, 262 insertions(+), 26 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 827379e8c..a09115239 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -236,7 +236,10 @@ extension RunnerTests { return false } let appFrame = app.frame - return appFrame.isEmpty || appFrame.intersects(frame) + if appFrame.isEmpty { + return true + } + return appFrame.contains(CGPoint(x: frame.midX, y: frame.midY)) } func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response { diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index 3425baa77..f54d670af 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -69,7 +69,7 @@ env: ['scroll', ['right']], [ '__maestroScrollUntilVisible', - ['label="Discover" || text="Discover" || id="Discover"', '5000', 'down'], + ['label="Discover" || text="Discover" || id="Discover"', '5000', 'up'], ], ['screenshot', ['./screens/form.png']], ['keyboard', ['dismiss']], @@ -128,6 +128,29 @@ output.result = SERVER_PATH + ':' + json(res.body).appviewDid ); }); +test('parseMaestroReplayFlow reports runScript http failures with command context', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-runscript-fail-')); + const scriptPath = path.join(root, 'setup.js'); + const flowPath = path.join(root, 'flow.yml'); + fs.writeFileSync(scriptPath, `output.result = http.post('http://127.0.0.1:1').body`); + + assert.throws( + () => + parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runScript: ./setup.js +`, + { sourcePath: flowPath }, + ), + (error) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + /runScript failed/.test(error.message) && + /http\.post failed/.test(error.message), + ); +}); + test('parseMaestroReplayFlow rejects unsupported Maestro commands', () => { assert.throws( () => parseMaestroReplayFlow('---\n- travelThroughTime: Save\n'), @@ -325,7 +348,7 @@ test('parseMaestroReplayFlow keeps visible-gated runFlow commands for runtime ev ]); }); -test('parseMaestroReplayFlow accepts launchApp reset options without state-reset side effects', () => { +test('parseMaestroReplayFlow accepts launchApp reset options', () => { const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- - launchApp: diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index 81153af5e..012e85226 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -142,9 +142,7 @@ export function convertScrollUntilVisible( assertOnlyKeys(value, 'scrollUntilVisible', ['element', 'direction', 'timeout']); const selector = maestroSelector(value.element, 'scrollUntilVisible.element', [], context); const direction = - typeof value.direction === 'string' - ? readScrollPositionalsFromDirectionSwipe(value.direction)[0] - : 'down'; + typeof value.direction === 'string' ? readScrollUntilVisibleDirection(value.direction) : 'down'; const timeoutMs = String(readTimeoutMs(value, 5000)); return [action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [selector, timeoutMs, direction])]; } @@ -198,6 +196,20 @@ function readScrollPositionalsFromDirectionSwipe(direction: string): string[] { } } +function readScrollUntilVisibleDirection(direction: string): string { + switch (direction.toLowerCase()) { + case 'up': + case 'down': + case 'left': + case 'right': + return direction.toLowerCase(); + default: + throw unsupportedMaestroSyntax( + 'Maestro scrollUntilVisible.direction must be UP, DOWN, LEFT, or RIGHT.', + ); + } +} + export function convertPressKey(value: unknown): SessionAction { const key = requireStringValue('pressKey', value).toLowerCase(); if (key === 'back') return action('back'); diff --git a/src/compat/maestro/run-script.ts b/src/compat/maestro/run-script.ts index 8f810aa2b..9e036568e 100644 --- a/src/compat/maestro/run-script.ts +++ b/src/compat/maestro/run-script.ts @@ -12,6 +12,8 @@ import { } from './support.ts'; import type { MaestroParseContext } from './types.ts'; +const RUN_SCRIPT_TIMEOUT_MS = 30_000; + type HttpResponse = { status: number; body: string; @@ -51,7 +53,7 @@ export function executeRunScript(value: unknown, context: MaestroParseContext): try { vm.runInNewContext(script, buildScriptGlobals(scriptEnv, output), { filename: scriptPath, - timeout: 30_000, + timeout: RUN_SCRIPT_TIMEOUT_MS, }); } catch (error) { throw new AppError( @@ -124,9 +126,32 @@ function runHttpRequestSync( headers: options?.headers ?? {}, body: options?.body ?? '', }), - timeoutMs: 30_000, + timeoutMs: RUN_SCRIPT_TIMEOUT_MS, + allowFailure: true, }); - return JSON.parse(result.stdout) as HttpResponse; + if (result.exitCode !== 0) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript http.${method.toLowerCase()} failed for ${url}: ${trimHttpErrorOutput(result.stderr)}`, + { + exitCode: result.exitCode, + stderr: result.stderr, + }, + ); + } + try { + return JSON.parse(result.stdout) as HttpResponse; + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript http.${method.toLowerCase()} returned invalid JSON for ${url}`, + { + stdout: result.stdout.slice(0, 1000), + stderr: result.stderr.slice(0, 1000), + }, + error instanceof Error ? error : undefined, + ); + } } function stringifyOutputValue(value: unknown): string { @@ -134,3 +159,8 @@ function stringifyOutputValue(value: unknown): string { if (typeof value === 'number' || typeof value === 'boolean') return String(value); return JSON.stringify(value); } + +function trimHttpErrorOutput(stderr: string): string { + const trimmed = stderr.trim(); + return trimmed.length > 0 ? trimmed.slice(0, 1000) : 'request process exited without stderr'; +} diff --git a/src/core/__tests__/dispatch-open.test.ts b/src/core/__tests__/dispatch-open.test.ts index c55cbbe68..807122348 100644 --- a/src/core/__tests__/dispatch-open.test.ts +++ b/src/core/__tests__/dispatch-open.test.ts @@ -23,3 +23,26 @@ test('dispatch open rejects URL as first argument when second URL is provided', }, ); }); + +test('dispatch open rejects Android launch arguments instead of dropping them', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + + await assert.rejects( + () => + dispatchCommand(device, 'open', ['com.example.app'], undefined, { + launchArgs: ['--fixture', 'demo'], + }), + (error: unknown) => { + assert.equal(error instanceof AppError, true); + assert.equal((error as AppError).code, 'UNSUPPORTED_OPERATION'); + assert.match((error as AppError).message, /Apple platforms/i); + return true; + }, + ); +}); diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index f4eea78d1..1e5b12397 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -225,6 +225,12 @@ async function handleOpenCommand( if (launchConsole && isDeepLinkTarget(app)) { throw new AppError('INVALID_ARGS', LAUNCH_CONSOLE_DIRECT_APP_ONLY_MESSAGE); } + if (device.platform === 'android' && context?.launchArgs && context.launchArgs.length > 0) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Launch arguments are currently supported only on Apple platforms.', + ); + } if (context?.maestroClearState) { if (isDeepLinkTarget(app)) { throw new AppError( diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index cb12bc227..6d9dc7453 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -447,10 +447,10 @@ test('runReplayScriptFile retries Maestro scrollUntilVisible with scroll probes' [ ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], ['find', ['Discover', 'wait', '500']], - ['scroll', ['down']], + ['scroll', ['up']], ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], ['find', ['Discover', 'wait', '500']], - ['scroll', ['down']], + ['scroll', ['up']], ['wait', ['label="Discover" || text="Discover" || id="Discover"', '200']], ], ); @@ -695,6 +695,61 @@ test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absen ); }); +test('runReplayScriptFile skips Maestro runFlow.when.visible commands on false predicate', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-false-skip', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { ok: true, data: { pass: false } }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']]], + ); +}); + +test('runReplayScriptFile propagates Maestro runFlow.when runtime errors', async () => { + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-runtime-error', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async () => ({ + ok: false, + error: { code: 'UNKNOWN', message: 'fetch failed' }, + }), + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.equal(response.error.code, 'UNKNOWN'); + assert.match(response.error.message, /fetch failed/); + } +}); + test('runReplayScriptFile runs Maestro runFlow.when.visible commands when present', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ @@ -733,6 +788,42 @@ test('runReplayScriptFile runs Maestro runFlow.when.visible commands when presen ); }); +test('runReplayScriptFile runs nested Maestro runtime commands inside runFlow.when', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-nested-runtime', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Feed', + ' commands:', + ' - scrollUntilVisible:', + ' element: Done', + ' direction: DOWN', + ' timeout: 500', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'is') return { ok: true, data: { pass: true } }; + if (req.command === 'wait') return { ok: true, data: { found: true } }; + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['is', ['visible', 'label="Feed" || text="Feed" || id="Feed"']], + ['wait', ['label="Done" || text="Done" || id="Done"', '500']], + ], + ); +}); + test('runReplayScriptFile reads shell env from request (client-collected), not daemon process.env', async () => { // Ensure the daemon's own process.env does NOT contain AD_VAR_APP. assert.equal(process.env.AD_VAR_APP, undefined); diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts index 10b7d1db3..56cac5e6a 100644 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -33,6 +33,10 @@ type MaestroTapOnParams = { invoke: MaestroRuntimeInvoke; }; +type MaestroRunFlowWhenCondition = + | { ok: true; mode: string; predicate: string; selector: string } + | { ok: false; response: DaemonResponse }; + export async function invokeMaestroRuntimeCommand(params: { command: string; baseReq: ReplayBaseRequest; @@ -254,26 +258,56 @@ async function invokeMaestroRunFlowWhen(params: { invoke: (req: DaemonRequest) => Promise; invokeReplayAction: MaestroReplayInvoker; }): Promise { - const [mode, selector] = params.positionals; - if ((mode !== 'visible' && mode !== 'notVisible') || !selector) { - return errorResponse( - 'INVALID_ARGS', - 'runFlow.when requires visible/notVisible and a selector.', - ); - } - const predicate = mode === 'visible' ? 'visible' : 'hidden'; + const condition = readMaestroRunFlowWhenCondition(params.positionals); + if (!condition.ok) return condition.response; const conditionResponse = await params.invoke({ ...params.baseReq, command: 'is', - positionals: [predicate, selector], + positionals: [condition.predicate, condition.selector], flags: { ...params.baseReq.flags, noRecord: true }, }); - if (!conditionResponse.ok) { - return { ok: true, data: { skipped: true, condition: mode, selector } }; + if (isMaestroWhenConditionMiss(conditionResponse)) { + return { + ok: true, + data: { skipped: true, condition: condition.mode, selector: condition.selector }, + }; } + if (!conditionResponse.ok) return conditionResponse; + return await invokeMaestroRunFlowWhenSteps(params, condition); +} +function readMaestroRunFlowWhenCondition(positionals: string[]): MaestroRunFlowWhenCondition { + const [mode, selector] = positionals; + if ((mode !== 'visible' && mode !== 'notVisible') || !selector) { + return { + ok: false, + response: errorResponse( + 'INVALID_ARGS', + 'runFlow.when requires visible/notVisible and a selector.', + ), + }; + } + return { + ok: true, + mode, + predicate: mode === 'visible' ? 'visible' : 'hidden', + selector, + }; +} + +async function invokeMaestroRunFlowWhenSteps( + params: { + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invokeReplayAction: MaestroReplayInvoker; + }, + condition: Extract, +): Promise { const steps = (params.batchSteps ?? []).map(batchStepToSessionAction); for (const [index, action] of steps.entries()) { + // Preserve stable parent-step ordering for nested runtime commands while + // keeping the substep distinguishable in traces. const response = await params.invokeReplayAction({ action, line: params.line, @@ -282,7 +316,16 @@ async function invokeMaestroRunFlowWhen(params: { if (!response.ok) return response; } - return { ok: true, data: { ran: steps.length, condition: mode, selector } }; + return { + ok: true, + data: { ran: steps.length, condition: condition.mode, selector: condition.selector }, + }; +} + +function isMaestroWhenConditionMiss(response: DaemonResponse): boolean { + if (response.ok) return response.data?.pass === false; + if (response.error.code !== 'COMMAND_FAILED') return false; + return response.error.details?.blockedBy !== 'android_foreground_surface'; } async function invokeMaestroPressEnter(params: { @@ -298,6 +341,9 @@ async function invokeMaestroPressEnter(params: { const message = response.error.message.toLowerCase(); if (!message.includes('fetch failed')) return response; + // Maestro compatibility: some iOS apps submit on Enter and immediately reset + // the runner transport. Treat this as recovered only after a fresh snapshot + // proves the runner connection is usable again; it does not assert UI state. const snapshotResponse = await params.invoke({ ...params.baseReq, command: 'snapshot', @@ -333,6 +379,8 @@ function extractMaestroVisibleTextQuery(selectorExpression: string): string | nu const chain = parseSelectorChain(selectorExpression); const terms = chain.selectors.flatMap((selector) => selector.terms); if (terms.length === 0) return null; + // Mixed selectors may encode more than a visible-text lookup, so they keep + // the exact selector path instead of fuzzy text fallback. if (!terms.some((term) => term.key === 'label' || term.key === 'text')) return null; if (!terms.every((term) => ['label', 'text', 'id'].includes(term.key))) return null; const values = terms.map((term) => (typeof term.value === 'string' ? term.value : '')); diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index ff6f42315..eb295a916 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -1263,7 +1263,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--maestro', usageDescription: - 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible, runScript file/env with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + + 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with Apple-platform launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible, runScript file/env with parse-time http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + 'Unsupported syntax fails loudly with a link to https://github.com/callstackincubator/agent-device/issues/558', }, { diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 24b264501..7846bf503 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -61,11 +61,11 @@ Maestro compatibility translates supported YAML commands into Agent Device repla - Supported and unsupported capabilities: https://github.com/callstackincubator/agent-device/issues/558 - New focused compatibility request: https://github.com/callstackincubator/agent-device/issues/new -Currently supported areas include app launch with launch arguments but without state-reset side effects, file and inline `runFlow` with `when.platform`, `when.visible`, and `when.notVisible`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional` and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as a Maestro compatibility feature for file/env scripts that use `http.post`, `json`, and `output` variables; it is not a native `.ad` command. +Currently supported areas include app launch with Apple-platform launch arguments and iOS simulator `clearState`, file and inline `runFlow` with `when.platform`, `when.visible`, and `when.notVisible`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional` and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as a Maestro compatibility feature for file/env scripts that use `http.post`, `json`, and `output` variables; it executes during flow parsing, can make network requests, and is not a native `.ad` command or security sandbox. Maestro `env` values use the same replay precedence as `.ad` files: flow `env` is the default, shell `AD_VAR_*` values override it, and CLI `-e KEY=VALUE` wins over both. -Unsupported Maestro features such as `repeat.while`, `runFlow.when.true`, full expression predicates, `evalScript`, text clearing, selector relations such as `index` / `childOf`, device utility commands, and app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. +Unsupported Maestro features such as `repeat.while`, `runFlow.when.true`, full expression predicates, `evalScript`, text clearing, selector relations such as `index` / `childOf`, device utility commands, Android app launch arguments, and Android app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. ## Run a lightweight `.ad` suite From 3d9ae49e5bd4cf05ac7f9cba9738a4b22f5c8873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 26 May 2026 09:07:03 +0200 Subject: [PATCH 4/8] fix: finalize Maestro replay compatibility --- .../RunnerTests+CommandExecution.swift | 54 +- .../RunnerTests+Interaction.swift | 218 +++++- .../RunnerTests+Models.swift | 1 + src/__tests__/client.test.ts | 15 + src/backend.ts | 2 +- src/cli/commands/client-command.ts | 13 +- src/cli/commands/generic.ts | 1 + src/client-types.ts | 5 +- src/commands/selector-read.ts | 14 +- src/commands/system.ts | 31 +- .../maestro/__tests__/replay-flow.test.ts | 175 ++++- src/compat/maestro/command-mapper.ts | 21 +- src/compat/maestro/device-actions.ts | 7 +- src/compat/maestro/flow-control.ts | 5 +- src/compat/maestro/interactions.ts | 113 ++- src/compat/maestro/replay-flow.ts | 78 ++- src/compat/maestro/run-script.ts | 54 +- src/compat/maestro/runtime-commands.ts | 6 +- src/core/__tests__/dispatch-keyboard.test.ts | 48 ++ src/core/__tests__/dispatch-open.test.ts | 45 +- src/core/capabilities.ts | 2 +- src/core/dispatch-context.ts | 14 +- src/core/dispatch-interactions.ts | 31 + src/core/dispatch.ts | 42 +- src/core/interactor-types.ts | 5 + src/daemon-client.ts | 10 +- src/daemon/__tests__/context.test.ts | 12 +- src/daemon/context.ts | 3 +- src/daemon/handlers/__tests__/find.test.ts | 60 +- .../interaction-touch-targets.test.ts | 13 + .../handlers/__tests__/interaction.test.ts | 45 +- .../__tests__/session-replay-vars.test.ts | 657 +++++++++++++++++- src/daemon/handlers/find.ts | 51 +- .../handlers/interaction-touch-targets.ts | 5 +- src/daemon/handlers/interaction-touch.ts | 84 ++- .../handlers/session-replay-action-runtime.ts | 148 ++++ .../session-replay-maestro-runtime.ts | 623 +++++++++++++++-- src/daemon/handlers/session-replay-runtime.ts | 112 +-- src/daemon/handlers/session.ts | 6 +- src/platforms/android/input-actions.ts | 4 + .../ios/__tests__/runner-client.test.ts | 114 +++ src/platforms/ios/interactions.ts | 17 + src/platforms/ios/runner-contract.ts | 1 + src/replay/vars.ts | 32 +- src/utils/__tests__/args.test.ts | 15 +- src/utils/command-schema.ts | 13 +- website/docs/docs/replay-e2e.md | 4 +- 47 files changed, 2673 insertions(+), 356 deletions(-) create mode 100644 src/core/__tests__/dispatch-keyboard.test.ts create mode 100644 src/daemon/handlers/session-replay-action-runtime.ts diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index c7dd5cf96..5556202a3 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -742,6 +742,25 @@ extension RunnerTests { dismissed: result.dismissed ) ) + case .keyboardReturn: + let result = pressKeyboardReturn(app: activeApp) + if !result.pressed { + return Response( + ok: false, + error: ErrorPayload( + code: "UNSUPPORTED_OPERATION", + message: "Unable to press the iOS keyboard return key" + ) + ) + } + return Response( + ok: true, + data: DataPayload( + message: "keyboardReturn", + visible: result.visible, + wasVisible: result.wasVisible + ) + ) case .alert: let action = (command.action ?? "get").lowercased() guard let alert = resolveAlert(app: activeApp) else { @@ -852,7 +871,27 @@ extension RunnerTests { } let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0 let textEntryMode = resolveTextEntryMode(command) - let target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) + let target: TextEntryTarget + if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue { + let match = findElement( + app: activeApp, + selectorKey: selectorKey, + selectorValue: selectorValue, + allowNonHittableFallback: command.allowNonHittableSelectorTap == true + ) + if match.isAmbiguous { + return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) + } + guard let element = match.element else { + return Response(ok: false, error: ErrorPayload(code: "NO_MATCH", message: "selector did not match an element")) + } + guard isTextEntryElement(element) else { + return Response(ok: false, error: ErrorPayload(code: "INVALID_TARGET", message: "selector did not match a text input")) + } + target = focusTextInputForTextEntry(app: activeApp, element: element) + } else { + target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) + } if textEntryMode == .replacement { guard target.element != nil else { let message = @@ -880,6 +919,17 @@ extension RunnerTests { ) ) } - return Response(ok: true, data: DataPayload(message: textResult.repaired ? "typed after repair" : "typed")) + let point = target.refreshPoint + let frame = activeApp.frame + return Response( + ok: true, + data: DataPayload( + message: textResult.repaired ? "typed after repair" : "typed", + x: point.map { Double($0.x) }, + y: point.map { Double($0.y) }, + referenceWidth: frame.isEmpty ? nil : Double(frame.width), + referenceHeight: frame.isEmpty ? nil : Double(frame.height) + ) + ) } } diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index a09115239..b07e4145f 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -335,7 +335,7 @@ extension RunnerTests { switch element.elementType { case .textField, .secureTextField, .searchField, .textView: let frame = element.frame - return !frame.isEmpty && frame.contains(point) + return !frame.isEmpty && frameContainsPoint(frame, point, tolerance: 2) default: return false } @@ -366,20 +366,31 @@ extension RunnerTests { return matched } + private func frameContainsPoint(_ frame: CGRect, _ point: CGPoint, tolerance: CGFloat) -> Bool { + point.x >= frame.minX - tolerance + && point.x <= frame.maxX + tolerance + && point.y >= frame.minY - tolerance + && point.y <= frame.maxY + tolerance + } + func focusedTextInput(app: XCUIApplication) -> XCUIElement? { +#if os(iOS) + return nil +#else var focused: XCUIElement? let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ - let candidate = app + let candidates = app .descendants(matching: .any) .matching(NSPredicate(format: "hasKeyboardFocus == 1")) - .firstMatch - guard candidate.exists else { return } - - switch candidate.elementType { - case .textField, .secureTextField, .searchField, .textView: - focused = candidate - default: - return + .allElementsBoundByIndex + for candidate in candidates where candidate.exists { + switch candidate.elementType { + case .textField, .secureTextField, .searchField, .textView: + focused = candidate + return + default: + continue + } } }) if let exceptionMessage { @@ -390,6 +401,7 @@ extension RunnerTests { return nil } return focused +#endif } func stabilizeTextInputBeforeTyping(app: XCUIApplication, target: XCUIElement?) -> XCUIElement? { @@ -449,6 +461,36 @@ extension RunnerTests { ) } + func focusTextInputForTextEntry(app: XCUIApplication, element: XCUIElement) -> TextEntryTarget { + let point = textEntryRefreshPoint(for: element) + if let point { + _ = tapAt(app: app, x: point.x, y: point.y) + } + let stabilized = stabilizeTextInputBeforeTyping(app: app, target: element) + let resolved = waitForTextEntryReadiness( + app: app, + target: TextEntryTarget( + element: stabilized ?? element, + refreshPoint: point, + prefersFocusedElement: false + ) + ) ?? stabilized ?? element + return TextEntryTarget( + element: resolved, + refreshPoint: textEntryRefreshPoint(for: resolved) ?? point, + prefersFocusedElement: false + ) + } + + func isTextEntryElement(_ element: XCUIElement) -> Bool { + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + return true + default: + return false + } + } + func resolveTextEntryMode(_ command: Command) -> TextTypingRepairMode { switch command.textEntryMode { case "append": @@ -629,7 +671,7 @@ extension RunnerTests { guard let observedText = editableTextValue(for: targetElement) else { return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil) } - guard observedText == expectedText else { + guard textEntryValueMatchesExpected(targetElement, observedText: observedText, expectedText: expectedText) else { return TextEntryResult( verified: false, repaired: repaired, @@ -645,7 +687,11 @@ extension RunnerTests { return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil) } latestObservedText = nextObservedText - guard nextObservedText == expectedText else { + guard textEntryValueMatchesExpected( + resolveTextEntryElement(app: app, target: target), + observedText: nextObservedText, + expectedText: expectedText + ) else { return TextEntryResult( verified: false, repaired: repaired, @@ -662,6 +708,28 @@ extension RunnerTests { ) } + private func textEntryValueMatchesExpected( + _ element: XCUIElement?, + observedText: String, + expectedText: String + ) -> Bool { + if observedText == expectedText { + return true + } + guard hasTextEntrySubmitSuffix(expectedText), element?.elementType != .textView else { + return false + } + var submittedText = expectedText + while hasTextEntrySubmitSuffix(submittedText) { + submittedText.removeLast() + } + return observedText == submittedText + } + + private func hasTextEntrySubmitSuffix(_ text: String) -> Bool { + text.hasSuffix("\n") || text.hasSuffix("\r") + } + private func expectedTextEntryValue( typedText: String, mode: TextTypingRepairMode, @@ -693,7 +761,11 @@ extension RunnerTests { guard let observedText = editableTextValue(for: resolveTextEntryElement(app: app, target: target)) else { return false } - if observedText == expectedText { + if textEntryValueMatchesExpected( + resolveTextEntryElement(app: app, target: target), + observedText: observedText, + expectedText: expectedText + ) { return false } latestObservedText = observedText @@ -710,7 +782,11 @@ extension RunnerTests { guard let latestObservedText else { return false } - guard latestObservedText != expectedText else { + guard !textEntryValueMatchesExpected( + resolveTextEntryElement(app: app, target: target), + observedText: latestObservedText, + expectedText: expectedText + ) else { return false } return isRepairableTextEntryMismatch( @@ -904,6 +980,85 @@ extension RunnerTests { #endif } + func pressKeyboardReturn(app: XCUIApplication) -> (wasVisible: Bool, pressed: Bool, visible: Bool) { +#if os(tvOS) + return (wasVisible: false, pressed: pressTvRemote(.select), visible: false) +#elseif os(iOS) + let wasVisible = isKeyboardVisible(app: app) + if tapKeyboardReturnControl(app: app) { + sleepFor(0.2) + return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app)) + } + + var typed = false + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + app.typeText(XCUIKeyboardKey.return.rawValue) + typed = true + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_RETURN_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + if let singleTarget = singleTextEntryElement(app: app) { + return pressKeyboardReturn(on: singleTarget, app: app, wasVisible: wasVisible) + } + return (wasVisible: wasVisible, pressed: false, visible: isKeyboardVisible(app: app)) + } + sleepFor(0.2) + return (wasVisible: wasVisible, pressed: typed, visible: isKeyboardVisible(app: app)) +#else + return (wasVisible: false, pressed: false, visible: false) +#endif + } + + private func pressKeyboardReturn( + on element: XCUIElement, + app: XCUIApplication, + wasVisible: Bool + ) -> (wasVisible: Bool, pressed: Bool, visible: Bool) { + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + element.tap() + element.typeText(XCUIKeyboardKey.return.rawValue) + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_RETURN_TARGET_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return (wasVisible: wasVisible, pressed: false, visible: isKeyboardVisible(app: app)) + } + sleepFor(0.2) + return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app)) + } + + private func singleTextEntryElement(app: XCUIApplication) -> XCUIElement? { +#if os(iOS) + var matches: [XCUIElement] = [] + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + matches = app.descendants(matching: .any).allElementsBoundByIndex.filter { element in + guard element.exists else { return false } + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + return true + default: + return false + } + } + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_RETURN_TEXT_ENTRY_QUERY_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return nil + } + return matches.count == 1 ? matches[0] : nil +#else + return nil +#endif + } + private func tapKeyboardDismissControl(app: XCUIApplication) -> Bool { #if os(tvOS) return false @@ -941,6 +1096,22 @@ extension RunnerTests { #endif } + private func tapKeyboardReturnControl(app: XCUIApplication) -> Bool { +#if os(iOS) + for label in ["return", "Return", "Enter", "Go", "Search", "Next", "Done", "Send", "Join"] { + let candidates = [ + app.keyboards.buttons[label], + app.keyboards.keys[label], + ] + if let hittable = candidates.first(where: { $0.exists && $0.isHittable }) { + hittable.tap() + return true + } + } +#endif + return false + } + private func isKeyboardAccessoryControl(_ element: XCUIElement, keyboardFrame: CGRect) -> Bool { let frame = element.frame guard !frame.isEmpty && !keyboardFrame.isEmpty else { @@ -1003,11 +1174,24 @@ extension RunnerTests { guard !normalizedValue.isEmpty else { return false } - guard let placeholder = element.placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines), - !placeholder.isEmpty else { + let placeholder = element.placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !placeholder.isEmpty && normalizedValue == placeholder { + return true + } + if isGenericTextInputLabel(normalizedValue) { + return true + } + let normalizedLabel = element.label.trimmingCharacters(in: .whitespacesAndNewlines) + return normalizedLabel == normalizedValue && isGenericTextInputLabel(normalizedLabel) + } + + private func isGenericTextInputLabel(_ value: String) -> Bool { + switch value { + case "Text input field": + return true + default: return false } - return normalizedValue == placeholder } private func readableText(for element: XCUIElement) -> String? { diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 13a295e69..0020cc81f 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -23,6 +23,7 @@ enum CommandType: String, Codable { case rotate case appSwitcher case keyboardDismiss + case keyboardReturn case alert case pinch case rotateGesture diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index 612c1b7b1..89791f066 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -253,6 +253,21 @@ test('replay.run keeps deprecated maestro option as backend alias', async () => assert.equal(setup.calls[0]?.flags?.replayBackend, 'maestro'); }); +test('replay.run forwards timeout budget', async () => { + const setup = createTransport(async () => ({ ok: true, data: {} })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await client.replay.run({ + path: './flows/mod-lists.yaml', + backend: 'maestro', + timeoutMs: 240_000, + }); + + assert.equal(setup.calls.length, 1); + assert.equal(setup.calls[0]?.command, 'replay'); + assert.equal(setup.calls[0]?.flags?.timeoutMs, 240_000); +}); + test('client.command.wait prepares selector options and rejects invalid selectors', async () => { const setup = createTransport(async () => ({ ok: true, diff --git a/src/backend.ts b/src/backend.ts index eae5c551f..9cd46589d 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -92,7 +92,7 @@ export type BackendBackOptions = { }; export type BackendKeyboardOptions = { - action: 'status' | 'get' | 'dismiss'; + action: 'status' | 'get' | 'dismiss' | 'enter' | 'return'; }; export type BackendKeyboardResult = { diff --git a/src/cli/commands/client-command.ts b/src/cli/commands/client-command.ts index 828846300..e52bddbf9 100644 --- a/src/cli/commands/client-command.ts +++ b/src/cli/commands/client-command.ts @@ -173,10 +173,19 @@ function readKeyboardAction( ): KeyboardCommandOptions['action'] | undefined { const action = value?.toLowerCase(); if (action === 'get') return 'status'; - if (action === undefined || action === 'status' || action === 'dismiss') { + if ( + action === undefined || + action === 'status' || + action === 'dismiss' || + action === 'enter' || + action === 'return' + ) { return action; } - throw new AppError('INVALID_ARGS', 'keyboard action must be status, get, or dismiss.'); + throw new AppError( + 'INVALID_ARGS', + 'keyboard action must be status, get, dismiss, enter, or return.', + ); } function readFiniteNumber(value: string | undefined, label: string): number | undefined { diff --git a/src/cli/commands/generic.ts b/src/cli/commands/generic.ts index b75902085..8fa93ad0e 100644 --- a/src/cli/commands/generic.ts +++ b/src/cli/commands/generic.ts @@ -61,6 +61,7 @@ const genericClientCommandRunners = { update: flags.replayUpdate, backend: flags.replayMaestro ? 'maestro' : undefined, env: flags.replayEnv, + timeoutMs: flags.timeoutMs, }), test: ({ client, positionals, flags }) => { announceReplayTestRun({ json: flags.json }); diff --git a/src/client-types.ts b/src/client-types.ts index 2a44be877..8d28aab89 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -389,7 +389,7 @@ export type RotateCommandOptions = DeviceCommandBaseOptions & { export type AppSwitcherCommandOptions = DeviceCommandBaseOptions; export type KeyboardCommandOptions = DeviceCommandBaseOptions & { - action?: 'status' | 'dismiss'; + action?: 'status' | 'dismiss' | 'enter' | 'return'; }; export type ClipboardCommandOptions = @@ -449,7 +449,7 @@ export type AppSwitcherCommandResult = CommandActionResult<'app-switcher'>; export type KeyboardCommandResult = DaemonResponseData & { platform?: 'android' | 'ios'; - action?: 'status' | 'dismiss'; + action?: 'status' | 'dismiss' | 'enter'; visible?: boolean; inputType?: string | null; inputMethodPackage?: string | null; @@ -681,6 +681,7 @@ export type ReplayRunOptions = AgentDeviceRequestOverrides & { maestro?: boolean; backend?: string; env?: string[]; + timeoutMs?: number; }; export type ReplayTestOptions = AgentDeviceRequestOverrides & diff --git a/src/commands/selector-read.ts b/src/commands/selector-read.ts index b8443d27f..526ec7d39 100644 --- a/src/commands/selector-read.ts +++ b/src/commands/selector-read.ts @@ -303,7 +303,12 @@ export const isCommand: RuntimeCommand = asyn disambiguateAmbiguous: false, }); if (!resolved) { - throw new AppError('COMMAND_FAILED', formatSelectorFailure(chain, [], { unique: true })); + throw new AppError('COMMAND_FAILED', formatSelectorFailure(chain, [], { unique: true }), { + command: 'is', + reason: 'selector_not_found', + predicate: options.predicate, + selector: chain.raw, + }); } const result = evaluateIsPredicate({ predicate: options.predicate, @@ -316,6 +321,13 @@ export const isCommand: RuntimeCommand = asyn throw new AppError( 'COMMAND_FAILED', `is ${options.predicate} failed for selector ${resolved.selector.raw}: ${result.details}`, + { + command: 'is', + reason: 'predicate_failed', + predicate: options.predicate, + selector: resolved.selector.raw, + predicateDetails: result.details, + }, ); } return { diff --git a/src/commands/system.ts b/src/commands/system.ts index adbfe7f01..20d234b90 100644 --- a/src/commands/system.ts +++ b/src/commands/system.ts @@ -44,7 +44,7 @@ export type SystemRotateCommandResult = { }; export type SystemKeyboardCommandOptions = CommandContext & { - action?: 'status' | 'get' | 'dismiss'; + action?: 'status' | 'get' | 'dismiss' | 'enter' | 'return'; }; export type SystemKeyboardCommandResult = @@ -60,6 +60,13 @@ export type SystemKeyboardCommandResult = state: BackendKeyboardResult; backendResult?: Record; message?: string; + } + | { + kind: 'keyboardEnterPressed'; + action: 'enter'; + state: BackendKeyboardResult; + backendResult?: Record; + message?: string; }; export type SystemClipboardCommandOptions = @@ -200,11 +207,29 @@ export const keyboardCommand: RuntimeCommand< throw new AppError('UNSUPPORTED_OPERATION', 'system.keyboard is not supported by this backend'); } const action = options.action ?? 'status'; - if (action !== 'status' && action !== 'get' && action !== 'dismiss') { - throw new AppError('INVALID_ARGS', 'system.keyboard action must be status, get, or dismiss'); + if ( + action !== 'status' && + action !== 'get' && + action !== 'dismiss' && + action !== 'enter' && + action !== 'return' + ) { + throw new AppError( + 'INVALID_ARGS', + 'system.keyboard action must be status, get, dismiss, enter, or return', + ); } const state = await runtime.backend.setKeyboard(toBackendContext(runtime, options), { action }); const formattedBackendResult = toBackendResult(state); + if (action === 'enter' || action === 'return') { + return { + kind: 'keyboardEnterPressed', + action: 'enter', + state: isKeyboardResult(state) ? state : {}, + ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + ...successText('Keyboard enter pressed'), + }; + } if (action === 'dismiss') { const dismissed = isKeyboardResult(state) ? state.dismissed : undefined; return { diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index f54d670af..27db41a1b 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -16,6 +16,7 @@ env: id: home-open-form - tapOn: point: 20%,20% + label: Dismiss save password prompt - doubleTapOn: id: release-notice delay: 150 @@ -62,7 +63,7 @@ env: ['__maestroTapOn', ['label="Full name" || text="Full name" || id="Full name"']], ['type', ['Ada Lovelace']], ['wait', ['label="Checkout form"', '5000']], - ['is', ['hidden', 'label="Missing banner"']], + ['__maestroAssertNotVisible', ['label="Missing banner"']], ['wait', ['id="submit-order"', '7000']], ['scroll', ['down']], ['scroll', ['down', '0.4']], @@ -79,8 +80,8 @@ env: assert.equal(parsed.actions[3]?.flags.doubleTap, true); assert.equal(parsed.actions[3]?.flags.intervalMs, 150); assert.equal(parsed.actions[4]?.flags.holdMs, 3000); - assert.equal(parsed.actions[1]?.flags.allowNonHittableSelectorTap, true); - assert.equal(parsed.actions[6]?.flags?.allowNonHittableSelectorTap, undefined); + assert.equal(parsed.actions[1]?.flags.maestro?.allowNonHittableSelectorTap, true); + assert.equal(parsed.actions[6]?.flags?.maestro?.allowNonHittableSelectorTap, undefined); }); test('parseMaestroReplayFlow maps iOS openLink through the app id when available', () => { @@ -98,17 +99,50 @@ test('parseMaestroReplayFlow maps iOS openLink through the app id when available ); }); -test('parseMaestroReplayFlow executes runScript and exposes output variables', () => { +test('parseMaestroReplayFlow converts Bluesky Maestro selector compatibility syntax', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- eraseText +- eraseText: 12 +- tapOn: + id: likeBtn + childOf: + id: postThreadItem-by-bob.test +- tapOn: + id: postDropdownBtn + index: 0 +- tapOn: + label: Display name metadata + text: Display name +- swipe: + label: Drag feed down + from: + id: feed-drag-handle + direction: UP + duration: 350 +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['type', ['\b'.repeat(50)]], + ['type', ['\b'.repeat(12)]], + [ + '__maestroTapOn', + ['id="likeBtn"', JSON.stringify({ childOf: 'id="postThreadItem-by-bob.test"' })], + ], + ['__maestroTapOn', ['id="postDropdownBtn"', JSON.stringify({ index: 0 })]], + ['__maestroTapOn', ['label="Display name"']], + ['__maestroSwipeOn', ['id="feed-drag-handle"', 'up', '350']], + ], + ); +}); + +test('parseMaestroReplayFlow preserves runScript as an ordered runtime action', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-runscript-')); const scriptPath = path.join(root, 'setup.js'); const flowPath = path.join(root, 'flow.yml'); - fs.writeFileSync( - scriptPath, - ` -var res = {body: '{"appviewDid":"did:plc:test"}'} -output.result = SERVER_PATH + ':' + json(res.body).appviewDid -`, - ); + fs.writeFileSync(scriptPath, `output.result = SERVER_PATH`); const parsed = parseMaestroReplayFlow( `appId: com.callstack.agentdevicelab @@ -124,30 +158,83 @@ output.result = SERVER_PATH + ':' + json(res.body).appviewDid assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), - [['type', ['local:did:plc:test']]], + [ + ['__maestroRunScript', [scriptPath]], + ['type', ['${output.result}']], + ], ); + assert.deepEqual(parsed.actions[0]?.flags.maestro?.runScriptEnv, { SERVER_PATH: 'local' }); }); -test('parseMaestroReplayFlow reports runScript http failures with command context', () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-runscript-fail-')); - const scriptPath = path.join(root, 'setup.js'); - const flowPath = path.join(root, 'flow.yml'); - fs.writeFileSync(scriptPath, `output.result = http.post('http://127.0.0.1:1').body`); +test('parseMaestroReplayFlow keeps focused inputText and pressKey Enter as separate actions', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- inputText: hello +- pressKey: Enter +- inputText: world +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['type', ['hello']], + ['__maestroPressEnter', []], + ['type', ['world']], + ], + ); + assert.deepEqual(parsed.actionLines, [3, 4, 5]); +}); + +test('parseMaestroReplayFlow marks tapOn before inputText for snapshot tap focus', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- tapOn: + id: editListNameInput +- inputText: Muted Users +`); + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['__maestroTapOn', ['id="editListNameInput"']], + ['type', ['Muted Users']], + ], + ); + assert.equal(parsed.actions[0]?.flags?.maestro?.allowNonHittableSelectorTap, undefined); +}); + +test('parseMaestroReplayFlow coalesces tapOn inputText while preserving pressKey Enter submit', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- tapOn: + id: e2eProxyHeaderInput +- inputText: \${output.result} +- pressKey: Enter +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['wait', ['id="e2eProxyHeaderInput"', '30000']], + ['fill', ['id="e2eProxyHeaderInput"', '${output.result}']], + ['__maestroPressEnter', []], + ], + ); + assert.deepEqual(parsed.actionLines, [3, 3, 6]); + assert.equal(parsed.actions[1]?.flags?.maestro?.allowNonHittableSelectorTap, true); +}); + +test('parseMaestroReplayFlow rejects relative runScript paths without source path', () => { assert.throws( () => - parseMaestroReplayFlow( - `appId: com.callstack.agentdevicelab + parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- - runScript: ./setup.js -`, - { sourcePath: flowPath }, - ), +`), (error) => error instanceof AppError && - error.code === 'COMMAND_FAILED' && - /runScript failed/.test(error.message) && - /http\.post failed/.test(error.message), + error.code === 'INVALID_ARGS' && + /runScript file paths/.test(error.message), ); }); @@ -343,7 +430,7 @@ test('parseMaestroReplayFlow keeps visible-gated runFlow commands for runtime ev { command: '__maestroTapOn', positionals: ['label="Continue" || text="Continue" || id="Continue"'], - flags: {}, + flags: { maestro: { allowNonHittableSelectorTap: true } }, }, ]); }); @@ -353,7 +440,6 @@ test('parseMaestroReplayFlow accepts launchApp reset options', () => { --- - launchApp: clearState: true - clearKeychain: true arguments: "-EXDevMenuIsOnboardingFinished": true launchArguments: @@ -368,8 +454,7 @@ test('parseMaestroReplayFlow accepts launchApp reset options', () => { 'open', ['com.callstack.agentdevicelab'], { - maestroClearState: true, - relaunch: true, + maestro: { clearState: true }, launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true', '-Example', 'ignored'], }, ], @@ -377,6 +462,38 @@ test('parseMaestroReplayFlow accepts launchApp reset options', () => { ); }); +test('parseMaestroReplayFlow rejects clearKeychain instead of ignoring it', () => { + assert.throws( + () => + parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- launchApp: + clearKeychain: true +`), + (error) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /clearKeychain/.test(error.message), + ); +}); + +test('parseMaestroReplayFlow relaunches launchApp only when clearState is absent', () => { + const withLaunchArgs = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- launchApp: + arguments: + "-Example": "value" +`); + const withStopApp = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- launchApp: + stopApp: true +`); + + assert.equal(withLaunchArgs.actions[0]?.flags.relaunch, true); + assert.equal(withStopApp.actions[0]?.flags.relaunch, true); +}); + test('parseMaestroReplayFlow rejects unsupported runtime-dependent flow control', () => { assert.throws( () => diff --git a/src/compat/maestro/command-mapper.ts b/src/compat/maestro/command-mapper.ts index 0d1f12c81..30e36d60d 100644 --- a/src/compat/maestro/command-mapper.ts +++ b/src/compat/maestro/command-mapper.ts @@ -3,6 +3,7 @@ import { AppError } from '../../utils/errors.ts'; import { convertLaunchApp, convertStopApp } from './device-actions.ts'; import { convertDoubleTapOn, + convertEraseText, convertExtendedWaitUntil, convertLongPressOn, convertPressKey, @@ -22,7 +23,8 @@ import { unsupportedCommand, } from './support.ts'; import { convertRepeat, convertRunFlow } from './flow-control.ts'; -import { executeRunScript } from './run-script.ts'; +import { convertRunScript } from './run-script.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import type { MaestroCommand, MaestroCommandMapperDeps, @@ -46,6 +48,7 @@ const MAP_COMMAND_HANDLERS: Record = { inputText: ({ value, context }) => [ action('type', [resolveMaestroString(readInputText(value), context)]), ], + eraseText: ({ value }) => [convertEraseText(value)], pasteText: ({ value, context, name }) => [ action('type', [resolveMaestroString(requireStringValue(name, value), context)]), ], @@ -54,7 +57,7 @@ const MAP_COMMAND_HANDLERS: Record = { action('wait', [maestroSelector(value, name, [], context), '5000']), ], assertNotVisible: ({ value, context, name }) => [ - action('is', ['hidden', maestroSelector(value, name, [], context)]), + action(MAESTRO_RUNTIME_COMMAND.assertNotVisible, [maestroSelector(value, name, [], context)]), ], extendedWaitUntil: ({ value, context }) => convertExtendedWaitUntil(value, context), takeScreenshot: ({ value, context, name }) => [ @@ -62,16 +65,15 @@ const MAP_COMMAND_HANDLERS: Record = { ], scroll: ({ value }) => [convertScroll(value)], scrollUntilVisible: ({ value, context }) => convertScrollUntilVisible(value, context), - swipe: ({ value }) => [convertSwipe(value)], + swipe: ({ value, context }) => [convertSwipe(value, context)], hideKeyboard: () => [action('keyboard', ['dismiss'])], pressKey: ({ value }) => [convertPressKey(value)], back: () => [action('back')], - waitForAnimationToEnd: ({ value }) => [action('wait', [String(readTimeoutMs(value, 250))])], + waitForAnimationToEnd: ({ value }) => [ + action(MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd, [String(readTimeoutMs(value, 15000))]), + ], stopApp: ({ value, config, context }) => [convertStopApp(value, config, context)], - runScript: ({ value, context }) => { - executeRunScript(value, context); - return []; - }, + runScript: ({ value, context }) => [convertRunScript(value, context)], runFlow: ({ value, config, context, deps }) => convertRunFlow(value, config, context, deps, convertCommandList), repeat: ({ value, config, context, deps }) => @@ -85,8 +87,9 @@ const SCALAR_COMMAND_HANDLERS: Record< launchApp: (config, context) => [convertLaunchApp(undefined, config, context)], scroll: () => [action('scroll', ['down'])], hideKeyboard: () => [action('keyboard', ['dismiss'])], + eraseText: () => [convertEraseText(undefined)], back: () => [action('back')], - waitForAnimationToEnd: () => [action('wait', ['250'])], + waitForAnimationToEnd: () => [action(MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd, ['15000'])], stopApp: (config, context) => [convertStopApp(undefined, config, context)], }; diff --git a/src/compat/maestro/device-actions.ts b/src/compat/maestro/device-actions.ts index 8dcaa62b0..b873acab2 100644 --- a/src/compat/maestro/device-actions.ts +++ b/src/compat/maestro/device-actions.ts @@ -32,16 +32,17 @@ export function convertLaunchApp( 'launchArguments', ]); rejectUnsupportedLaunchOption(value, 'permissions'); + rejectUnsupportedLaunchOption(value, 'clearKeychain'); const appId = resolveMaestroString( typeof value.appId === 'string' ? value.appId : requireAppId(config, 'launchApp'), context, ); const launchArgs = readLaunchArgs(value, context); const shouldClearState = value.clearState === true; - const shouldRelaunch = value.stopApp === true || shouldClearState || launchArgs.length > 0; + const shouldRelaunch = !shouldClearState && (value.stopApp === true || launchArgs.length > 0); return action('open', [appId], { - relaunch: shouldRelaunch, - ...(shouldClearState ? { maestroClearState: true } : {}), + ...(shouldRelaunch ? { relaunch: true } : {}), + ...(shouldClearState ? { maestro: { clearState: true } } : {}), ...(launchArgs.length > 0 ? { launchArgs } : {}), }); } diff --git a/src/compat/maestro/flow-control.ts b/src/compat/maestro/flow-control.ts index d11e8af87..513d0a1cb 100644 --- a/src/compat/maestro/flow-control.ts +++ b/src/compat/maestro/flow-control.ts @@ -20,7 +20,10 @@ import type { MaestroParseContext, } from './types.ts'; -const MAX_REPEAT_EXPANSIONS = 100; +// repeat.times is expanded at parse time for deterministic replay traces. Keep +// a guardrail until repeat can execute as a runtime loop without materializing +// every child action. +const MAX_REPEAT_EXPANSIONS = 1000; type ConvertCommandList = ( commands: MaestroCommand[], diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index 012e85226..eeb9f9695 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -17,14 +17,18 @@ import { import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import type { MaestroParseContext } from './types.ts'; +type SwipeDirection = 'up' | 'down' | 'left' | 'right'; + export function convertTapOn(value: unknown, context: MaestroParseContext): SessionAction { if (typeof value === 'string') { - return action(MAESTRO_RUNTIME_COMMAND.tapOn, [ - visibleTextSelector(resolveMaestroString(value, context)), - ]); + return action( + MAESTRO_RUNTIME_COMMAND.tapOn, + [visibleTextSelector(resolveMaestroString(value, context))], + maestroTapOnFlags(value), + ); } if (isPlainRecord(value) && typeof value.point === 'string') { - assertOnlyKeys(value, 'tapOn', ['point', 'repeat', 'delay']); + assertOnlyKeys(value, 'tapOn', ['point', 'repeat', 'delay', 'optional', 'label']); const point = parseMaestroPoint(value.point); if (point.kind === 'percent') { return action( @@ -39,7 +43,9 @@ export function convertTapOn(value: unknown, context: MaestroParseContext): Sess assertOnlyKeys(value, 'tapOn', [ 'id', 'text', + 'childOf', 'enabled', + 'index', 'selected', 'repeat', 'delay', @@ -47,10 +53,19 @@ export function convertTapOn(value: unknown, context: MaestroParseContext): Sess 'label', ]); } + const flags = maestroTapOnFlags(value); return action( MAESTRO_RUNTIME_COMMAND.tapOn, - [maestroSelector(value, 'tapOn', ['repeat', 'delay', 'optional', 'label'], context)], - { ...tapFlags(value), allowNonHittableSelectorTap: true }, + [ + maestroSelector( + value, + 'tapOn', + ['repeat', 'delay', 'optional', 'label', 'index', 'childOf'], + context, + ), + ...maestroTapOnRuntimeOptions(value, context), + ], + flags, ); } @@ -94,6 +109,26 @@ export function readInputText(value: unknown): string { return value.text; } +export function convertEraseText(value: unknown): SessionAction { + if (value === null || value === undefined) return action('type', ['\b'.repeat(50)]); + if (typeof value === 'number' && Number.isInteger(value) && value > 0) { + return action('type', ['\b'.repeat(value)]); + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'eraseText expects empty, a positive count, or a map.'); + } + assertOnlyKeys(value, 'eraseText', ['charactersToErase']); + if (value.charactersToErase === undefined) return action('type', ['\b'.repeat(50)]); + if ( + typeof value.charactersToErase !== 'number' || + !Number.isInteger(value.charactersToErase) || + value.charactersToErase <= 0 + ) { + throw new AppError('INVALID_ARGS', 'eraseText.charactersToErase must be a positive integer.'); + } + return action('type', ['\b'.repeat(value.charactersToErase)]); +} + export function convertExtendedWaitUntil( value: unknown, context: MaestroParseContext, @@ -147,11 +182,22 @@ export function convertScrollUntilVisible( return [action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [selector, timeoutMs, direction])]; } -export function convertSwipe(value: unknown): SessionAction { +export function convertSwipe(value: unknown, context: MaestroParseContext): SessionAction { if (!isPlainRecord(value)) { throw new AppError('INVALID_ARGS', 'swipe expects a map.'); } - assertOnlyKeys(value, 'swipe', ['start', 'end', 'direction', 'duration']); + assertOnlyKeys(value, 'swipe', ['start', 'end', 'direction', 'duration', 'from', 'label']); + const from = value.from ?? (typeof value.label === 'string' ? value.label : undefined); + if (from !== undefined) { + const direction = readSwipeDirection( + typeof value.direction === 'string' ? value.direction : 'up', + ); + return action(MAESTRO_RUNTIME_COMMAND.swipeOn, [ + maestroSelector(from, 'swipe.from', [], context), + direction, + ...swipeDurationPositionals(value), + ]); + } if (typeof value.direction === 'string') { return action('scroll', readScrollPositionalsFromDirectionSwipe(value.direction)); } @@ -182,7 +228,7 @@ export function convertSwipe(value: unknown): SessionAction { } function readScrollPositionalsFromDirectionSwipe(direction: string): string[] { - switch (direction.toLowerCase()) { + switch (readSwipeDirection(direction)) { case 'up': return ['down']; case 'down': @@ -191,6 +237,17 @@ function readScrollPositionalsFromDirectionSwipe(direction: string): string[] { return ['right']; case 'right': return ['left']; + } +} + +function readSwipeDirection(direction: string): SwipeDirection { + const normalized = direction.toLowerCase(); + switch (normalized) { + case 'up': + case 'down': + case 'left': + case 'right': + return normalized; default: throw unsupportedMaestroSyntax('Maestro swipe direction must be UP, DOWN, LEFT, or RIGHT.'); } @@ -235,6 +292,8 @@ export function maestroSelector( terms.push(selectorTerm('id', resolveMaestroString(value.id, context))); if (typeof value.text === 'string') terms.push(selectorTerm('label', resolveMaestroString(value.text, context))); + if (typeof value.label === 'string' && terms.length === 0) + terms.push(selectorTerm('label', resolveMaestroString(value.label, context))); if (typeof value.enabled === 'boolean') terms.push(selectorTerm('enabled', String(value.enabled))); if (typeof value.selected === 'boolean') @@ -242,7 +301,7 @@ export function maestroSelector( if (terms.length === 0) { throw new AppError( 'INVALID_ARGS', - `${command} selector map must include one of id, text, enabled, or selected.`, + `${command} selector map must include one of id, text, label, enabled, or selected.`, ); } return terms.join(' '); @@ -256,6 +315,27 @@ function visibleTextSelector(value: string): string { ].join(' || '); } +function maestroTapOnRuntimeOptions(value: unknown, context: MaestroParseContext): string[] { + if (!isPlainRecord(value)) return []; + const options: { index?: number; childOf?: string } = {}; + if (value.index !== undefined) { + if (typeof value.index !== 'number' || !Number.isInteger(value.index) || value.index < 0) { + throw new AppError('INVALID_ARGS', 'tapOn.index must be a non-negative integer.'); + } + options.index = value.index; + } + if (value.childOf !== undefined) { + options.childOf = maestroSelector(value.childOf, 'tapOn.childOf', [], context); + } + return Object.keys(options).length > 0 ? [JSON.stringify(options)] : []; +} + +function swipeDurationPositionals(value: Record): string[] { + return typeof value.duration === 'number' && Number.isFinite(value.duration) + ? [String(Math.max(16, Math.floor(value.duration)))] + : []; +} + function selectorTerm(key: string, value: string): string { return `${key}=${JSON.stringify(value)}`; } @@ -267,10 +347,21 @@ function tapFlags(value: unknown): SessionAction['flags'] | undefined { const delay = nonNegativeInteger(value.delay); if (repeat && repeat > 1) flags.count = repeat; if (delay !== undefined) flags.intervalMs = delay; - if (value.optional === true) flags.maestroOptional = true; + if (value.optional === true) flags.maestro = { optional: true }; return Object.keys(flags).length > 0 ? flags : undefined; } +function maestroTapOnFlags(value: unknown): SessionAction['flags'] { + const flags = tapFlags(value) ?? {}; + return { + ...flags, + maestro: { + ...(flags.maestro ?? {}), + allowNonHittableSelectorTap: true, + }, + }; +} + function doubleTapFlags(value: unknown): SessionAction['flags'] { const flags: SessionAction['flags'] = { doubleTap: true }; if (isPlainRecord(value) && typeof value.delay === 'number' && Number.isInteger(value.delay)) { diff --git a/src/compat/maestro/replay-flow.ts b/src/compat/maestro/replay-flow.ts index e6b28cdaa..c1b0f669c 100644 --- a/src/compat/maestro/replay-flow.ts +++ b/src/compat/maestro/replay-flow.ts @@ -4,6 +4,7 @@ import { parseAllDocuments } from 'yaml'; import type { SessionAction } from '../../daemon/types.ts'; import { AppError } from '../../utils/errors.ts'; import { convertMaestroCommandWithLine } from './command-mapper.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import { isPlainRecord, normalizeCommandList, normalizePlatform, readEnvMap } from './support.ts'; import type { MaestroCommand, @@ -74,7 +75,82 @@ function convertRootCommands(params: { actions.push(...converted); converted.forEach(() => actionLines.push(line)); } - return { actions, actionLines }; + return optimizeInputTextActions(actions, actionLines); +} + +function optimizeInputTextActions( + actions: SessionAction[], + actionLines: number[], +): { actions: SessionAction[]; actionLines: number[] } { + const maestroTapTimeoutMs = '30000'; + const mergedActions: SessionAction[] = []; + const mergedLines: number[] = []; + for (let index = 0; index < actions.length; index += 1) { + const action = actions[index]; + const nextAction = actions[index + 1]; + const typedAfterTap = readPlainTypeText(nextAction); + if (typedAfterTap !== null) { + const tapSelector = readPlainMaestroTapSelector(action); + const pressEnterAfterType = + actions[index + 2]?.command === MAESTRO_RUNTIME_COMMAND.pressEnter; + if (tapSelector !== null && pressEnterAfterType) { + mergedActions.push({ + ...action, + command: 'wait', + positionals: [tapSelector, maestroTapTimeoutMs], + }); + mergedLines.push(actionLines[index] ?? 1); + mergedActions.push({ + ...nextAction, + command: 'fill', + positionals: [tapSelector, typedAfterTap], + flags: action.flags, + }); + mergedLines.push(actionLines[index] ?? 1); + mergedActions.push(actions[index + 2] as SessionAction); + mergedLines.push(actionLines[index + 2] ?? actionLines[index] ?? 1); + index += 2; + continue; + } + if (tapSelector !== null) { + mergedActions.push(clearMaestroNonHittableTap(action)); + mergedLines.push(actionLines[index] ?? 1); + continue; + } + } + mergedActions.push(action); + mergedLines.push(actionLines[index] ?? 1); + } + return { actions: mergedActions, actionLines: mergedLines }; +} + +function clearMaestroNonHittableTap(action: SessionAction): SessionAction { + const maestro = { ...(action.flags?.maestro ?? {}) }; + delete maestro.allowNonHittableSelectorTap; + return { + ...action, + flags: { + ...(action.flags ?? {}), + maestro: { + ...maestro, + }, + }, + }; +} + +function readPlainMaestroTapSelector(action: SessionAction | undefined): string | null { + if (action?.command !== MAESTRO_RUNTIME_COMMAND.tapOn) return null; + const [selector, ...rest] = action.positionals ?? []; + if (rest.length > 0 || typeof selector !== 'string') return null; + return selector; +} + +function readPlainTypeText(action: SessionAction | undefined): string | null { + if (action?.command !== 'type') return null; + if (action.flags && Object.keys(action.flags).length > 0) return null; + const [text, ...rest] = action.positionals ?? []; + if (rest.length > 0 || typeof text !== 'string') return null; + return text; } function parseYamlDocuments(script: string): unknown[] { diff --git a/src/compat/maestro/run-script.ts b/src/compat/maestro/run-script.ts index 9e036568e..49f9f242e 100644 --- a/src/compat/maestro/run-script.ts +++ b/src/compat/maestro/run-script.ts @@ -1,9 +1,12 @@ import fs from 'node:fs'; import path from 'node:path'; import vm from 'node:vm'; +import type { SessionAction } from '../../daemon/types.ts'; import { AppError } from '../../utils/errors.ts'; import { runCmdSync } from '../../utils/exec.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import { + action, assertOnlyKeys, isPlainRecord, readEnvMap, @@ -23,6 +26,10 @@ type HttpResponse = { const HTTP_REQUEST_SCRIPT = ` const fs = require('node:fs'); const input = JSON.parse(fs.readFileSync(0, 'utf8')); +if (typeof fetch !== 'function') { + console.error('global fetch is required for Maestro runScript http helpers'); + process.exit(1); +} fetch(input.url, { method: input.method, headers: input.headers, @@ -39,19 +46,30 @@ fetch(input.url, { }); `; -export function executeRunScript(value: unknown, context: MaestroParseContext): void { +export function convertRunScript(value: unknown, context: MaestroParseContext): SessionAction { const scriptConfig = readRunScriptConfig(value, context); const scriptPath = resolveRunScriptPath(scriptConfig.file, context); + return action(MAESTRO_RUNTIME_COMMAND.runScript, [scriptPath], { + ...(Object.keys(scriptConfig.env).length > 0 + ? { maestro: { runScriptEnv: scriptConfig.env } } + : {}), + }); +} + +export function executeRunScriptFile(params: { + scriptPath: string; + env: Record; +}): Record { + const { scriptPath, env } = params; const script = fs.readFileSync(scriptPath, 'utf8'); const output: Record = {}; - const scriptEnv = { - ...context.env, - ...scriptConfig.env, - ...context.envOverrides, - }; try { - vm.runInNewContext(script, buildScriptGlobals(scriptEnv, output), { + // Compatibility note: node:vm is not a security sandbox. Maestro runScript + // files are trusted flow-local setup code; the timeout only bounds + // synchronous script execution. Async http.post work is bounded separately + // by the child process timeout in runHttpRequestSync. + vm.runInNewContext(script, buildScriptGlobals(env, output), { filename: scriptPath, timeout: RUN_SCRIPT_TIMEOUT_MS, }); @@ -64,9 +82,13 @@ export function executeRunScript(value: unknown, context: MaestroParseContext): ); } - for (const [key, rawValue] of Object.entries(output)) { - context.env[`output.${key}`] = stringifyOutputValue(rawValue); - } + validateOutputKeys(output, scriptPath); + return Object.fromEntries( + Object.entries(output).map(([key, rawValue]) => [ + `output.${key}`, + stringifyOutputValue(rawValue), + ]), + ); } function readRunScriptConfig( @@ -119,6 +141,8 @@ function runHttpRequestSync( url: string, options?: { headers?: Record; body?: string }, ): HttpResponse { + // Keep http.post synchronous from the flow author's point of view while the + // network request remains timeout-bounded independently from node:vm. const result = runCmdSync(process.execPath, ['-e', HTTP_REQUEST_SCRIPT], { stdin: JSON.stringify({ method, @@ -154,6 +178,16 @@ function runHttpRequestSync( } } +function validateOutputKeys(output: Record, scriptPath: string): void { + for (const key of Object.keys(output)) { + if (!key.includes('.')) continue; + throw new AppError('INVALID_ARGS', `Maestro runScript output key cannot contain ".": ${key}`, { + scriptPath, + key, + }); + } +} + function stringifyOutputValue(value: unknown): string { if (typeof value === 'string') return value; if (typeof value === 'number' || typeof value === 'boolean') return String(value); diff --git a/src/compat/maestro/runtime-commands.ts b/src/compat/maestro/runtime-commands.ts index 02b816a4c..8e354f362 100644 --- a/src/compat/maestro/runtime-commands.ts +++ b/src/compat/maestro/runtime-commands.ts @@ -1,7 +1,11 @@ export const MAESTRO_RUNTIME_COMMAND = { - pressEnter: '__maestroPressEnter', runFlowWhen: '__maestroRunFlowWhen', + runScript: '__maestroRunScript', + assertNotVisible: '__maestroAssertNotVisible', + pressEnter: '__maestroPressEnter', + waitForAnimationToEnd: '__maestroWaitForAnimationToEnd', scrollUntilVisible: '__maestroScrollUntilVisible', + swipeOn: '__maestroSwipeOn', tapOn: '__maestroTapOn', tapPointPercent: '__maestroTapPointPercent', } as const; diff --git a/src/core/__tests__/dispatch-keyboard.test.ts b/src/core/__tests__/dispatch-keyboard.test.ts new file mode 100644 index 000000000..8cd0a8c2e --- /dev/null +++ b/src/core/__tests__/dispatch-keyboard.test.ts @@ -0,0 +1,48 @@ +import { beforeEach, test, vi } from 'vitest'; +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; + +vi.mock('../../platforms/ios/runner-client.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, runIosRunnerCommand: vi.fn() }; +}); + +import { dispatchCommand } from '../dispatch.ts'; +import { runIosRunnerCommand } from '../../platforms/ios/runner-client.ts'; +import { ANDROID_EMULATOR, IOS_DEVICE } from '../../__tests__/test-utils/device-fixtures.ts'; +import { withMockedAdb } from '../../__tests__/test-utils/mocked-binaries.ts'; + +const mockRunIosRunnerCommand = vi.mocked(runIosRunnerCommand); + +beforeEach(() => { + vi.resetAllMocks(); + mockRunIosRunnerCommand.mockResolvedValue({ + message: 'keyboardReturn', + wasVisible: true, + visible: false, + }); +}); + +test('dispatch keyboard enter sends Android ENTER keyevent', async () => { + await withMockedAdb('agent-device-dispatch-keyboard-enter-', async (argsLogPath) => { + const result = await dispatchCommand(ANDROID_EMULATOR, 'keyboard', ['enter']); + + assert.equal(result?.action, 'enter'); + const logged = await fs.readFile(argsLogPath, 'utf8'); + assert.match(logged, /shell\ninput\nkeyevent\nENTER/); + }); +}); + +test('dispatch keyboard enter sends native iOS keyboard return command', async () => { + const result = await dispatchCommand(IOS_DEVICE, 'keyboard', ['return'], undefined, { + appBundleId: 'com.example.app', + }); + + assert.equal(result?.action, 'enter'); + assert.equal(result?.wasVisible, true); + assert.equal(mockRunIosRunnerCommand.mock.calls.length, 1); + assert.deepEqual(mockRunIosRunnerCommand.mock.calls[0]?.[1], { + command: 'keyboardReturn', + appBundleId: 'com.example.app', + }); +}); diff --git a/src/core/__tests__/dispatch-open.test.ts b/src/core/__tests__/dispatch-open.test.ts index 807122348..1cf399a73 100644 --- a/src/core/__tests__/dispatch-open.test.ts +++ b/src/core/__tests__/dispatch-open.test.ts @@ -1,8 +1,30 @@ -import { test } from 'vitest'; +import { beforeEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import { dispatchCommand } from '../dispatch.ts'; import { AppError } from '../../utils/errors.ts'; import type { DeviceInfo } from '../../utils/device.ts'; +import { clearIosSimulatorAppState, openIosApp } from '../../platforms/ios/apps.ts'; +import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; + +vi.mock('../../platforms/ios/apps.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + clearIosSimulatorAppState: vi.fn(async () => ({ + bundleId: 'com.example.app', + containerPath: '/tmp/com.example.app', + })), + openIosApp: vi.fn(async () => {}), + }; +}); + +const mockClearIosSimulatorAppState = vi.mocked(clearIosSimulatorAppState); +const mockOpenIosApp = vi.mocked(openIosApp); + +beforeEach(() => { + mockClearIosSimulatorAppState.mockClear(); + mockOpenIosApp.mockClear(); +}); test('dispatch open rejects URL as first argument when second URL is provided', async () => { const device: DeviceInfo = { @@ -46,3 +68,24 @@ test('dispatch open rejects Android launch arguments instead of dropping them', }, ); }); + +test('dispatch open clears Maestro iOS simulator state and launches once', async () => { + const result = await dispatchCommand(IOS_SIMULATOR, 'open', ['com.example.app'], undefined, { + clearAppState: true, + launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true'], + }); + + assert.equal(result?.app, 'com.example.app'); + assert.equal(mockClearIosSimulatorAppState.mock.calls.length, 1); + assert.deepEqual(mockClearIosSimulatorAppState.mock.calls[0]?.slice(0, 2), [ + IOS_SIMULATOR, + 'com.example.app', + ]); + assert.equal(mockOpenIosApp.mock.calls.length, 1); + assert.equal(mockOpenIosApp.mock.calls[0]?.[0], IOS_SIMULATOR); + assert.equal(mockOpenIosApp.mock.calls[0]?.[1], 'com.example.app'); + assert.deepEqual(mockOpenIosApp.mock.calls[0]?.[2]?.launchArgs, [ + '-EXDevMenuIsOnboardingFinished', + 'true', + ]); +}); diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index b96573010..8ff7bfeb7 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -96,7 +96,7 @@ const COMMAND_CAPABILITY_MATRIX: Record = { device.kind === 'simulator', }, keyboard: { - // iOS only supports keyboard dismiss; status/get remains Android-only. + // iOS only supports keyboard dismiss/enter; status/get remains Android-only. apple: { simulator: true, device: true }, android: { emulator: true, device: true, unknown: true }, linux: LINUX_NONE, diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index 9bf959b8b..bb8952881 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -10,13 +10,18 @@ export type BatchStep = { runtime?: unknown; }; +export type MaestroRuntimeFlags = { + allowNonHittableSelectorTap?: boolean; + clearState?: boolean; + optional?: boolean; + runScriptEnv?: Record; +}; + export type CommandFlags = Omit & { batchSteps?: BatchStep[]; launchArgs?: string[]; + maestro?: MaestroRuntimeFlags; replayBackend?: string; - allowNonHittableSelectorTap?: boolean; - maestroOptional?: boolean; - maestroClearState?: boolean; }; export type DispatchContext = ScreenshotDispatchFlags & { @@ -25,7 +30,7 @@ export type DispatchContext = ScreenshotDispatchFlags & { activity?: string; launchConsole?: string; launchArgs?: string[]; - maestroClearState?: boolean; + clearAppState?: boolean; verbose?: boolean; logPath?: string; traceLogPath?: string; @@ -46,7 +51,6 @@ export type DispatchContext = ScreenshotDispatchFlags & { pauseMs?: number; pattern?: 'one-way' | 'ping-pong'; surface?: SessionSurface; - allowNonHittableSelectorTap?: boolean; directElementSelector?: { key: 'id' | 'label' | 'text' | 'value'; value: string; diff --git a/src/core/dispatch-interactions.ts b/src/core/dispatch-interactions.ts index 530862e7f..76b78fa64 100644 --- a/src/core/dispatch-interactions.ts +++ b/src/core/dispatch-interactions.ts @@ -86,6 +86,15 @@ export async function handleFillCommand( positionals: string[], context: DispatchContext | undefined, ): Promise> { + if (context?.directElementSelector) { + return await handleDirectElementSelectorFill( + interactor, + context.directElementSelector, + positionals, + context, + ); + } + const x = Number(positionals[0]); const y = Number(positionals[1]); const text = positionals.slice(2).join(' '); @@ -97,6 +106,28 @@ export async function handleFillCommand( return { x, y, text, delayMs, ...successText(formatTextLengthMessage('Filled', text)) }; } +async function handleDirectElementSelectorFill( + interactor: Interactor, + selector: NonNullable, + positionals: string[], + context: DispatchContext, +): Promise> { + if (!interactor.fillElementSelector) { + throw new AppError('UNSUPPORTED_OPERATION', 'direct element selector fill is not supported'); + } + const text = positionals.join(' '); + if (!text) throw new AppError('INVALID_ARGS', 'fill requires text'); + const delayMs = requireIntInRange(context.delayMs ?? 0, 'delay-ms', 0, 10_000); + const result = await interactor.fillElementSelector(selector, text, delayMs); + return { + selector: selector.raw, + text, + delayMs, + ...(result ?? {}), + ...successText(formatTextLengthMessage('Filled', text)), + }; +} + export async function handlePressCommand( device: DeviceInfo, interactor: Interactor, diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 1e5b12397..61a57b554 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -6,6 +6,7 @@ import { dismissAndroidKeyboard, getAndroidKeyboardState, } from '../platforms/android/device-input-state.ts'; +import { pressAndroidEnter } from '../platforms/android/input-actions.ts'; import { pushAndroidNotification } from '../platforms/android/notifications.ts'; import { getInteractor } from './interactors.ts'; import type { Interactor, RunnerContext } from './interactor-types.ts'; @@ -231,7 +232,7 @@ async function handleOpenCommand( 'Launch arguments are currently supported only on Apple platforms.', ); } - if (context?.maestroClearState) { + if (context?.clearAppState) { if (isDeepLinkTarget(app)) { throw new AppError( 'INVALID_ARGS', @@ -289,13 +290,30 @@ async function handleKeyboardCommand( runnerCtx: RunnerContext, ): Promise> { const action = (positionals[0] ?? 'status').toLowerCase(); - if (action !== 'status' && action !== 'get' && action !== 'dismiss') { - throw new AppError('INVALID_ARGS', 'keyboard requires a subcommand: status, get, or dismiss'); + if ( + action !== 'status' && + action !== 'get' && + action !== 'dismiss' && + action !== 'enter' && + action !== 'return' + ) { + throw new AppError( + 'INVALID_ARGS', + 'keyboard requires a subcommand: status, get, dismiss, enter, or return', + ); } if (positionals.length > 1) { throw new AppError('INVALID_ARGS', 'keyboard accepts at most one subcommand argument'); } if (device.platform === 'android') { + if (action === 'enter' || action === 'return') { + await pressAndroidEnter(device); + return { + platform: 'android', + action: 'enter', + ...successText('Keyboard enter pressed'), + }; + } if (action === 'dismiss') { const result = await dismissAndroidKeyboard(device); return { @@ -327,12 +345,26 @@ async function handleKeyboardCommand( }; } if (device.platform === 'ios') { - if (action !== 'dismiss') { + if (action !== 'dismiss' && action !== 'enter' && action !== 'return') { throw new AppError( 'UNSUPPORTED_OPERATION', - 'keyboard status/get is currently supported only on Android; use keyboard dismiss on iOS', + 'keyboard status/get is currently supported only on Android; use keyboard dismiss or enter on iOS', ); } + if (action === 'enter' || action === 'return') { + const result = await runIosRunnerCommand( + device, + { command: 'keyboardReturn', appBundleId: context?.appBundleId }, + runnerCtx, + ); + return { + platform: 'ios', + action: 'enter', + visible: result.visible, + wasVisible: result.wasVisible, + ...successText('Keyboard enter pressed'), + }; + } const result = await runIosRunnerCommand( device, { command: 'keyboardDismiss', appBundleId: context?.appBundleId }, diff --git a/src/core/interactor-types.ts b/src/core/interactor-types.ts index c87d19ab8..6b5abfa36 100644 --- a/src/core/interactor-types.ts +++ b/src/core/interactor-types.ts @@ -82,6 +82,11 @@ export type Interactor = { longPress(x: number, y: number, durationMs?: number): Promise | void>; focus(x: number, y: number): Promise | void>; type(text: string, delayMs?: number): Promise; + fillElementSelector?( + selector: ElementSelectorTapOptions, + text: string, + delayMs?: number, + ): Promise | void>; fill( x: number, y: number, diff --git a/src/daemon-client.ts b/src/daemon-client.ts index ba17f166e..2faf868c8 100644 --- a/src/daemon-client.ts +++ b/src/daemon-client.ts @@ -117,7 +117,7 @@ export async function sendToDaemon(req: Omit): Promise await ensureDaemon(settings), @@ -168,6 +168,14 @@ export async function sendToDaemon(req: Omit): Promise): number | undefined { + if (req.command === 'test') return undefined; + if (req.command === 'replay' && typeof req.flags?.timeoutMs === 'number') { + return req.flags.timeoutMs; + } + return REQUEST_TIMEOUT_MS; +} + export async function openApp(options: OpenAppOptions = {}): Promise { const { session = 'default', diff --git a/src/daemon/__tests__/context.test.ts b/src/daemon/__tests__/context.test.ts index bf117b083..35559226d 100644 --- a/src/daemon/__tests__/context.test.ts +++ b/src/daemon/__tests__/context.test.ts @@ -14,16 +14,10 @@ test('contextFromFlags forwards scroll pixels from CLI flags', () => { assert.equal(context.pixels, 240); }); -test('contextFromFlags forwards internal non-hittable selector tap flag', () => { - const flags: CommandFlags = { allowNonHittableSelectorTap: true }; +test('contextFromFlags maps Maestro clearState to generic app-state clearing', () => { + const flags: CommandFlags = { maestro: { clearState: true } }; const context = contextFromFlags('/tmp/agent-device.log', flags); - assert.equal(context.allowNonHittableSelectorTap, true); -}); - -test('contextFromFlags forwards Maestro clearState launch compatibility flag', () => { - const flags: CommandFlags = { maestroClearState: true }; - const context = contextFromFlags('/tmp/agent-device.log', flags); - assert.equal(context.maestroClearState, true); + assert.equal(context.clearAppState, true); }); test('contextFromFlags forwards screenshot flags from CLI flags', () => { diff --git a/src/daemon/context.ts b/src/daemon/context.ts index a3fd128c7..8234a521b 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -25,7 +25,7 @@ export function contextFromFlags( activity: flags?.activity, launchConsole: flags?.launchConsole, launchArgs: flags?.launchArgs, - maestroClearState: flags?.maestroClearState, + clearAppState: flags?.maestro?.clearState, verbose: flags?.verbose, logPath, traceLogPath, @@ -46,6 +46,5 @@ export function contextFromFlags( backMode: flags?.backMode, pauseMs: flags?.pauseMs, pattern: flags?.pattern, - allowNonHittableSelectorTap: flags?.allowNonHittableSelectorTap, }; } diff --git a/src/daemon/handlers/__tests__/find.test.ts b/src/daemon/handlers/__tests__/find.test.ts index 9b7d165ba..8c9b92c0d 100644 --- a/src/daemon/handlers/__tests__/find.test.ts +++ b/src/daemon/handlers/__tests__/find.test.ts @@ -95,6 +95,7 @@ test('handleFindCommands click returns deterministic metadata across locator var expectedLocator: 'any', expectedQuery: 'Increment', expectedCoordinates: { x: 100, y: 50 }, + expectedRef: '@e2', }, ]; @@ -104,7 +105,7 @@ test('handleFindCommands click returns deterministic metadata across locator var if (!response.ok) return; const data = response.data as Record; expect(Object.keys(data).sort()).toEqual(scenario.expectedKeys); - expect(data.ref).toBe('@e1'); + expect(data.ref).toBe(scenario.expectedRef); expect(data.locator).toBe(scenario.expectedLocator); expect(data.query).toBe(scenario.expectedQuery); @@ -117,7 +118,7 @@ test('handleFindCommands click returns deterministic metadata across locator var } expect(invokeCalls.length).toBe(1); - expect(invokeCalls[0].positionals?.[0]).toBe('@e1'); + expect(invokeCalls[0].positionals?.[0]).toBe(scenario.expectedRef); } }); @@ -157,6 +158,61 @@ test('handleFindCommands click prefers on-screen duplicate text matches', async expect(invokeCalls[0].positionals?.[0]).toBe('@e3'); }); +test('handleFindCommands click prefers semantic controls over matching containers', async () => { + const { response, invokeCalls } = await runFindClickScenario({ + positionals: ['Later', 'click'], + flags: { findFirst: true }, + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Application', + hittable: true, + rect: { x: 0, y: 0, width: 440, height: 956 }, + }, + { + index: 1, + ref: 'e2', + type: 'Element(5)', + label: 'Dialog', + hittable: true, + rect: { x: 60, y: 356, width: 320, height: 272 }, + parentIndex: 0, + }, + { + index: 2, + ref: 'e3', + type: 'ScrollView', + label: 'Later', + hittable: false, + rect: { x: 60, y: 548, width: 320, height: 80 }, + parentIndex: 1, + }, + { + index: 3, + ref: 'e4', + type: 'Other', + label: 'Later', + hittable: false, + rect: { x: 76, y: 564, width: 288, height: 48 }, + parentIndex: 2, + }, + { + index: 4, + ref: 'e5', + type: 'Button', + label: 'Later', + hittable: false, + rect: { x: 76, y: 564, width: 140, height: 48 }, + parentIndex: 3, + }, + ], + }); + + expect(response.ok).toBe(true); + expect(invokeCalls[0].positionals?.[0]).toBe('@e5'); +}); + test('handleFindCommands wait bypasses snapshot cache while Android freshness recovery is active', async () => { const sessionName = 'android-find-wait'; const session: SessionState = { diff --git a/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts b/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts index eed6d7af5..fc29098f4 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts +++ b/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts @@ -52,6 +52,19 @@ test('parseFillTarget reads selector text through shared fill codec', () => { }); }); +test('parseFillTarget preserves selector text whitespace', () => { + const parsed = parseFillTarget(['label="Command"', 'submit\n']); + + expect(parsed).toEqual({ + ok: true, + target: { + kind: 'selector', + selector: 'label="Command"', + }, + text: 'submit\n', + }); +}); + test('parseFillTarget rejects invalid coordinates instead of treating them as a point', () => { const parsed = parseFillTarget(['10', 'not-y', 'text']); diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 7a4531eb0..dd9147f01 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -417,6 +417,49 @@ test('click simple iOS id selector uses direct runner selector tap without snaps } }); +test('fill simple iOS id selector uses direct runner selector fill without snapshot coordinates', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-direct-selector-fill'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + + mockDispatch.mockResolvedValue({ + message: 'filled', + x: 439.5, + y: 100.5, + referenceWidth: 440, + referenceHeight: 956, + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['id="email"', 'ada@example.com'], + flags: { delayMs: 25 }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + expect(mockDispatch).toHaveBeenCalledTimes(1); + expect(mockDispatch.mock.calls[0]?.[1]).toBe('fill'); + expect(mockDispatch.mock.calls[0]?.[2]).toEqual(['ada@example.com']); + const context = mockDispatch.mock.calls[0]?.[4] as Record; + expect(context.directElementSelector).toEqual({ + key: 'id', + value: 'email', + raw: 'id="email"', + }); + expect(context.delayMs).toBe(25); + if (response?.ok) { + expect(response.data?.selector).toBe('id="email"'); + expect(response.data?.text).toBe('ada@example.com'); + } +}); + test('click simple iOS selector forwards Maestro non-hittable tap backdoor', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-maestro-selector-fallback'; @@ -436,7 +479,7 @@ test('click simple iOS selector forwards Maestro non-hittable tap backdoor', asy session: sessionName, command: 'click', positionals: ['id="e2eSignInAlice"'], - flags: { allowNonHittableSelectorTap: true }, + flags: { maestro: { allowNonHittableSelectorTap: true } }, }, sessionName, sessionStore, diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index 6d9dc7453..66f01ad0d 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -32,6 +32,7 @@ type CapturedInvocation = { async function runReplayFixture(params: { label: string; script: string; + files?: Record; flags?: CommandFlags; invoke?: (req: DaemonRequest) => Promise; }): Promise<{ @@ -41,11 +42,15 @@ async function runReplayFixture(params: { scriptPath: string; }> { const root = fs.mkdtempSync(path.join(os.tmpdir(), `agent-device-replay-${params.label}-`)); + for (const [name, contents] of Object.entries(params.files ?? {})) { + fs.writeFileSync(path.join(root, name), contents); + } const scriptPath = path.join(root, 'flow.ad'); fs.writeFileSync(scriptPath, params.script); const calls: CapturedInvocation[] = []; - const defaultInvoke = async (req: DaemonRequest): Promise => { + const invoke = async (req: DaemonRequest): Promise => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (params.invoke) return await params.invoke(req); return { ok: true, data: {} }; }; const response = await runReplayScriptFile({ @@ -60,7 +65,7 @@ async function runReplayFixture(params: { sessionName: 's', logPath: path.join(root, 'log'), sessionStore: new SessionStore(path.join(root, 'state')), - invoke: params.invoke ?? defaultInvoke, + invoke, }); return { response, calls, root, scriptPath }; } @@ -401,11 +406,105 @@ test('runReplayScriptFile applies CLI env overrides before Maestro compat mappin replayShellEnv: { AD_VAR_BUTTON_ID: 'shell-button' }, replayEnv: ['APP_ID=cli-app'], }, + invoke: async (req) => { + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'shell-button', + rect: { x: 20, y: 40, width: 120, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, }); assert.equal(response.ok, true); assert.deepEqual(calls[0]?.positionals, ['cli-app']); - assert.deepEqual(calls[1]?.positionals, ['id="shell-button"']); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['open', ['cli-app']], + ['snapshot', []], + ['click', ['80', '62']], + ], + ); +}); + +test('runReplayScriptFile runs Maestro runScript in replay order and exposes output variables', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-runtime', + files: { + 'setup.js': ` +var res = {body: '{"appviewDid":"did:plc:test"}'} +output.result = SERVER_PATH + ':' + json(res.body).appviewDid +`, + }, + script: [ + 'appId: demo.app', + '---', + '- runScript:', + ' file: ./setup.js', + ' env:', + ' SERVER_PATH: local', + '- inputText: ${output.result}', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['type', ['local:did:plc:test']]], + ); +}); + +test('runReplayScriptFile reports Maestro runScript failures at the runScript step', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-fail', + files: { + 'setup.js': `output.result = http.post('http://127.0.0.1:1').body`, + }, + script: ['appId: demo.app', '---', '- runScript: ./setup.js', '- inputText: never', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /runScript failed/); + assert.match(response.error.message, /http\.post failed/); + } + assert.equal(calls.length, 0); +}); + +test('runReplayScriptFile rejects Maestro runScript output keys containing dots', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-dotted-output', + files: { + 'setup.js': `output['nested.value'] = 'ambiguous'`, + }, + script: ['appId: demo.app', '---', '- runScript: ./setup.js', '- inputText: never', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /output key cannot contain/); + } + assert.equal(calls.length, 0); }); test('runReplayScriptFile retries Maestro scrollUntilVisible with scroll probes', async () => { @@ -503,8 +602,208 @@ test('runReplayScriptFile lets Maestro tapOn use fuzzy visible text matching', a assert.equal(response.ok, true); assert.deepEqual( calls.map((call) => [call.command, call.positionals]), - [['find', ['Discover', 'click']]], + [ + ['snapshot', []], + ['find', ['Discover', 'click']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); + assert.equal(calls[1]?.flags?.findFirst, true); +}); + +test('runReplayScriptFile lets exact Maestro text tapOn win before fuzzy matching', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-exact-before-fuzzy', + script: ['appId: demo.app', '---', '- tapOn: Mute accounts', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Block accounts', + rect: { x: 10, y: 600, width: 240, height: 44 }, + }, + { + index: 2, + label: 'Mute accounts', + rect: { x: 10, y: 540, width: 240, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['130', '562']], + ], + ); +}); + +test('runReplayScriptFile prefers on-screen Maestro text tapOn matches', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-onscreen', + script: ['appId: demo.app', '---', '- tapOn: Sign in', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'Button', + label: 'Sign in', + rect: { x: -328, y: 182, width: 328, height: 42 }, + }, + { + index: 2, + type: 'Button', + label: 'Sign in', + rect: { x: 56, y: 842, width: 328, height: 56 }, + }, + ], + metadata: { referenceWidth: 440, referenceHeight: 956 }, + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['220', '870']], + ], + ); +}); + +test('runReplayScriptFile taps Maestro text near the label in large action containers', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-action-container', + script: ['appId: demo.app', '---', '- tapOn: Mute accounts', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'ScrollView', + label: 'Mute accounts', + rect: { x: 8, y: 805, width: 424, height: 93 }, + }, + { + index: 2, + parentIndex: 1, + type: 'Other', + label: 'Block accounts', + rect: { x: 31, y: 835, width: 377, height: 42 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['92', '829']], + ], + ); +}); + +test('runReplayScriptFile prefers actionable Maestro tapOn matches over broad ancestors', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-prefers-actionable-match', + script: ['appId: demo.app', '---', '- tapOn: New list', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'Other', + label: 'New list', + rect: { x: 0, y: 0, width: 440, height: 956 }, + }, + { + index: 2, + type: 'Button', + label: 'New list', + rect: { x: 349, y: 67, width: 75, height: 33 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['387', '84']], + ], + ); +}); + +test('runReplayScriptFile treats absent Maestro assertNotVisible targets as passing', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-assert-not-visible-absent', + script: ['appId: demo.app', '---', '- assertNotVisible: Feeds ✨', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'Selector did not match' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['is', ['visible', 'label="Feeds ✨" || text="Feeds ✨" || id="Feeds ✨"']]], ); + assert.equal(calls[0]?.flags?.noRecord, true); }); test('runReplayScriptFile retries Maestro fuzzy tapOn without raw selector fallback', async () => { @@ -531,33 +830,35 @@ test('runReplayScriptFile retries Maestro fuzzy tapOn without raw selector fallb assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ + ['snapshot', []], ['find', ['Discover', 'click']], + ['snapshot', []], ['find', ['Discover', 'click']], ], ); }); -test('runReplayScriptFile lets optional Maestro fuzzy tapOn hit native alert labels', async () => { +test('runReplayScriptFile lets optional Maestro fuzzy tapOn click first visible match', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ - label: 'maestro-tap-visible-text-optional-native-label', + label: 'maestro-tap-visible-text-optional-first-match', script: [ 'appId: demo.app', '---', '- tapOn:', - ' text: Not Now', + ' text: Later', ' optional: true', '', ].join('\n'), flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'click' && req.positionals?.[0] === 'label="Not Now"') { - return { ok: true, data: { dismissed: true } }; + if (req.command === 'find' && req.flags?.findFirst === true) { + return { ok: true, data: { ref: '@e4', x: 220, y: 720 } }; } return { ok: false, - error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + error: { code: 'AMBIGUOUS_MATCH', message: 'matched multiple elements' }, }; }, }); @@ -566,10 +867,11 @@ test('runReplayScriptFile lets optional Maestro fuzzy tapOn hit native alert lab assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ - ['find', ['Not Now', 'click']], - ['click', ['label="Not Now"']], + ['snapshot', []], + ['find', ['Later', 'click']], ], ); + assert.equal(calls[1]?.flags?.findFirst, true); }); test('runReplayScriptFile resolves Maestro percentage point taps from snapshot size', async () => { @@ -609,17 +911,37 @@ test('runReplayScriptFile resolves Maestro percentage point taps from snapshot s assert.equal(calls[0]?.flags?.noRecord, true); }); -test('runReplayScriptFile retries Maestro tapOn until the selector appears', async () => { +test('runReplayScriptFile retries Maestro id tapOn through snapshot coordinates', async () => { const calls: CapturedInvocation[] = []; - let clickAttempts = 0; + let snapshotAttempts = 0; const { response } = await runReplayFixture({ label: 'maestro-tap-on-retry', script: ['appId: demo.app', '---', '- tapOn:', ' id: delayedButton', ''].join('\n'), flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - clickAttempts += 1; - if (clickAttempts === 3) return { ok: true, data: {} }; + if (req.command === 'snapshot') { + snapshotAttempts += 1; + if (snapshotAttempts === 3) { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'delayedButton', + rect: { x: 20, y: 40, width: 120, height: 44 }, + }, + ], + }, + }; + } + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + } + if (req.command === 'click') return { ok: true, data: {} }; return { ok: false, error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, @@ -631,25 +953,249 @@ test('runReplayScriptFile retries Maestro tapOn until the selector appears', asy assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ - ['click', ['id="delayedButton"']], - ['click', ['id="delayedButton"']], - ['click', ['id="delayedButton"']], + ['snapshot', []], + ['snapshot', []], + ['snapshot', []], + ['click', ['80', '62']], + ], + ); +}); + +test('runReplayScriptFile resolves Maestro tapOn index and childOf from snapshots', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-index-childof', + script: [ + 'appId: demo.app', + '---', + '- tapOn:', + ' id: likeBtn', + ' childOf:', + ' id: postThreadItem-by-bob.test', + '- tapOn:', + ' id: postDropdownBtn', + ' index: 1', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { index: 1, identifier: 'postThreadItem-by-alice.test' }, + { + index: 2, + parentIndex: 1, + identifier: 'likeBtn', + rect: { x: 10, y: 10, width: 40, height: 20 }, + }, + { index: 10, identifier: 'postThreadItem-by-bob.test' }, + { + index: 11, + parentIndex: 10, + identifier: 'likeBtn', + rect: { x: 20, y: 120, width: 40, height: 20 }, + }, + { + index: 20, + identifier: 'postDropdownBtn', + rect: { x: 100, y: 200, width: 40, height: 20 }, + }, + { + index: 21, + identifier: 'postDropdownBtn', + rect: { x: 200, y: 300, width: 40, height: 20 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['40', '130']], + ['snapshot', []], + ['click', ['220', '310']], ], ); + assert.equal(calls[0]?.flags?.noRecord, true); }); -test('runReplayScriptFile recovers Maestro enter submit after iOS runner transport reset', async () => { +test('runReplayScriptFile lets snapshot id tap handle Maestro one-point edge controls', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ - label: 'maestro-press-enter-recover', + label: 'maestro-tap-edge-rect', + script: ['appId: demo.app', '---', '- tapOn:', ' id: e2eSignInAlice', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'e2eSignInAlice', + rect: { x: 0, y: 0, width: 1, height: 1 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['0', '0']], + ], + ); +}); + +test('runReplayScriptFile keeps Maestro text-entry tapOn on the snapshot path', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-input-text-snapshot', + script: [ + 'appId: demo.app', + '---', + '- tapOn:', + ' id: editListNameInput', + '- inputText: Muted Users', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'editListNameInput', + rect: { x: 20, y: 100, width: 200, height: 40 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['120', '120']], + ['type', ['Muted Users']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile resolves Maestro swipe.label from a labeled element rect', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-swipe-label', + script: [ + 'appId: demo.app', + '---', + '- swipe:', + ' label: Thread body', + ' direction: UP', + ' duration: 400', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Thread body', + rect: { x: 10, y: 100, width: 200, height: 300 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['swipe', ['110', '250', '110', '8', '400']], + ], + ); +}); + +test('runReplayScriptFile maps Maestro enter to keyboard enter', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-press-enter', script: ['appId: demo.app', '---', '- pressKey: Enter', ''].join('\n'), flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'snapshot') return { ok: true, data: {} }; + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['keyboard', ['enter']]], + ); +}); + +test('runReplayScriptFile waits for Maestro animation snapshots to stabilize', async () => { + const calls: CapturedInvocation[] = []; + let snapshots = 0; + const { response } = await runReplayFixture({ + label: 'maestro-wait-animation-stable', + script: ['appId: demo.app', '---', '- waitForAnimationToEnd', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + snapshots += 1; + const y = snapshots === 1 ? 100 : 120; return { - ok: false, - error: { code: 'UNKNOWN', message: 'fetch failed' }, + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Animating', + rect: { x: 10, y, width: 100, height: 40 }, + }, + ], + }, }; }, }); @@ -658,10 +1204,37 @@ test('runReplayScriptFile recovers Maestro enter submit after iOS runner transpo assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ - ['type', ['\n']], + ['snapshot', []], + ['snapshot', []], ['snapshot', []], ], ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile falls back to newline type when keyboard enter is unsupported', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-press-enter-fallback', + script: ['appId: demo.app', '---', '- pressKey: Enter', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'keyboard') { + return { ok: false, error: { code: 'UNSUPPORTED_OPERATION', message: 'unsupported' } }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['keyboard', ['enter']], + ['type', ['\n']], + ], + ); }); test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absent', async () => { @@ -683,7 +1256,11 @@ test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absen calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); return { ok: false, - error: { code: 'COMMAND_FAILED', message: 'not visible' }, + error: { + code: 'COMMAND_FAILED', + message: 'not visible', + details: { command: 'is', reason: 'selector_not_found' }, + }, }; }, }); @@ -750,6 +1327,33 @@ test('runReplayScriptFile propagates Maestro runFlow.when runtime errors', async } }); +test('runReplayScriptFile propagates Maestro runFlow.when COMMAND_FAILED errors without condition-miss details', async () => { + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-command-error', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async () => ({ + ok: false, + error: { code: 'COMMAND_FAILED', message: 'snapshot failed' }, + }), + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.equal(response.error.code, 'COMMAND_FAILED'); + assert.match(response.error.message, /snapshot failed/); + } +}); + test('runReplayScriptFile runs Maestro runFlow.when.visible commands when present', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ @@ -783,6 +1387,7 @@ test('runReplayScriptFile runs Maestro runFlow.when.visible commands when presen calls.map((call) => [call.command, call.positionals]), [ ['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']], + ['snapshot', []], ['find', ['Continue', 'click']], ], ); diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 66363d85b..7efc4988d 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -6,7 +6,11 @@ import type { DaemonRequest, DaemonResponse } from '../types.ts'; import { SessionStore } from '../session-store.ts'; import { contextFromFlags } from '../context.ts'; import { ensureDeviceReady } from '../device-ready.ts'; -import { extractNodeText, findNearestHittableAncestor } from '../snapshot-processing.ts'; +import { extractNodeText } from '../snapshot-processing.ts'; +import { + resolveActionableTouchNode, + resolveActionableTouchResolution, +} from '../../commands/interaction-targeting.ts'; import { readTextForNode } from './interaction-read.ts'; import { captureSnapshot } from './snapshot-capture.ts'; import { setSessionSnapshot } from '../session-snapshot.ts'; @@ -229,19 +233,50 @@ function preferOnscreenMatches( center.y <= viewport.y + viewport.height ); }); - return onscreen.length > 0 ? onscreen : matches; + return rankInteractiveMatches(onscreen.length > 0 ? onscreen : matches, nodes); +} + +function rankInteractiveMatches( + matches: SnapshotState['nodes'], + nodes: SnapshotState['nodes'], +): SnapshotState['nodes'] { + if (matches.length < 2) return matches; + return matches + .map((node, index) => ({ node, index, score: interactiveMatchScore(node, nodes) })) + .sort((left, right) => { + if (right.score !== left.score) return right.score - left.score; + return rectArea(left.node) - rectArea(right.node) || left.index - right.index; + }) + .map((entry) => entry.node); +} + +function interactiveMatchScore( + node: SnapshotState['nodes'][number], + nodes: SnapshotState['nodes'], +): number { + const resolution = resolveActionableTouchResolution(nodes, node); + if (resolution.reason === 'semantic-target' && resolution.node.rect) return 4; + if (resolution.reason === 'same-rect-descendant' && resolution.node.rect) return 4; + if ( + resolution.reason === 'hittable-ancestor' && + resolution.node.rect && + !isRootInteractionContainer(resolution.node, nodes[0]) + ) { + return 2; + } + if (node.hittable && node.rect && !isRootInteractionContainer(node, nodes[0])) return 3; + return node.rect ? 1 : 0; +} + +function rectArea(node: SnapshotState['nodes'][number]): number { + return node.rect ? node.rect.width * node.rect.height : Number.POSITIVE_INFINITY; } function resolveInteractiveMatchNode( nodes: SnapshotState['nodes'], node: SnapshotState['nodes'][number], ): SnapshotState['nodes'][number] { - const ancestor = findNearestHittableAncestor(nodes, node); - if (!ancestor) return node; - if (node.rect && isRootInteractionContainer(ancestor, nodes[0])) { - return node; - } - return ancestor; + return resolveActionableTouchNode(nodes, node); } function isRootInteractionContainer( diff --git a/src/daemon/handlers/interaction-touch-targets.ts b/src/daemon/handlers/interaction-touch-targets.ts index 264302ef4..d682531c0 100644 --- a/src/daemon/handlers/interaction-touch-targets.ts +++ b/src/daemon/handlers/interaction-touch-targets.ts @@ -110,8 +110,7 @@ export function parseFillTarget(positionals: string[]): ParsedFillTarget { ), }; } - const text = parsed.text.trim(); - if (!text) { + if (!parsed.text.trim()) { return { ok: false, response: errorResponse('INVALID_ARGS', 'fill requires text after selector'), @@ -120,7 +119,7 @@ export function parseFillTarget(positionals: string[]): ParsedFillTarget { return { ok: true, target: { kind: 'selector', selector: parsed.target.selector }, - text, + text: parsed.text, }; } diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 00ce38d36..ebd7ba9f3 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -262,7 +262,7 @@ function readDirectIosSelectorTapTarget(params: { if (!selector) return null; return { ...selector, - ...(flags?.allowNonHittableSelectorTap ? { allowNonHittableTap: true } : {}), + ...(flags?.maestro?.allowNonHittableSelectorTap ? { allowNonHittableTap: true } : {}), }; } @@ -372,6 +372,20 @@ async function dispatchFillViaRuntime( if (invalidRefFlagsResponse) return invalidRefFlagsResponse; await refreshAndroidRefSnapshotIfFreshnessActive(params, session); } + const directSelector = readDirectIosSelectorFillTarget({ + session, + target: parsedTarget.target, + flags: req.flags, + }); + if (directSelector) { + const directResponse = await dispatchDirectIosSelectorFill( + params, + session, + directSelector, + parsedTarget.text, + ); + if (directResponse) return directResponse; + } return await dispatchRuntimeInteraction(params, { run: async (runtime) => @@ -413,6 +427,74 @@ async function dispatchFillViaRuntime( }); } +function readDirectIosSelectorFillTarget(params: { + session: SessionState; + target: InteractionTarget; + flags: CommandFlags | undefined; +}): DirectIosSelectorTarget | null { + const { session, target, flags } = params; + if (target.kind !== 'selector') return null; + const selector = readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); + if (!selector) return null; + return { + ...selector, + ...(flags?.maestro?.allowNonHittableSelectorTap ? { allowNonHittableTap: true } : {}), + }; +} + +async function dispatchDirectIosSelectorFill( + params: InteractionHandlerParams, + session: SessionState, + selector: DirectIosSelectorTarget, + text: string, +): Promise { + const actionStartedAt = Date.now(); + try { + const data = + (await dispatchCommand(session.device, 'fill', [text], params.req.flags?.out, { + ...params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), + directElementSelector: selector, + surface: session.surface, + })) ?? {}; + const actionFinishedAt = Date.now(); + const point = readPointFromDirectSelectorTapResult(data); + const responseData = buildTouchVisualizationResult({ + data, + fallbackX: point.x, + fallbackY: point.y, + referenceFrame: readReferenceFrameFromDirectSelectorTapResult(data), + extra: { + selector: selector.raw, + text, + }, + }); + return finalizeTouchInteraction({ + session, + sessionStore: params.sessionStore, + command: params.req.command, + positionals: params.req.positionals ?? [], + flags: params.req.flags, + result: responseData, + responseData, + actionStartedAt, + actionFinishedAt, + }); + } catch (error) { + if (!isDirectIosSelectorFallbackError(error)) { + return { ok: false, error: normalizeError(error) }; + } + emitDiagnostic({ + level: 'debug', + phase: 'ios_direct_selector_fill_fallback', + data: { + selector: selector.raw, + error: error instanceof Error ? error.message : String(error), + }, + }); + return null; + } +} + async function dispatchRuntimeInteraction< TResult extends PressCommandResult | FillCommandResult | LongPressCommandResult, >( diff --git a/src/daemon/handlers/session-replay-action-runtime.ts b/src/daemon/handlers/session-replay-action-runtime.ts new file mode 100644 index 000000000..6ad7e658e --- /dev/null +++ b/src/daemon/handlers/session-replay-action-runtime.ts @@ -0,0 +1,148 @@ +import fs from 'node:fs'; +import type { CommandFlags } from '../../core/dispatch.ts'; +import { + mergeReplayVarScopeValues, + resolveReplayAction, + type ReplayVarScope, +} from '../../replay/vars.ts'; +import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; +import { mergeParentFlags } from './handler-utils.ts'; +import { invokeMaestroRuntimeCommand } from './session-replay-maestro-runtime.ts'; + +type ReplayBaseRequest = Omit; + +type ReplayActionInvoker = (params: { + action: SessionAction; + line: number; + step: number; +}) => Promise; + +export async function invokeReplayAction(params: { + req: DaemonRequest; + sessionName: string; + action: SessionAction; + scope: ReplayVarScope; + filePath: string; + line: number; + step: number; + tracePath?: string; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const { req, sessionName, action, scope, filePath, line, step, tracePath, invoke } = params; + const resolved = resolveReplayAction(action, scope, { file: filePath, line }); + const invokeNestedReplayAction: ReplayActionInvoker = (nested) => + invokeReplayAction({ + req, + sessionName, + action: nested.action, + scope, + filePath, + line: nested.line, + step: nested.step, + tracePath, + invoke, + }); + const startedAt = Date.now(); + appendReplayTraceEvent(tracePath, { + type: 'replay_action_start', + ts: new Date(startedAt).toISOString(), + replayPath: filePath, + line, + step, + command: resolved.command, + positionals: resolved.positionals ?? [], + }); + + const response = await invokeResolvedReplayAction({ + req, + sessionName, + resolved, + scope, + line, + step, + invoke, + invokeReplayAction: invokeNestedReplayAction, + }); + + const finishedAt = Date.now(); + appendReplayTraceEvent(tracePath, { + type: 'replay_action_stop', + ts: new Date(finishedAt).toISOString(), + replayPath: filePath, + line, + step, + command: resolved.command, + ok: response.ok, + durationMs: finishedAt - startedAt, + errorCode: response.ok ? undefined : response.error.code, + }); + return response; +} + +async function invokeResolvedReplayAction(params: { + req: DaemonRequest; + sessionName: string; + resolved: SessionAction; + scope: ReplayVarScope; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: ReplayActionInvoker; +}): Promise { + const { req, sessionName, resolved, scope, line, step, invoke, invokeReplayAction } = params; + const flags = buildReplayActionFlags(req.flags, resolved.flags); + const baseReq: ReplayBaseRequest = { + token: req.token, + session: sessionName, + flags, + runtime: resolved.runtime, + meta: req.meta, + }; + const response = + (await invokeMaestroRuntimeCommand({ + command: resolved.command, + baseReq, + positionals: resolved.positionals ?? [], + batchSteps: resolved.flags?.batchSteps, + scope, + line, + step, + invoke, + invokeReplayAction, + })) ?? + (await invoke({ + ...baseReq, + command: resolved.command, + positionals: resolved.positionals ?? [], + })); + if (response.ok) { + const outputEnv = readReplayOutputEnv(response.data); + if (outputEnv) mergeReplayVarScopeValues(scope, outputEnv); + } + return response; +} + +function readReplayOutputEnv(data: unknown): Record | null { + if (!data || typeof data !== 'object') return null; + const raw = (data as { outputEnv?: unknown }).outputEnv; + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null; + const entries = Object.entries(raw).filter( + (entry): entry is [string, string] => typeof entry[1] === 'string', + ); + return entries.length > 0 ? Object.fromEntries(entries) : null; +} + +function appendReplayTraceEvent( + tracePath: string | undefined, + event: Record, +): void { + if (!tracePath) return; + fs.appendFileSync(tracePath, `${JSON.stringify(event)}\n`); +} + +export function buildReplayActionFlags( + parentFlags: CommandFlags | undefined, + actionFlags: SessionAction['flags'] | undefined, +): CommandFlags { + return mergeParentFlags(parentFlags, { ...(actionFlags ?? {}) }); +} diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts index 56cac5e6a..00923591d 100644 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -1,15 +1,22 @@ import { type CommandFlags } from '../../core/dispatch.ts'; import { MAESTRO_RUNTIME_COMMAND } from '../../compat/maestro/runtime-commands.ts'; -import type { SnapshotState } from '../../utils/snapshot.ts'; +import { executeRunScriptFile } from '../../compat/maestro/run-script.ts'; +import type { Platform } from '../../utils/device.ts'; +import { type Rect, type SnapshotNode, type SnapshotState } from '../../utils/snapshot.ts'; import { sleep } from '../../utils/timeouts.ts'; +import { asAppError } from '../../utils/errors.ts'; +import type { ReplayVarScope } from '../../replay/vars.ts'; import { parseSelectorChain } from '../selectors.ts'; -import { getSnapshotReferenceFrame } from '../touch-reference-frame.ts'; +import { matchesSelector } from '../selectors-match.ts'; +import { getSnapshotReferenceFrame, type TouchReferenceFrame } from '../touch-reference-frame.ts'; import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; import { errorResponse } from './response.ts'; const MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS = 500; const MAESTRO_TAP_ON_TIMEOUT_MS = 30000; +const MAESTRO_OPTIONAL_TAP_ON_TIMEOUT_MS = 3000; const MAESTRO_TAP_ON_RETRY_MS = 250; +const MAESTRO_ANIMATION_POLL_MS = 250; type ReplayBaseRequest = Omit; @@ -20,6 +27,7 @@ type MaestroReplayInvoker = (params: { }) => Promise; type MaestroRuntimeInvoke = (req: DaemonRequest) => Promise; +type FailedDaemonResponse = Extract; type MaestroScrollUntilVisibleParams = { baseReq: ReplayBaseRequest; @@ -33,36 +41,144 @@ type MaestroTapOnParams = { invoke: MaestroRuntimeInvoke; }; +type MaestroTapOnOptions = { + childOf?: string; + index?: number; +}; + type MaestroRunFlowWhenCondition = | { ok: true; mode: string; predicate: string; selector: string } | { ok: false; response: DaemonResponse }; +type MaestroSnapshotTarget = { + node: SnapshotNode; + rect: Rect; + frame?: TouchReferenceFrame; +}; + export async function invokeMaestroRuntimeCommand(params: { command: string; baseReq: ReplayBaseRequest; positionals: string[]; batchSteps: CommandFlags['batchSteps'] | undefined; + scope: ReplayVarScope; line: number; step: number; invoke: (req: DaemonRequest) => Promise; invokeReplayAction: MaestroReplayInvoker; }): Promise { switch (params.command) { + case MAESTRO_RUNTIME_COMMAND.assertNotVisible: + return await invokeMaestroAssertNotVisible(params); + case MAESTRO_RUNTIME_COMMAND.pressEnter: + return await invokeMaestroPressEnter(params); + case MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd: + return await invokeMaestroWaitForAnimationToEnd(params); case MAESTRO_RUNTIME_COMMAND.scrollUntilVisible: return await invokeMaestroScrollUntilVisible(params); + case MAESTRO_RUNTIME_COMMAND.swipeOn: + return await invokeMaestroSwipeOn(params); case MAESTRO_RUNTIME_COMMAND.tapOn: return await invokeMaestroTapOn(params); case MAESTRO_RUNTIME_COMMAND.tapPointPercent: return await invokeMaestroTapPointPercent(params); case MAESTRO_RUNTIME_COMMAND.runFlowWhen: return await invokeMaestroRunFlowWhen(params); - case MAESTRO_RUNTIME_COMMAND.pressEnter: - return await invokeMaestroPressEnter(params); + case MAESTRO_RUNTIME_COMMAND.runScript: + return invokeMaestroRunScript(params); default: return undefined; } } +async function invokeMaestroPressEnter(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; +}): Promise { + const keyboardResponse = await params.invoke({ + ...params.baseReq, + command: 'keyboard', + positionals: ['enter'], + }); + if (keyboardResponse.ok) return keyboardResponse; + + return await params.invoke({ + ...params.baseReq, + command: 'type', + positionals: ['\n'], + }); +} + +async function invokeMaestroAssertNotVisible(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const [selector] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'assertNotVisible requires a selector.'); + } + const response = await params.invoke({ + ...params.baseReq, + command: 'is', + positionals: ['visible', selector], + flags: { ...params.baseReq.flags, noRecord: true }, + }); + if (!response.ok) { + return { ok: true, data: { pass: true, selector, absent: true } }; + } + if (response.data?.pass === false) { + return { ok: true, data: { pass: true, selector } }; + } + return errorResponse('COMMAND_FAILED', `Expected not visible but matched: ${selector}`); +} + +async function invokeMaestroWaitForAnimationToEnd(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const timeoutMs = Number(params.positionals[0] ?? 15000); + if (!Number.isFinite(timeoutMs) || timeoutMs < 0) { + return errorResponse('INVALID_ARGS', 'waitForAnimationToEnd timeout must be a number.'); + } + const startedAt = Date.now(); + let previousSignature: string | undefined; + let lastResponse: DaemonResponse | undefined; + + while (Date.now() - startedAt < timeoutMs) { + const response = await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, + }); + if (!response.ok) { + lastResponse = response; + await sleep(MAESTRO_ANIMATION_POLL_MS); + continue; + } + const snapshot = readSnapshotState(response.data); + if (!snapshot) return response; + const signature = snapshotStabilitySignature(snapshot); + if (previousSignature === signature) { + return { ok: true, data: { stable: true, timeoutMs } }; + } + previousSignature = signature; + lastResponse = response; + await sleep(MAESTRO_ANIMATION_POLL_MS); + } + + return lastResponse?.ok === false + ? lastResponse + : { ok: true, data: { stable: false, timeoutMs } }; +} + async function invokeMaestroScrollUntilVisible( params: MaestroScrollUntilVisibleParams, ): Promise { @@ -76,17 +192,17 @@ async function invokeMaestroScrollUntilVisible( } const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); const attempts = Math.max(1, Math.ceil(timeoutMs / MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS)); - let lastWaitResponse: DaemonResponse | undefined; + let lastWaitResponse: FailedDaemonResponse | null = null; for (let index = 0; index < attempts; index += 1) { - const probe = await probeMaestroScrollVisibility( + const probeResponse = await probeMaestroScrollVisibility( params, selector, fuzzyTextQuery, scrollProbeMs(timeoutMs, index), ); - if (probe.visible) return probe.response; - lastWaitResponse = probe.response; + if (probeResponse.ok) return probeResponse; + lastWaitResponse = probeResponse; if (index === attempts - 1) break; @@ -106,21 +222,20 @@ async function probeMaestroScrollVisibility( selector: string, fuzzyTextQuery: string | null, probeMs: number, -): Promise<{ visible: boolean; response: DaemonResponse }> { +): Promise { const waitResponse = await params.invoke({ ...params.baseReq, command: 'wait', positionals: [selector, String(probeMs)], }); - if (waitResponse.ok) return { visible: true, response: waitResponse }; - if (!fuzzyTextQuery) return { visible: false, response: waitResponse }; + if (waitResponse.ok || !fuzzyTextQuery) return waitResponse; const fuzzyResponse = await params.invoke({ ...params.baseReq, command: 'find', positionals: [fuzzyTextQuery, 'wait', String(probeMs)], }); - return { visible: fuzzyResponse.ok, response: fuzzyResponse }; + return fuzzyResponse; } function scrollProbeMs(timeoutMs: number, index: number): number { @@ -192,34 +307,54 @@ function readSnapshotState(data: unknown): SnapshotState | undefined { return undefined; } +function snapshotStabilitySignature(snapshot: SnapshotState): string { + return JSON.stringify( + snapshot.nodes.map((node) => ({ + index: node.index, + parentIndex: node.parentIndex, + type: node.type, + identifier: node.identifier, + label: node.label, + value: node.value, + rect: node.rect + ? { + x: Math.round(node.rect.x), + y: Math.round(node.rect.y), + width: Math.round(node.rect.width), + height: Math.round(node.rect.height), + } + : undefined, + })), + ); +} + async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { - const [selector] = params.positionals; + const [selector, rawOptions] = params.positionals; if (!selector) { return errorResponse('INVALID_ARGS', 'tapOn requires a selector.'); } + const options = readMaestroTapOnOptions(rawOptions); + if (!options.ok) return options.response; const startedAt = Date.now(); const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + const timeoutMs = + params.baseReq.flags?.maestro?.optional === true + ? MAESTRO_OPTIONAL_TAP_ON_TIMEOUT_MS + : MAESTRO_TAP_ON_TIMEOUT_MS; let lastResponse: DaemonResponse | undefined; - while (Date.now() - startedAt < MAESTRO_TAP_ON_TIMEOUT_MS) { + while (Date.now() - startedAt < timeoutMs) { + const attempt = await invokeMaestroSnapshotTapOn(params, selector, options.value ?? {}); + if (attempt.ok) return attempt; + lastResponse = attempt; if (fuzzyTextQuery) { - const attempt = await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); - if (!attempt.retry) return attempt.response; - lastResponse = attempt.response; - await sleep(MAESTRO_TAP_ON_RETRY_MS); - continue; + const fuzzyAttempt = await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); + if (!fuzzyAttempt.retry) return fuzzyAttempt.response; + lastResponse = fuzzyAttempt.response; } - - const clickResponse = await params.invoke({ - ...params.baseReq, - command: 'click', - positionals: [selector], - }); - if (clickResponse.ok) return clickResponse; - lastResponse = clickResponse; await sleep(MAESTRO_TAP_ON_RETRY_MS); } - if (params.baseReq.flags?.maestroOptional === true) { + if (params.baseReq.flags?.maestro?.optional === true) { return { ok: true, data: { skipped: true, optional: true, selector } }; } return ( @@ -227,6 +362,45 @@ async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { + const target = await resolveMaestroSnapshotTarget(params, selector, options, 'tapOn'); + if (!target.ok) return target.response; + const point = pointForMaestroTapOnTarget(target.target, selector); + return await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [String(point.x), String(point.y)], + }); +} + +async function invokeMaestroSwipeOn(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const [selector, direction = 'up', durationMs] = params.positionals; + if (!selector) return errorResponse('INVALID_ARGS', 'swipe.label requires a label selector.'); + const target = await resolveMaestroSnapshotTarget(params, selector, {}, 'swipe.label'); + if (!target.ok) return target.response; + const swipe = swipeCoordinatesFromTarget(target.target, direction); + if (!swipe.ok) return swipe.response; + return await params.invoke({ + ...params.baseReq, + command: 'swipe', + positionals: [ + String(swipe.start.x), + String(swipe.start.y), + String(swipe.end.x), + String(swipe.end.y), + ...(durationMs ? [durationMs] : []), + ], + }); +} + async function invokeMaestroFuzzyTapOn( params: MaestroTapOnParams, query: string, @@ -235,18 +409,350 @@ async function invokeMaestroFuzzyTapOn( ...params.baseReq, command: 'find', positionals: [query, 'click'], + flags: { + ...params.baseReq.flags, + findFirst: true, + }, }); if (findResponse.ok) return { retry: false, response: findResponse }; - if (params.baseReq.flags?.maestroOptional !== true) { - return { retry: true, response: findResponse }; - } + return { retry: true, response: findResponse }; +} - const nativeLabelResponse = await params.invoke({ +async function resolveMaestroSnapshotTarget( + params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + }, + selector: string, + options: MaestroTapOnOptions, + commandLabel: string, +): Promise<{ ok: true; target: MaestroSnapshotTarget } | { ok: false; response: DaemonResponse }> { + const snapshotResponse = await params.invoke({ ...params.baseReq, - command: 'click', - positionals: [simpleLabelSelector(query)], + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, }); - return { retry: !nativeLabelResponse.ok, response: nativeLabelResponse }; + if (!snapshotResponse.ok) return { ok: false, response: snapshotResponse }; + + const snapshot = readSnapshotState(snapshotResponse.data); + if (!snapshot) { + return { + ok: false, + response: errorResponse( + 'COMMAND_FAILED', + `Unable to read snapshot data for ${commandLabel}.`, + ), + }; + } + + const frame = getSnapshotReferenceFrame(snapshot); + const resolution = resolveMaestroNodeFromSnapshot( + snapshot, + selector, + options, + readMaestroSelectorPlatform(params.baseReq.flags), + frame, + ); + if (!resolution.ok) { + return { + ok: false, + response: errorResponse('ELEMENT_NOT_FOUND', resolution.message), + }; + } + return { + ok: true, + target: { + node: resolution.node, + rect: resolution.rect, + frame, + }, + }; +} + +function resolveMaestroNodeFromSnapshot( + snapshot: SnapshotState, + selector: string, + options: MaestroTapOnOptions, + platform: Platform, + frame: TouchReferenceFrame | undefined, +): { ok: true; node: SnapshotNode; rect: Rect } | { ok: false; message: string } { + let matches = findMaestroSelectorMatches(snapshot, selector, platform); + if (options.childOf) { + const parents = findMaestroSelectorMatches(snapshot, options.childOf, platform); + if (parents.length === 0) { + return { ok: false, message: `Maestro childOf parent did not match: ${options.childOf}` }; + } + matches = matches.filter((node) => + parents.some((parent) => isDescendantOfSnapshotNode(snapshot.nodes, node, parent)), + ); + } + + const target = selectMaestroSnapshotMatch( + snapshot.nodes, + matches, + options.index, + extractMaestroVisibleTextQuery(selector) !== null, + frame, + ); + if (!target) { + const index = options.index ?? 0; + return { + ok: false, + message: `Maestro selector did not match index ${index}: ${selector}`, + }; + } + return { ok: true, node: target.node, rect: target.rect }; +} + +function findMaestroSelectorMatches( + snapshot: SnapshotState, + selectorExpression: string, + platform: Platform, +): SnapshotNode[] { + const chain = parseSelectorChain(selectorExpression); + for (const selector of chain.selectors) { + const matches = snapshot.nodes.filter((node) => matchesSelector(node, selector, platform)); + if (matches.length > 0) return matches; + } + return []; +} + +function resolveNodeRect(nodes: SnapshotState['nodes'], node: SnapshotNode): Rect | null { + if (node.rect && node.rect.width > 0 && node.rect.height > 0) return node.rect; + let current: SnapshotNode | undefined = node; + const byIndex = new Map(nodes.map((candidate) => [candidate.index, candidate])); + while (typeof current.parentIndex === 'number') { + current = byIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; + if (!current) return null; + if (current.rect && current.rect.width > 0 && current.rect.height > 0) return current.rect; + } + return null; +} + +function selectMaestroSnapshotMatch( + nodes: SnapshotState['nodes'], + matches: SnapshotNode[], + index: number | undefined, + preferOnScreen: boolean, + frame: TouchReferenceFrame | undefined, +): { node: SnapshotNode; rect: Rect } | null { + const resolved = matches + .map((node) => { + const rect = resolveNodeRect(nodes, node); + return rect ? { node, rect } : null; + }) + .filter((candidate): candidate is { node: SnapshotNode; rect: Rect } => Boolean(candidate)); + const candidates = + preferOnScreen && index === undefined ? preferOnScreenMatches(resolved, frame) : resolved; + if (index !== undefined) return candidates[index] ?? null; + return candidates.sort(compareMaestroSnapshotMatches)[0] ?? null; +} + +function preferOnScreenMatches( + matches: { node: SnapshotNode; rect: Rect }[], + frame: TouchReferenceFrame | undefined, +): { node: SnapshotNode; rect: Rect }[] { + const onScreen = matches.filter((match) => isRectOnScreen(match.rect, frame)); + return onScreen.length > 0 ? onScreen : matches; +} + +function isRectOnScreen(rect: Rect, frame: TouchReferenceFrame | undefined): boolean { + const maxX = frame?.referenceWidth ?? Number.POSITIVE_INFINITY; + const maxY = frame?.referenceHeight ?? Number.POSITIVE_INFINITY; + return rect.x < maxX && rect.y < maxY && rect.x + rect.width > 0 && rect.y + rect.height > 0; +} + +function compareMaestroSnapshotMatches( + left: { node: SnapshotNode; rect: Rect }, + right: { node: SnapshotNode; rect: Rect }, +): number { + const typeRank = maestroTapTargetTypeRank(left.node) - maestroTapTargetTypeRank(right.node); + if (typeRank !== 0) return typeRank; + + const areaRank = left.rect.width * left.rect.height - right.rect.width * right.rect.height; + if (areaRank !== 0) return areaRank; + + return (right.node.depth ?? 0) - (left.node.depth ?? 0); +} + +function maestroTapTargetTypeRank(node: SnapshotNode): number { + switch (node.type?.toLowerCase()) { + case 'button': + case 'link': + case 'textfield': + case 'textview': + case 'searchfield': + case 'switch': + case 'slider': + return 0; + case 'cell': + return 1; + case 'statictext': + return 2; + default: + return 3; + } +} + +function isDescendantOfSnapshotNode( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + ancestor: SnapshotNode, +): boolean { + let current: SnapshotNode | undefined = node; + const byIndex = new Map(nodes.map((candidate) => [candidate.index, candidate])); + while (typeof current.parentIndex === 'number') { + current = byIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; + if (!current) return false; + if (current === ancestor || current.index === ancestor.index) return true; + } + return false; +} + +function readMaestroTapOnOptions( + rawOptions: string | undefined, +): { ok: true; value: MaestroTapOnOptions | null } | { ok: false; response: DaemonResponse } { + if (!rawOptions) return { ok: true, value: null }; + try { + const value = JSON.parse(rawOptions) as MaestroTapOnOptions; + return { ok: true, value }; + } catch { + return { + ok: false, + response: errorResponse('INVALID_ARGS', 'tapOn runtime options must be valid JSON.'), + }; + } +} + +function readMaestroSelectorPlatform(flags: ReplayBaseRequest['flags']): Platform { + return flags?.platform === 'android' ? 'android' : 'ios'; +} + +function swipeCoordinatesFromTarget( + target: MaestroSnapshotTarget, + direction: string, +): + | { ok: true; start: { x: number; y: number }; end: { x: number; y: number } } + | { ok: false; response: DaemonResponse } { + const center = pointInsideRect(target.rect); + const frame = target.frame; + const horizontalDistance = swipeDistance(frame?.referenceWidth, target.rect.width); + const verticalDistance = swipeDistance(frame?.referenceHeight, target.rect.height); + const minX = 8; + const minY = 8; + const maxX = frame ? frame.referenceWidth - 8 : center.x + horizontalDistance; + const maxY = frame ? frame.referenceHeight - 8 : center.y + verticalDistance; + switch (direction.toLowerCase()) { + case 'up': + return { + ok: true, + start: center, + end: { x: center.x, y: clampCoordinate(center.y - verticalDistance, minY, maxY) }, + }; + case 'down': + return { + ok: true, + start: center, + end: { x: center.x, y: clampCoordinate(center.y + verticalDistance, minY, maxY) }, + }; + case 'left': + return { + ok: true, + start: center, + end: { x: clampCoordinate(center.x - horizontalDistance, minX, maxX), y: center.y }, + }; + case 'right': + return { + ok: true, + start: center, + end: { x: clampCoordinate(center.x + horizontalDistance, minX, maxX), y: center.y }, + }; + default: + return { + ok: false, + response: errorResponse( + 'INVALID_ARGS', + 'swipe.label direction must be up, down, left, or right.', + ), + }; + } +} + +function swipeDistance(frameSize: number | undefined, rectSize: number): number { + const screenRelative = typeof frameSize === 'number' ? frameSize * 0.35 : 0; + return Math.round(Math.min(360, Math.max(120, screenRelative, rectSize * 1.5))); +} + +function clampCoordinate(value: number, min: number, max: number): number { + return Math.round(Math.min(max, Math.max(min, value))); +} + +function pointInsideRect(rect: Rect): { x: number; y: number } { + return { + x: interiorCoordinate(rect.x, rect.width), + y: interiorCoordinate(rect.y, rect.height), + }; +} + +function pointForMaestroTapOnTarget( + target: MaestroSnapshotTarget, + selector: string, +): { x: number; y: number } { + if (!shouldBiasMaestroVisibleTextTap(target.node, selector, target.rect)) { + return pointInsideRect(target.rect); + } + return { + x: interiorCoordinate(target.rect.x, Math.min(target.rect.width, 168)), + y: interiorCoordinate(target.rect.y, Math.min(target.rect.height, 48)), + }; +} + +function shouldBiasMaestroVisibleTextTap( + node: SnapshotNode, + selector: string, + rect: Rect, +): boolean { + if (!extractMaestroVisibleTextQuery(selector)) return false; + if (rect.height < 70 || rect.width < 120) return false; + const type = node.type?.toLowerCase(); + return type === 'cell' || type === 'other' || type === 'scrollview'; +} + +function interiorCoordinate(origin: number, size: number): number { + if (size <= 1) return Math.floor(origin); + const min = Math.ceil(origin); + const max = Math.floor(origin + size - 1); + return clampCoordinate(origin + size / 2, min, max); +} + +function invokeMaestroRunScript(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + scope: ReplayVarScope; +}): DaemonResponse { + const [scriptPath] = params.positionals; + if (!scriptPath) { + return errorResponse('INVALID_ARGS', 'runScript requires a file path.'); + } + try { + const outputEnv = executeRunScriptFile({ + scriptPath, + env: { + ...params.scope.values, + ...(params.baseReq.flags?.maestro?.runScriptEnv ?? {}), + }, + }); + return { ok: true, data: { outputEnv } }; + } catch (error) { + const appError = asAppError(error); + return errorResponse(appError.code, appError.message, appError.details); + } } async function invokeMaestroRunFlowWhen(params: { @@ -324,40 +830,11 @@ async function invokeMaestroRunFlowWhenSteps( function isMaestroWhenConditionMiss(response: DaemonResponse): boolean { if (response.ok) return response.data?.pass === false; - if (response.error.code !== 'COMMAND_FAILED') return false; - return response.error.details?.blockedBy !== 'android_foreground_surface'; -} - -async function invokeMaestroPressEnter(params: { - baseReq: ReplayBaseRequest; - invoke: (req: DaemonRequest) => Promise; -}): Promise { - const response = await params.invoke({ - ...params.baseReq, - command: 'type', - positionals: ['\n'], - }); - if (response.ok) return response; - const message = response.error.message.toLowerCase(); - if (!message.includes('fetch failed')) return response; - - // Maestro compatibility: some iOS apps submit on Enter and immediately reset - // the runner transport. Treat this as recovered only after a fresh snapshot - // proves the runner connection is usable again; it does not assert UI state. - const snapshotResponse = await params.invoke({ - ...params.baseReq, - command: 'snapshot', - positionals: [], - flags: { ...params.baseReq.flags, noRecord: true }, - }); - if (!snapshotResponse.ok) return response; - return { - ok: true, - data: { - recovered: true, - warning: 'Enter key submit reset the iOS runner transport; recovered after snapshot.', - }, - }; + const details = response.error.details; + return ( + details?.command === 'is' && + (details.reason === 'selector_not_found' || details.reason === 'predicate_failed') + ); } function batchStepToSessionAction( @@ -389,16 +866,12 @@ function extractMaestroVisibleTextQuery(selectorExpression: string): string | nu return first; } -function simpleLabelSelector(value: string): string { - return `label=${JSON.stringify(value)}`; -} - function withMaestroScrollTimeoutContext( - response: DaemonResponse | undefined, + response: FailedDaemonResponse | null, selector: string, timeoutMs: number, ): DaemonResponse { - if (!response || response.ok) { + if (!response) { return errorResponse( 'COMMAND_FAILED', `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}`, diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index b2774457d..9b830fee0 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -8,17 +8,14 @@ import { SessionStore } from '../session-store.ts'; import { type ReplayScriptMetadata, writeReplayScript } from '../../replay/script.ts'; import { healReplayAction } from './session-replay-heal.ts'; import { formatScriptActionSummary } from '../../replay/script-utils.ts'; -import { mergeParentFlags } from './handler-utils.ts'; import { errorResponse } from './response.ts'; -import { invokeMaestroRuntimeCommand } from './session-replay-maestro-runtime.ts'; +import { invokeReplayAction } from './session-replay-action-runtime.ts'; import { buildReplayVarScope, collectReplayShellEnv, parseReplayCliEnvEntries, readReplayCliEnvEntries, readReplayShellEnvSource, - resolveReplayAction, - type ReplayVarScope, } from '../../replay/vars.ts'; // fallow-ignore-next-line complexity @@ -158,87 +155,6 @@ export async function runReplayScriptFile(params: { } } -async function invokeReplayAction(params: { - req: DaemonRequest; - sessionName: string; - action: SessionAction; - scope: ReplayVarScope; - filePath: string; - line: number; - step: number; - tracePath?: string; - invoke: (req: DaemonRequest) => Promise; -}): Promise { - const { req, sessionName, action, scope, filePath, line, step, tracePath, invoke } = params; - const resolved = resolveReplayAction(action, scope, { file: filePath, line }); - const startedAt = Date.now(); - appendReplayTraceEvent(tracePath, { - type: 'replay_action_start', - ts: new Date(startedAt).toISOString(), - replayPath: filePath, - line, - step, - command: resolved.command, - positionals: resolved.positionals ?? [], - }); - const flags = buildReplayActionFlags(req.flags, resolved.flags); - const baseReq = { - token: req.token, - session: sessionName, - flags, - runtime: resolved.runtime, - meta: req.meta, - }; - const response = - (await invokeMaestroRuntimeCommand({ - command: resolved.command, - baseReq, - positionals: resolved.positionals ?? [], - batchSteps: resolved.flags?.batchSteps, - line, - step, - invoke, - invokeReplayAction: async (nested) => - await invokeReplayAction({ - req, - sessionName, - action: nested.action, - scope, - filePath, - line: nested.line, - step: nested.step, - tracePath, - invoke, - }), - })) ?? - (await invoke({ - ...baseReq, - command: resolved.command, - positionals: resolved.positionals ?? [], - })); - const finishedAt = Date.now(); - appendReplayTraceEvent(tracePath, { - type: 'replay_action_stop', - ts: new Date(finishedAt).toISOString(), - replayPath: filePath, - line, - step, - command: resolved.command, - ok: response.ok, - durationMs: finishedAt - startedAt, - errorCode: response.ok ? undefined : response.error.code, - }); - return response; -} - -function appendReplayTraceEvent( - tracePath: string | undefined, - event: Record, -): void { - if (!tracePath) return; - fs.appendFileSync(tracePath, `${JSON.stringify(event)}\n`); -} - // fallow-ignore-next-line complexity function buildReplayBuiltinVars(params: { req: DaemonRequest; @@ -340,29 +256,21 @@ function isReplayArtifactPath(candidate: string): boolean { } } -export function buildReplayActionFlags( - parentFlags: CommandFlags | undefined, - actionFlags: SessionAction['flags'] | undefined, -): CommandFlags { - return mergeParentFlags(parentFlags, { ...(actionFlags ?? {}) }); -} - // fallow-ignore-next-line complexity function actionsContainInterpolation(actions: SessionAction[]): boolean { for (const action of actions) { for (const positional of action.positionals ?? []) { if (typeof positional === 'string' && positional.includes('${')) return true; } - if (action.flags) { - for (const value of Object.values(action.flags)) { - if (typeof value === 'string' && value.includes('${')) return true; - } - } - if (action.runtime) { - for (const value of Object.values(action.runtime)) { - if (typeof value === 'string' && value.includes('${')) return true; - } - } + if (containsInterpolation(action.flags)) return true; + if (containsInterpolation(action.runtime)) return true; } return false; } + +function containsInterpolation(value: unknown): boolean { + if (typeof value === 'string') return value.includes('${'); + if (Array.isArray(value)) return value.some(containsInterpolation); + if (value && typeof value === 'object') return Object.values(value).some(containsInterpolation); + return false; +} diff --git a/src/daemon/handlers/session.ts b/src/daemon/handlers/session.ts index 571c09f0b..08c0c05c3 100644 --- a/src/daemon/handlers/session.ts +++ b/src/daemon/handlers/session.ts @@ -203,13 +203,15 @@ export async function handleSessionCommands(params: { if (req.command === PUBLIC_COMMANDS.keyboard) { const session = sessionStore.get(sessionName); const keyboardAction = req.positionals?.[0]?.trim().toLowerCase(); - if (!session && keyboardAction === 'dismiss') { + const needsForegroundIosApp = + keyboardAction === 'dismiss' || keyboardAction === 'enter' || keyboardAction === 'return'; + if (!session && needsForegroundIosApp) { const flags = req.flags ?? {}; const normalizedPlatform = normalizePlatformSelector(flags.platform); if (normalizedPlatform === 'ios') { return errorResponse( 'SESSION_NOT_FOUND', - 'iOS keyboard dismiss requires an active session so the target app stays foregrounded. Run open first.', + 'iOS keyboard action requires an active session so the target app stays foregrounded. Run open first.', ); } } diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index 45c750521..4a43fd173 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -47,6 +47,10 @@ export async function homeAndroid(device: DeviceInfo): Promise { await runAndroidAdb(device, ['shell', 'input', 'keyevent', '3']); } +export async function pressAndroidEnter(device: DeviceInfo): Promise { + await runAndroidAdb(device, ['shell', 'input', 'keyevent', 'ENTER']); +} + export async function rotateAndroid( device: DeviceInfo, orientation: DeviceRotation, diff --git a/src/platforms/ios/__tests__/runner-client.test.ts b/src/platforms/ios/__tests__/runner-client.test.ts index 424179bae..ff12c6da4 100644 --- a/src/platforms/ios/__tests__/runner-client.test.ts +++ b/src/platforms/ios/__tests__/runner-client.test.ts @@ -31,6 +31,7 @@ vi.mock('../runner-macos-products.ts', async () => { import type { DeviceInfo } from '../../../utils/device.ts'; import { AppError } from '../../../utils/errors.ts'; +import type { RunnerCommand } from '../runner-contract.ts'; import { assertSafeDerivedCleanup, isRetryableRunnerError, @@ -98,6 +99,70 @@ const macOsDevice: DeviceInfo = { booted: true, }; +const runnerProtocolCommandFixtures: Record = { + tap: { command: 'tap', x: 120, y: 240 }, + mouseClick: { command: 'mouseClick', x: 120, y: 240, button: 'secondary' }, + tapSeries: { command: 'tapSeries', x: 120, y: 240, count: 2, intervalMs: 80 }, + longPress: { command: 'longPress', x: 120, y: 240, durationMs: 750 }, + interactionFrame: { command: 'interactionFrame' }, + drag: { command: 'drag', x: 120, y: 240, x2: 300, y2: 420, durationMs: 400 }, + dragSeries: { + command: 'dragSeries', + x: 120, + y: 240, + x2: 300, + y2: 420, + count: 2, + pauseMs: 100, + pattern: 'ping-pong', + }, + remotePress: { command: 'remotePress', remoteButton: 'down', durationMs: 250 }, + type: { command: 'type', text: 'hello', delayMs: 20, textEntryMode: 'replace' }, + swipe: { command: 'swipe', direction: 'down', durationMs: 250 }, + findText: { command: 'findText', text: 'Settings' }, + querySelector: { command: 'querySelector', selectorKey: 'id', selectorValue: 'submit' }, + readText: { command: 'readText' }, + snapshot: { + command: 'snapshot', + interactiveOnly: true, + compact: true, + depth: 2, + scope: 'app', + raw: false, + }, + screenshot: { command: 'screenshot', outPath: '/tmp/runner-screenshot.png', fullscreen: true }, + back: { command: 'back' }, + backInApp: { command: 'backInApp' }, + backSystem: { command: 'backSystem' }, + home: { command: 'home' }, + rotate: { command: 'rotate', orientation: 'landscape-left' }, + appSwitcher: { command: 'appSwitcher' }, + keyboardDismiss: { command: 'keyboardDismiss' }, + keyboardReturn: { command: 'keyboardReturn' }, + alert: { command: 'alert', action: 'accept' }, + pinch: { command: 'pinch', scale: 0.5 }, + rotateGesture: { command: 'rotateGesture', degrees: 35, x: 200, y: 420, velocity: 1 }, + transformGesture: { + command: 'transformGesture', + x: 200, + y: 420, + dx: 80, + dy: -40, + scale: 2, + degrees: 35, + durationMs: 700, + }, + recordStart: { + command: 'recordStart', + outPath: '/tmp/runner-recording.mp4', + fps: 30, + quality: 7, + }, + recordStop: { command: 'recordStop' }, + uptime: { command: 'uptime' }, + shutdown: { command: 'shutdown' }, +}; + const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../'); async function makeTmpDir(): Promise { @@ -266,6 +331,55 @@ test('resolveRunnerDestination uses simulator destination for simulators', () => assert.equal(resolveRunnerDestination(iosSimulator), 'platform=iOS Simulator,id=sim-1'); }); +test('runner protocol fixtures cover every runner command with JSON-safe samples', () => { + const commands = Object.keys(runnerProtocolCommandFixtures).sort(); + assert.deepEqual(commands, [ + 'alert', + 'appSwitcher', + 'back', + 'backInApp', + 'backSystem', + 'drag', + 'dragSeries', + 'findText', + 'home', + 'interactionFrame', + 'keyboardDismiss', + 'keyboardReturn', + 'longPress', + 'mouseClick', + 'pinch', + 'querySelector', + 'readText', + 'recordStart', + 'recordStop', + 'remotePress', + 'rotate', + 'rotateGesture', + 'screenshot', + 'shutdown', + 'snapshot', + 'swipe', + 'tap', + 'tapSeries', + 'transformGesture', + 'type', + 'uptime', + ]); + + const roundTrip = JSON.parse(JSON.stringify(runnerProtocolCommandFixtures)) as Record< + string, + Record + >; + assert.equal(roundTrip.tap.command, 'tap'); + assert.equal(roundTrip.mouseClick.button, 'secondary'); + assert.equal(roundTrip.snapshot.scope, 'app'); + assert.equal(roundTrip.screenshot.fullscreen, true); + assert.equal(roundTrip.rotate.orientation, 'landscape-left'); + assert.equal(roundTrip.recordStart.fps, 30); + assert.equal(roundTrip.recordStart.quality, 7); +}); + test('resolveRunnerDestination uses device destination for physical devices', () => { assert.equal(resolveRunnerDestination(iosDevice), 'platform=iOS,id=00008110-000E12341234002E'); }); diff --git a/src/platforms/ios/interactions.ts b/src/platforms/ios/interactions.ts index eadbd1798..8cac7b47b 100644 --- a/src/platforms/ios/interactions.ts +++ b/src/platforms/ios/interactions.ts @@ -39,6 +39,7 @@ type IosRunnerOverrides = Pick< | 'longPress' | 'focus' | 'type' + | 'fillElementSelector' | 'fill' | 'scroll' | 'pinch' @@ -166,6 +167,22 @@ export function iosRunnerOverrides( runnerOpts, ); }, + fillElementSelector: async (selector, text, delayMs) => { + return await runIosRunnerCommand( + device, + { + command: 'type', + selectorKey: selector.key, + selectorValue: selector.value, + allowNonHittableSelectorTap: selector.allowNonHittableTap, + text, + delayMs, + textEntryMode: 'replace', + appBundleId: ctx.appBundleId, + }, + runnerOpts, + ); + }, fill: async (x, y, text, delayMs) => { return await runIosRunnerCommand( device, diff --git a/src/platforms/ios/runner-contract.ts b/src/platforms/ios/runner-contract.ts index e87b6ccf9..a7149a8e7 100644 --- a/src/platforms/ios/runner-contract.ts +++ b/src/platforms/ios/runner-contract.ts @@ -34,6 +34,7 @@ export type RunnerCommand = { | 'transformGesture' | 'appSwitcher' | 'keyboardDismiss' + | 'keyboardReturn' | 'alert' | 'pinch' | 'recordStart' diff --git a/src/replay/vars.ts b/src/replay/vars.ts index dc3fb7d54..969bca5d0 100644 --- a/src/replay/vars.ts +++ b/src/replay/vars.ts @@ -2,7 +2,7 @@ import { AppError } from '../utils/errors.ts'; import type { SessionAction } from '../daemon/types.ts'; export type ReplayVarScope = { - readonly values: Readonly>; + readonly values: Record; }; export type ReplayVarSources = { @@ -13,7 +13,7 @@ export type ReplayVarSources = { }; export const REPLAY_VAR_KEY_RE = /^[A-Z_][A-Z0-9_]*$/; -const INTERPOLATION_RE = /(\\\$\{)|\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-((?:[^}\\]|\\.)*))?\}/g; +const INTERPOLATION_RE = /(\\\$\{)|\$\{([A-Za-z_][A-Za-z0-9_.]*)(?::-((?:[^}\\]|\\.)*))?\}/g; const SHELL_PREFIX = 'AD_VAR_'; const RESERVED_NAMESPACE_PREFIX = 'AD_'; @@ -53,6 +53,13 @@ export function buildReplayVarScope(sources: ReplayVarSources): ReplayVarScope { return { values: merged }; } +export function mergeReplayVarScopeValues( + scope: ReplayVarScope, + values: Record, +): void { + Object.assign(scope.values, values); +} + export function collectReplayShellEnv(processEnv: NodeJS.ProcessEnv): Record { const result: Record = {}; for (const [rawKey, value] of Object.entries(processEnv)) { @@ -156,11 +163,20 @@ function resolveStringProps( loc: { file: string; line: number }, ): T | undefined { if (!obj) return obj; - const next: Record = { ...(obj as Record) }; - for (const [key, value] of Object.entries(next)) { - if (typeof value === 'string') { - next[key] = resolveReplayString(value, scope, loc); - } + return resolveStringValue(obj, scope, loc) as T; +} + +function resolveStringValue( + value: unknown, + scope: ReplayVarScope, + loc: { file: string; line: number }, +): unknown { + if (typeof value === 'string') return resolveReplayString(value, scope, loc); + if (Array.isArray(value)) return value.map((entry) => resolveStringValue(entry, scope, loc)); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [key, resolveStringValue(entry, scope, loc)]), + ); } - return next as T; + return value; } diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index 4ac3e631f..d0c2ee7c2 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -120,13 +120,14 @@ test('parseArgs recognizes command-specific flag combinations', async () => { }, { label: 'replay maestro flow', - argv: ['replay', './flow.yaml', '--maestro', '--env', 'USER=Ada'], + argv: ['replay', './flow.yaml', '--maestro', '--env', 'USER=Ada', '--timeout', '240000'], strictFlags: true, assertParsed: (parsed) => { assert.equal(parsed.command, 'replay'); assert.deepEqual(parsed.positionals, ['./flow.yaml']); assert.equal(parsed.flags.replayMaestro, true); assert.deepEqual(parsed.flags.replayEnv, ['USER=Ada']); + assert.equal(parsed.flags.timeoutMs, 240000); }, }, ]; @@ -369,6 +370,10 @@ test('parseArgs accepts keyboard subcommands', () => { const dismiss = parseArgs(['keyboard', 'dismiss'], { strictFlags: true }); assert.equal(dismiss.command, 'keyboard'); assert.deepEqual(dismiss.positionals, ['dismiss']); + + const enter = parseArgs(['keyboard', 'enter'], { strictFlags: true }); + assert.equal(enter.command, 'keyboard'); + assert.deepEqual(enter.positionals, ['enter']); }); test('parseArgs accepts scroll pixel distance flag', () => { @@ -918,6 +923,7 @@ test('usageForCommand includes Maestro replay flag', () => { assert.match(help, /doubleTapOn/); assert.match(help, /pasteText/); assert.match(help, /runFlow file\/inline/); + assert.match(help, /ordered trusted runScript/); assert.match(help, /repeat\.times/); assert.match(help, /stopApp/); assert.match(help, /Unsupported syntax fails loudly/); @@ -1424,8 +1430,11 @@ test('clipboard command usage is documented', () => { test('keyboard command usage is documented', () => { const help = usageForCommand('keyboard'); if (help === null) throw new Error('Expected command help text'); - assert.match(help, /keyboard \[status\|get\|dismiss\]/); - assert.match(help, /Inspect Android keyboard visibility\/type or dismiss the device keyboard/); + assert.match(help, /keyboard \[status\|get\|dismiss\|enter\|return\]/); + assert.match( + help, + /Inspect Android keyboard visibility\/type or press\/dismiss the device keyboard/, + ); }); test('rotate command usage is documented', () => { diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index eb295a916..d4decd992 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -1263,7 +1263,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--maestro', usageDescription: - 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with Apple-platform launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible, runScript file/env with parse-time http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + + 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with Apple-platform launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible, ordered trusted runScript file/env steps with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional, index, childOf, label, and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, eraseText for focused fields, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe plus swipe.label, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + 'Unsupported syntax fails loudly with a link to https://github.com/callstackincubator/agent-device/issues/558', }, { @@ -1288,7 +1288,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'int', min: 1, usageLabel: '--timeout ', - usageDescription: 'Test: maximum wall-clock time per script attempt', + usageDescription: 'Replay/Test: maximum wall-clock time per script attempt', }, { key: 'retries', @@ -1599,9 +1599,10 @@ const COMMAND_SCHEMAS: Record = { allowedFlags: [], }, keyboard: { - usageOverride: 'keyboard [status|get|dismiss]', - helpDescription: 'Inspect Android keyboard visibility/type or dismiss the device keyboard', - summary: 'Inspect or dismiss the device keyboard', + usageOverride: 'keyboard [status|get|dismiss|enter|return]', + helpDescription: + 'Inspect Android keyboard visibility/type or press/dismiss the device keyboard', + summary: 'Inspect, press, or dismiss the device keyboard', positionalArgs: ['action?'], allowedFlags: [], }, @@ -1676,7 +1677,7 @@ const COMMAND_SCHEMAS: Record = { replay: { helpDescription: 'Replay a recorded session', positionalArgs: ['path'], - allowedFlags: ['replayUpdate', 'replayMaestro', 'replayEnv'], + allowedFlags: ['replayUpdate', 'replayMaestro', 'replayEnv', 'timeoutMs'], skipCapabilityCheck: true, }, test: { diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 7846bf503..a42384dcd 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -61,11 +61,11 @@ Maestro compatibility translates supported YAML commands into Agent Device repla - Supported and unsupported capabilities: https://github.com/callstackincubator/agent-device/issues/558 - New focused compatibility request: https://github.com/callstackincubator/agent-device/issues/new -Currently supported areas include app launch with Apple-platform launch arguments and iOS simulator `clearState`, file and inline `runFlow` with `when.platform`, `when.visible`, and `when.notVisible`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional` and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as a Maestro compatibility feature for file/env scripts that use `http.post`, `json`, and `output` variables; it executes during flow parsing, can make network requests, and is not a native `.ad` command or security sandbox. +Currently supported areas include app launch with Apple-platform launch arguments and iOS simulator `clearState`, file and inline `runFlow` with `when.platform`, `when.visible`, and `when.notVisible`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional`, `index`, `childOf`, `label`, and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, focused-field `eraseText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, `swipe.label`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as an ordered Maestro compatibility step for trusted file/env scripts that use `http.post`, `json`, and `output` variables; it can make network requests, and is not a native `.ad` command or security sandbox. Script execution uses Node `vm` only for compatibility isolation, not for security; the script timeout bounds synchronous execution, while `http.post` requests are bounded by the helper process timeout. Output keys cannot contain `.` because exported variables are addressed as `output.`. Maestro `env` values use the same replay precedence as `.ad` files: flow `env` is the default, shell `AD_VAR_*` values override it, and CLI `-e KEY=VALUE` wins over both. -Unsupported Maestro features such as `repeat.while`, `runFlow.when.true`, full expression predicates, `evalScript`, text clearing, selector relations such as `index` / `childOf`, device utility commands, Android app launch arguments, and Android app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. +Unsupported Maestro features such as `repeat.while`, `runFlow.when.true`, full expression predicates, `evalScript`, device utility commands, Android app launch arguments, and Android app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. ## Run a lightweight `.ad` suite From d7a7ce47fa5b7c9ae50fc76882aeece0fb82f952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 26 May 2026 09:55:45 +0200 Subject: [PATCH 5/8] refactor: tighten Maestro replay compatibility boundary --- .../RunnerTests+CommandExecution.swift | 4 +- .../RunnerTests+Models.swift | 2 +- .../maestro/__tests__/replay-flow.test.ts | 12 +-- src/compat/maestro/device-actions.ts | 2 +- src/compat/maestro/interactions.ts | 2 +- src/compat/maestro/replay-flow.ts | 2 +- src/core/dispatch-context.ts | 6 +- src/core/dispatch.ts | 4 +- src/core/interactor-types.ts | 2 +- src/daemon/__tests__/context.test.ts | 4 +- src/daemon/context.ts | 2 +- src/daemon/direct-ios-selector.ts | 2 +- .../handlers/__tests__/interaction.test.ts | 6 +- src/daemon/handlers/interaction-touch.ts | 8 +- .../session-replay-maestro-runtime.ts | 81 ++++++++++++++----- src/platforms/ios/apps.ts | 2 +- src/platforms/ios/interactions.ts | 4 +- src/platforms/ios/runner-contract.ts | 2 +- 18 files changed, 94 insertions(+), 53 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 5556202a3..0965ab9ce 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -256,7 +256,7 @@ extension RunnerTests { app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue, - allowNonHittableFallback: command.allowNonHittableSelectorTap == true + allowNonHittableFallback: command.allowNonHittableCoordinateFallback == true ) if match.isAmbiguous { return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) @@ -877,7 +877,7 @@ extension RunnerTests { app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue, - allowNonHittableFallback: command.allowNonHittableSelectorTap == true + allowNonHittableFallback: command.allowNonHittableCoordinateFallback == true ) if match.isAmbiguous { return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 0020cc81f..5e60b4906 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -40,7 +40,7 @@ struct Command: Codable { let text: String? let selectorKey: String? let selectorValue: String? - let allowNonHittableSelectorTap: Bool? + let allowNonHittableCoordinateFallback: Bool? let delayMs: Int? let textEntryMode: String? let clearFirst: Bool? diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index 27db41a1b..30153fb6b 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -80,8 +80,8 @@ env: assert.equal(parsed.actions[3]?.flags.doubleTap, true); assert.equal(parsed.actions[3]?.flags.intervalMs, 150); assert.equal(parsed.actions[4]?.flags.holdMs, 3000); - assert.equal(parsed.actions[1]?.flags.maestro?.allowNonHittableSelectorTap, true); - assert.equal(parsed.actions[6]?.flags?.maestro?.allowNonHittableSelectorTap, undefined); + assert.equal(parsed.actions[1]?.flags.maestro?.allowNonHittableCoordinateFallback, true); + assert.equal(parsed.actions[6]?.flags?.maestro?.allowNonHittableCoordinateFallback, undefined); }); test('parseMaestroReplayFlow maps iOS openLink through the app id when available', () => { @@ -200,7 +200,7 @@ test('parseMaestroReplayFlow marks tapOn before inputText for snapshot tap focus ['type', ['Muted Users']], ], ); - assert.equal(parsed.actions[0]?.flags?.maestro?.allowNonHittableSelectorTap, undefined); + assert.equal(parsed.actions[0]?.flags?.maestro?.allowNonHittableCoordinateFallback, undefined); }); test('parseMaestroReplayFlow coalesces tapOn inputText while preserving pressKey Enter submit', () => { @@ -221,7 +221,7 @@ test('parseMaestroReplayFlow coalesces tapOn inputText while preserving pressKey ], ); assert.deepEqual(parsed.actionLines, [3, 3, 6]); - assert.equal(parsed.actions[1]?.flags?.maestro?.allowNonHittableSelectorTap, true); + assert.equal(parsed.actions[1]?.flags?.maestro?.allowNonHittableCoordinateFallback, true); }); test('parseMaestroReplayFlow rejects relative runScript paths without source path', () => { @@ -430,7 +430,7 @@ test('parseMaestroReplayFlow keeps visible-gated runFlow commands for runtime ev { command: '__maestroTapOn', positionals: ['label="Continue" || text="Continue" || id="Continue"'], - flags: { maestro: { allowNonHittableSelectorTap: true } }, + flags: { maestro: { allowNonHittableCoordinateFallback: true } }, }, ]); }); @@ -454,7 +454,7 @@ test('parseMaestroReplayFlow accepts launchApp reset options', () => { 'open', ['com.callstack.agentdevicelab'], { - maestro: { clearState: true }, + clearAppState: true, launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true', '-Example', 'ignored'], }, ], diff --git a/src/compat/maestro/device-actions.ts b/src/compat/maestro/device-actions.ts index b873acab2..433629b0e 100644 --- a/src/compat/maestro/device-actions.ts +++ b/src/compat/maestro/device-actions.ts @@ -42,7 +42,7 @@ export function convertLaunchApp( const shouldRelaunch = !shouldClearState && (value.stopApp === true || launchArgs.length > 0); return action('open', [appId], { ...(shouldRelaunch ? { relaunch: true } : {}), - ...(shouldClearState ? { maestro: { clearState: true } } : {}), + ...(shouldClearState ? { clearAppState: true } : {}), ...(launchArgs.length > 0 ? { launchArgs } : {}), }); } diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index eeb9f9695..6d18ff55f 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -357,7 +357,7 @@ function maestroTapOnFlags(value: unknown): SessionAction['flags'] { ...flags, maestro: { ...(flags.maestro ?? {}), - allowNonHittableSelectorTap: true, + allowNonHittableCoordinateFallback: true, }, }; } diff --git a/src/compat/maestro/replay-flow.ts b/src/compat/maestro/replay-flow.ts index c1b0f669c..a6dd4fa86 100644 --- a/src/compat/maestro/replay-flow.ts +++ b/src/compat/maestro/replay-flow.ts @@ -126,7 +126,7 @@ function optimizeInputTextActions( function clearMaestroNonHittableTap(action: SessionAction): SessionAction { const maestro = { ...(action.flags?.maestro ?? {}) }; - delete maestro.allowNonHittableSelectorTap; + delete maestro.allowNonHittableCoordinateFallback; return { ...action, flags: { diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index bb8952881..27b5b2f59 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -11,14 +11,14 @@ export type BatchStep = { }; export type MaestroRuntimeFlags = { - allowNonHittableSelectorTap?: boolean; - clearState?: boolean; + allowNonHittableCoordinateFallback?: boolean; optional?: boolean; runScriptEnv?: Record; }; export type CommandFlags = Omit & { batchSteps?: BatchStep[]; + clearAppState?: boolean; launchArgs?: string[]; maestro?: MaestroRuntimeFlags; replayBackend?: string; @@ -55,6 +55,6 @@ export type DispatchContext = ScreenshotDispatchFlags & { key: 'id' | 'label' | 'text' | 'value'; value: string; raw: string; - allowNonHittableTap?: boolean; + allowNonHittableCoordinateFallback?: boolean; }; }; diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 61a57b554..507086c7b 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -236,13 +236,13 @@ async function handleOpenCommand( if (isDeepLinkTarget(app)) { throw new AppError( 'INVALID_ARGS', - 'Maestro launchApp.clearState requires an app target, not a deep link.', + 'Clearing app state requires an app target, not a deep link.', ); } if (device.platform !== 'ios' || device.kind !== 'simulator') { throw new AppError( 'UNSUPPORTED_OPERATION', - 'Maestro launchApp.clearState is currently supported only on iOS simulators.', + 'Clearing app state is currently supported only on iOS simulators.', ); } await clearIosSimulatorAppState(device, app); diff --git a/src/core/interactor-types.ts b/src/core/interactor-types.ts index 6b5abfa36..2ddf377a8 100644 --- a/src/core/interactor-types.ts +++ b/src/core/interactor-types.ts @@ -29,7 +29,7 @@ export type ScreenshotOptions = { export type ElementSelectorTapOptions = { key: 'id' | 'label' | 'text' | 'value'; value: string; - allowNonHittableTap?: boolean; + allowNonHittableCoordinateFallback?: boolean; }; export type SnapshotOptions = BaseSnapshotOptions & { diff --git a/src/daemon/__tests__/context.test.ts b/src/daemon/__tests__/context.test.ts index 35559226d..92c1b6339 100644 --- a/src/daemon/__tests__/context.test.ts +++ b/src/daemon/__tests__/context.test.ts @@ -14,8 +14,8 @@ test('contextFromFlags forwards scroll pixels from CLI flags', () => { assert.equal(context.pixels, 240); }); -test('contextFromFlags maps Maestro clearState to generic app-state clearing', () => { - const flags: CommandFlags = { maestro: { clearState: true } }; +test('contextFromFlags forwards generic app-state clearing', () => { + const flags: CommandFlags = { clearAppState: true }; const context = contextFromFlags('/tmp/agent-device.log', flags); assert.equal(context.clearAppState, true); }); diff --git a/src/daemon/context.ts b/src/daemon/context.ts index 8234a521b..7947241d0 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -25,7 +25,7 @@ export function contextFromFlags( activity: flags?.activity, launchConsole: flags?.launchConsole, launchArgs: flags?.launchArgs, - clearAppState: flags?.maestro?.clearState, + clearAppState: flags?.clearAppState, verbose: flags?.verbose, logPath, traceLogPath, diff --git a/src/daemon/direct-ios-selector.ts b/src/daemon/direct-ios-selector.ts index aa0e82058..60dfe075f 100644 --- a/src/daemon/direct-ios-selector.ts +++ b/src/daemon/direct-ios-selector.ts @@ -6,7 +6,7 @@ export type DirectIosSelectorTarget = { key: 'id' | 'label' | 'text' | 'value'; value: string; raw: string; - allowNonHittableTap?: boolean; + allowNonHittableCoordinateFallback?: boolean; }; export function readSimpleIosSelectorTarget(params: { diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index dd9147f01..db0f9b06c 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -460,7 +460,7 @@ test('fill simple iOS id selector uses direct runner selector fill without snaps } }); -test('click simple iOS selector forwards Maestro non-hittable tap backdoor', async () => { +test('click simple iOS selector forwards Maestro non-hittable coordinate fallback', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-maestro-selector-fallback'; sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); @@ -479,7 +479,7 @@ test('click simple iOS selector forwards Maestro non-hittable tap backdoor', asy session: sessionName, command: 'click', positionals: ['id="e2eSignInAlice"'], - flags: { maestro: { allowNonHittableSelectorTap: true } }, + flags: { maestro: { allowNonHittableCoordinateFallback: true } }, }, sessionName, sessionStore, @@ -493,7 +493,7 @@ test('click simple iOS selector forwards Maestro non-hittable tap backdoor', asy key: 'id', value: 'e2eSignInAlice', raw: 'id="e2eSignInAlice"', - allowNonHittableTap: true, + allowNonHittableCoordinateFallback: true, }); }); diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index ebd7ba9f3..878cd94df 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -262,7 +262,9 @@ function readDirectIosSelectorTapTarget(params: { if (!selector) return null; return { ...selector, - ...(flags?.maestro?.allowNonHittableSelectorTap ? { allowNonHittableTap: true } : {}), + ...(flags?.maestro?.allowNonHittableCoordinateFallback + ? { allowNonHittableCoordinateFallback: true } + : {}), }; } @@ -438,7 +440,9 @@ function readDirectIosSelectorFillTarget(params: { if (!selector) return null; return { ...selector, - ...(flags?.maestro?.allowNonHittableSelectorTap ? { allowNonHittableTap: true } : {}), + ...(flags?.maestro?.allowNonHittableCoordinateFallback + ? { allowNonHittableCoordinateFallback: true } + : {}), }; } diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts index 00923591d..6d24c063e 100644 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -12,11 +12,27 @@ import { getSnapshotReferenceFrame, type TouchReferenceFrame } from '../touch-re import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; import { errorResponse } from './response.ts'; -const MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS = 500; -const MAESTRO_TAP_ON_TIMEOUT_MS = 30000; -const MAESTRO_OPTIONAL_TAP_ON_TIMEOUT_MS = 3000; -const MAESTRO_TAP_ON_RETRY_MS = 250; -const MAESTRO_ANIMATION_POLL_MS = 250; +// Keep Maestro timing and target-selection heuristics behind one policy so +// generic Agent Device command behavior does not inherit compatibility rules. +const MAESTRO_REPLAY_POLICY = { + animationPollMs: 250, + scrollUntilVisibleProbeMs: 500, + tapOnRetryMs: 250, + tapOnTimeoutMs: 30000, + optionalTapOnTimeoutMs: 3000, + swipe: { + screenRatio: 0.35, + minDistancePx: 120, + maxDistancePx: 360, + marginPx: 8, + }, + largeTextContainerBias: { + minWidth: 120, + minHeight: 70, + width: 168, + height: 48, + }, +} as const; type ReplayBaseRequest = Omit; @@ -160,7 +176,7 @@ async function invokeMaestroWaitForAnimationToEnd(params: { }); if (!response.ok) { lastResponse = response; - await sleep(MAESTRO_ANIMATION_POLL_MS); + await sleep(MAESTRO_REPLAY_POLICY.animationPollMs); continue; } const snapshot = readSnapshotState(response.data); @@ -171,7 +187,7 @@ async function invokeMaestroWaitForAnimationToEnd(params: { } previousSignature = signature; lastResponse = response; - await sleep(MAESTRO_ANIMATION_POLL_MS); + await sleep(MAESTRO_REPLAY_POLICY.animationPollMs); } return lastResponse?.ok === false @@ -191,7 +207,10 @@ async function invokeMaestroScrollUntilVisible( return errorResponse('INVALID_ARGS', 'scrollUntilVisible timeout must be a positive number.'); } const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); - const attempts = Math.max(1, Math.ceil(timeoutMs / MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS)); + const attempts = Math.max( + 1, + Math.ceil(timeoutMs / MAESTRO_REPLAY_POLICY.scrollUntilVisibleProbeMs), + ); let lastWaitResponse: FailedDaemonResponse | null = null; for (let index = 0; index < attempts; index += 1) { @@ -240,8 +259,8 @@ async function probeMaestroScrollVisibility( function scrollProbeMs(timeoutMs: number, index: number): number { return Math.min( - MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS, - Math.max(1, timeoutMs - index * MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS), + MAESTRO_REPLAY_POLICY.scrollUntilVisibleProbeMs, + Math.max(1, timeoutMs - index * MAESTRO_REPLAY_POLICY.scrollUntilVisibleProbeMs), ); } @@ -339,8 +358,8 @@ async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise Date: Tue, 26 May 2026 10:09:16 +0200 Subject: [PATCH 6/8] refactor: reduce Maestro replay quality debt --- src/commands/selector-read.ts | 31 +- src/commands/system.ts | 77 ++-- src/compat/maestro/interactions.ts | 63 +++- src/compat/maestro/replay-flow.ts | 71 ++-- src/core/dispatch.ts | 353 ++++++++++-------- src/daemon/handlers/interaction-touch.ts | 105 +++--- .../session-replay-maestro-runtime.ts | 178 +++++---- 7 files changed, 510 insertions(+), 368 deletions(-) diff --git a/src/commands/selector-read.ts b/src/commands/selector-read.ts index 526ec7d39..0940189d6 100644 --- a/src/commands/selector-read.ts +++ b/src/commands/selector-read.ts @@ -166,13 +166,7 @@ export const findCommand: RuntimeCommand { + const capture = await captureSelectorSnapshot(runtime, options, { + updateSession: true, + scope: shouldScopeFind(locator) ? options.query : undefined, + }); + const match = findBestMatchesByLocator(capture.snapshot.nodes, locator, options.query, { + requireRect: false, + }).matches[0]; + return { capture, match }; +} + async function waitForSelector( runtime: AgentDeviceRuntime, options: WaitCommandOptions, diff --git a/src/commands/system.ts b/src/commands/system.ts index 20d234b90..a8cbc5116 100644 --- a/src/commands/system.ts +++ b/src/commands/system.ts @@ -207,13 +207,7 @@ export const keyboardCommand: RuntimeCommand< throw new AppError('UNSUPPORTED_OPERATION', 'system.keyboard is not supported by this backend'); } const action = options.action ?? 'status'; - if ( - action !== 'status' && - action !== 'get' && - action !== 'dismiss' && - action !== 'enter' && - action !== 'return' - ) { + if (!isKeyboardAction(action)) { throw new AppError( 'INVALID_ARGS', 'system.keyboard action must be status, get, dismiss, enter, or return', @@ -221,11 +215,12 @@ export const keyboardCommand: RuntimeCommand< } const state = await runtime.backend.setKeyboard(toBackendContext(runtime, options), { action }); const formattedBackendResult = toBackendResult(state); + const keyboardState = isKeyboardResult(state) ? state : {}; if (action === 'enter' || action === 'return') { return { kind: 'keyboardEnterPressed', action: 'enter', - state: isKeyboardResult(state) ? state : {}, + state: keyboardState, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), ...successText('Keyboard enter pressed'), }; @@ -235,7 +230,7 @@ export const keyboardCommand: RuntimeCommand< return { kind: 'keyboardDismissed', action, - state: isKeyboardResult(state) ? state : {}, + state: keyboardState, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), ...successText(dismissed === false ? 'Keyboard already hidden' : 'Keyboard dismissed'), }; @@ -243,7 +238,7 @@ export const keyboardCommand: RuntimeCommand< return { kind: 'keyboardState', action, - state: isKeyboardResult(state) ? state : {}, + state: keyboardState, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), }; }; @@ -373,25 +368,41 @@ function normalizeAlertResult( action: BackendAlertAction, result: BackendAlertResult, ): SystemAlertCommandResult { - if (action === 'get') { - if (result.kind !== 'alertStatus') { - throw new AppError('COMMAND_FAILED', 'system.alert get returned an invalid backend result'); - } - return { kind: 'alertStatus', action, alert: result.alert }; + switch (action) { + case 'get': + return normalizeAlertStatusResult(result); + case 'wait': + return normalizeAlertWaitResult(result); + default: + return normalizeAlertHandledResult(action, result); } - if (action === 'wait') { - if (result.kind !== 'alertWait') { - throw new AppError('COMMAND_FAILED', 'system.alert wait returned an invalid backend result'); - } - return { - kind: 'alertWait', - action, - alert: result.alert, - ...(result.waitedMs !== undefined ? { waitedMs: result.waitedMs } : {}), - ...(result.timedOut !== undefined ? { timedOut: result.timedOut } : {}), - ...successText(result.alert ? 'Alert visible' : 'Alert wait timed out'), - }; +} + +function normalizeAlertStatusResult(result: BackendAlertResult): SystemAlertCommandResult { + if (result.kind !== 'alertStatus') { + throw new AppError('COMMAND_FAILED', 'system.alert get returned an invalid backend result'); + } + return { kind: 'alertStatus', action: 'get', alert: result.alert }; +} + +function normalizeAlertWaitResult(result: BackendAlertResult): SystemAlertCommandResult { + if (result.kind !== 'alertWait') { + throw new AppError('COMMAND_FAILED', 'system.alert wait returned an invalid backend result'); } + return { + kind: 'alertWait', + action: 'wait', + alert: result.alert, + ...(result.waitedMs !== undefined ? { waitedMs: result.waitedMs } : {}), + ...(result.timedOut !== undefined ? { timedOut: result.timedOut } : {}), + ...successText(result.alert ? 'Alert visible' : 'Alert wait timed out'), + }; +} + +function normalizeAlertHandledResult( + action: Extract, + result: BackendAlertResult, +): SystemAlertCommandResult { if (result.kind !== 'alertHandled') { throw new AppError( 'COMMAND_FAILED', @@ -408,6 +419,18 @@ function normalizeAlertResult( }; } +function isKeyboardAction( + action: string, +): action is 'status' | 'get' | 'dismiss' | 'enter' | 'return' { + return ( + action === 'status' || + action === 'get' || + action === 'dismiss' || + action === 'enter' || + action === 'return' + ); +} + function isKeyboardResult(value: unknown): value is BackendKeyboardResult { return Boolean(value && typeof value === 'object'); } diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index 6d18ff55f..897ec824c 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -189,27 +189,56 @@ export function convertSwipe(value: unknown, context: MaestroParseContext): Sess assertOnlyKeys(value, 'swipe', ['start', 'end', 'direction', 'duration', 'from', 'label']); const from = value.from ?? (typeof value.label === 'string' ? value.label : undefined); if (from !== undefined) { - const direction = readSwipeDirection( - typeof value.direction === 'string' ? value.direction : 'up', - ); - return action(MAESTRO_RUNTIME_COMMAND.swipeOn, [ - maestroSelector(from, 'swipe.from', [], context), - direction, - ...swipeDurationPositionals(value), - ]); + return convertTargetedSwipe(value, from, context); } if (typeof value.direction === 'string') { return action('scroll', readScrollPositionalsFromDirectionSwipe(value.direction)); } - if (typeof value.start !== 'string' || typeof value.end !== 'string') { - throw unsupportedMaestroSyntax('Only Maestro swipe start/end coordinates are supported.'); - } - const start = parseMaestroPoint(value.start); - const end = parseMaestroPoint(value.end); - const durationMs = - typeof value.duration === 'number' && Number.isFinite(value.duration) - ? String(Math.max(16, Math.floor(value.duration))) - : undefined; + return convertCoordinateSwipe(value); +} + +function convertTargetedSwipe( + value: Record, + from: unknown, + context: MaestroParseContext, +): SessionAction { + const direction = readSwipeDirection( + typeof value.direction === 'string' ? value.direction : 'up', + ); + return action(MAESTRO_RUNTIME_COMMAND.swipeOn, [ + maestroSelector(from, 'swipe.from', [], context), + direction, + ...swipeDurationPositionals(value), + ]); +} + +function convertCoordinateSwipe(value: Record): SessionAction { + const { start, end } = readCoordinateSwipePoints(value); + const durationMs = readSwipeDurationMs(value.duration); + return convertCoordinateSwipePoints(start, end, durationMs); +} + +function readCoordinateSwipePoints(value: Record): { + start: ReturnType; + end: ReturnType; +} { + if (typeof value.start === 'string' && typeof value.end === 'string') { + return { start: parseMaestroPoint(value.start), end: parseMaestroPoint(value.end) }; + } + throw unsupportedMaestroSyntax('Only Maestro swipe start/end coordinates are supported.'); +} + +function readSwipeDurationMs(duration: unknown): string | undefined { + return typeof duration === 'number' && Number.isFinite(duration) + ? String(Math.max(16, Math.floor(duration))) + : undefined; +} + +function convertCoordinateSwipePoints( + start: ReturnType, + end: ReturnType, + durationMs: string | undefined, +): SessionAction { if (start.kind === 'absolute' && end.kind === 'absolute') { return action('swipe', [ String(start.x), diff --git a/src/compat/maestro/replay-flow.ts b/src/compat/maestro/replay-flow.ts index a6dd4fa86..eab80c650 100644 --- a/src/compat/maestro/replay-flow.ts +++ b/src/compat/maestro/replay-flow.ts @@ -82,41 +82,16 @@ function optimizeInputTextActions( actions: SessionAction[], actionLines: number[], ): { actions: SessionAction[]; actionLines: number[] } { - const maestroTapTimeoutMs = '30000'; const mergedActions: SessionAction[] = []; const mergedLines: number[] = []; for (let index = 0; index < actions.length; index += 1) { const action = actions[index]; - const nextAction = actions[index + 1]; - const typedAfterTap = readPlainTypeText(nextAction); - if (typedAfterTap !== null) { - const tapSelector = readPlainMaestroTapSelector(action); - const pressEnterAfterType = - actions[index + 2]?.command === MAESTRO_RUNTIME_COMMAND.pressEnter; - if (tapSelector !== null && pressEnterAfterType) { - mergedActions.push({ - ...action, - command: 'wait', - positionals: [tapSelector, maestroTapTimeoutMs], - }); - mergedLines.push(actionLines[index] ?? 1); - mergedActions.push({ - ...nextAction, - command: 'fill', - positionals: [tapSelector, typedAfterTap], - flags: action.flags, - }); - mergedLines.push(actionLines[index] ?? 1); - mergedActions.push(actions[index + 2] as SessionAction); - mergedLines.push(actionLines[index + 2] ?? actionLines[index] ?? 1); - index += 2; - continue; - } - if (tapSelector !== null) { - mergedActions.push(clearMaestroNonHittableTap(action)); - mergedLines.push(actionLines[index] ?? 1); - continue; - } + const optimized = optimizeTypedAfterTap(actions, actionLines, index); + if (optimized) { + mergedActions.push(...optimized.actions); + mergedLines.push(...optimized.actionLines); + index += optimized.consumed - 1; + continue; } mergedActions.push(action); mergedLines.push(actionLines[index] ?? 1); @@ -124,6 +99,40 @@ function optimizeInputTextActions( return { actions: mergedActions, actionLines: mergedLines }; } +function optimizeTypedAfterTap( + actions: SessionAction[], + actionLines: number[], + index: number, +): { actions: SessionAction[]; actionLines: number[]; consumed: number } | null { + const action = actions[index]; + const nextAction = actions[index + 1]; + const typedAfterTap = readPlainTypeText(nextAction); + const tapSelector = readPlainMaestroTapSelector(action); + if (typedAfterTap === null || tapSelector === null) return null; + const line = actionLines[index] ?? 1; + if (actions[index + 2]?.command !== MAESTRO_RUNTIME_COMMAND.pressEnter) { + return { actions: [clearMaestroNonHittableTap(action)], actionLines: [line], consumed: 1 }; + } + return { + actions: [ + { + ...action, + command: 'wait', + positionals: [tapSelector, '30000'], + }, + { + ...nextAction, + command: 'fill', + positionals: [tapSelector, typedAfterTap], + flags: action.flags, + }, + actions[index + 2] as SessionAction, + ], + actionLines: [line, line, actionLines[index + 2] ?? line], + consumed: 3, + }; +} + function clearMaestroNonHittableTap(action: SessionAction): SessionAction { const maestro = { ...(action.flags?.maestro ?? {}) }; delete maestro.allowNonHittableCoordinateFallback; diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 507086c7b..05888e718 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -44,7 +44,105 @@ import { parseDeviceRotation } from './device-rotation.ts'; export { resolveTargetDevice } from './dispatch-resolve.ts'; export type { BatchStep, CommandFlags, DispatchContext } from './dispatch-context.ts'; -// fallow-ignore-next-line complexity +type DispatchCommandHandlerParams = { + device: DeviceInfo; + interactor: Interactor; + positionals: string[]; + outPath?: string; + context?: DispatchContext; + runnerCtx: RunnerContext; +}; + +type DispatchCommandHandler = ( + params: DispatchCommandHandlerParams, +) => Promise | void> | Record | void; + +const DISPATCH_COMMAND_HANDLERS: Record = { + open: ({ device, interactor, positionals, context }) => + handleOpenCommand(device, interactor, positionals, context), + close: async ({ interactor, positionals }) => { + const app = positionals[0]; + if (!app) { + return { closed: 'session', ...successText('Closed session') }; + } + await interactor.close(app); + return { app, ...successText(`Closed: ${app}`) }; + }, + press: ({ device, interactor, positionals, context }) => + handlePressCommand(device, interactor, positionals, context), + swipe: ({ device, interactor, positionals, context }) => + handleSwipeCommand(device, interactor, positionals, context), + pan: ({ interactor, positionals }) => handlePanCommand(interactor, positionals), + fling: ({ interactor, positionals }) => handleFlingCommand(interactor, positionals), + longpress: ({ interactor, positionals }) => handleLongPressCommand(interactor, positionals), + focus: ({ interactor, positionals }) => handleFocusCommand(interactor, positionals), + type: ({ interactor, positionals, context }) => + handleTypeCommand(interactor, positionals, context), + fill: ({ interactor, positionals, context }) => + handleFillCommand(interactor, positionals, context), + scroll: ({ interactor, positionals, context }) => + handleScrollCommand(interactor, positionals, context), + pinch: ({ device, interactor, positionals, context }) => + handlePinchCommand(device, interactor, positionals, context), + 'rotate-gesture': ({ device, interactor, positionals }) => + handleRotateGestureCommand(device, interactor, positionals), + 'transform-gesture': ({ device, interactor, positionals }) => + handleTransformGestureCommand(device, interactor, positionals), + 'trigger-app-event': async ({ device, interactor, positionals, context }) => { + const { eventName, payload } = parseTriggerAppEventArgs(positionals); + const eventUrl = resolveAppEventUrl(device.platform, eventName, payload); + await interactor.open(eventUrl, { appBundleId: context?.appBundleId }); + return { + event: eventName, + eventUrl, + transport: 'deep-link', + ...successText(`Triggered app event: ${eventName}`), + }; + }, + screenshot: async ({ interactor, positionals, outPath, context }) => { + const positionalPath = positionals[0]; + const screenshotPath = positionalPath ?? outPath ?? `./screenshot-${Date.now()}.png`; + await fs.mkdir(pathModule.dirname(screenshotPath), { recursive: true }); + const screenshotOptions = screenshotOptionsFromFlags(context); + await interactor.screenshot(screenshotPath, { + appBundleId: context?.appBundleId, + fullscreen: screenshotOptions.fullscreen, + stabilize: screenshotOptions.stabilize, + surface: context?.surface, + }); + return { path: screenshotPath, ...successText(`Saved screenshot: ${screenshotPath}`) }; + }, + back: async ({ interactor, context }) => { + await interactor.back(context?.backMode); + return { action: 'back', mode: context?.backMode ?? 'in-app', ...successText('Back') }; + }, + home: async ({ interactor }) => { + await interactor.home(); + return { action: 'home', ...successText('Home') }; + }, + rotate: async ({ interactor, positionals }) => { + const orientation = parseDeviceRotation(positionals[0]); + await interactor.rotate(orientation); + return { + action: 'rotate', + orientation, + ...successText(`Rotated to ${orientation}`), + }; + }, + 'app-switcher': async ({ interactor }) => { + await interactor.appSwitcher(); + return { action: 'app-switcher', ...successText('Opened app switcher') }; + }, + clipboard: ({ interactor, positionals }) => handleClipboardCommand(interactor, positionals), + keyboard: ({ device, positionals, context, runnerCtx }) => + handleKeyboardCommand(device, positionals, context, runnerCtx), + settings: ({ device, interactor, positionals, context }) => + handleSettingsCommand(device, interactor, positionals, context), + push: ({ device, positionals, context }) => handlePushCommand(device, positionals, context), + snapshot: ({ interactor, context }) => handleSnapshotCommand(interactor, context), + read: ({ device, positionals, context }) => handleReadCommand(device, positionals, context), +}; + export async function dispatchCommand( device: DeviceInfo, command: string, @@ -72,98 +170,9 @@ export async function dispatchCommand( return await withDiagnosticTimer( 'platform_command', async () => { - switch (command) { - case 'open': - return handleOpenCommand(device, interactor, positionals, context); - case 'close': { - const app = positionals[0]; - if (!app) { - return { closed: 'session', ...successText('Closed session') }; - } - await interactor.close(app); - return { app, ...successText(`Closed: ${app}`) }; - } - case 'press': - return handlePressCommand(device, interactor, positionals, context); - case 'swipe': - return handleSwipeCommand(device, interactor, positionals, context); - case 'pan': - return handlePanCommand(interactor, positionals); - case 'fling': - return handleFlingCommand(interactor, positionals); - case 'longpress': - return handleLongPressCommand(interactor, positionals); - case 'focus': - return handleFocusCommand(interactor, positionals); - case 'type': - return handleTypeCommand(interactor, positionals, context); - case 'fill': - return handleFillCommand(interactor, positionals, context); - case 'scroll': - return handleScrollCommand(interactor, positionals, context); - case 'pinch': - return handlePinchCommand(device, interactor, positionals, context); - case 'rotate-gesture': - return handleRotateGestureCommand(device, interactor, positionals); - case 'transform-gesture': - return handleTransformGestureCommand(device, interactor, positionals); - case 'trigger-app-event': { - const { eventName, payload } = parseTriggerAppEventArgs(positionals); - const eventUrl = resolveAppEventUrl(device.platform, eventName, payload); - await interactor.open(eventUrl, { appBundleId: context?.appBundleId }); - return { - event: eventName, - eventUrl, - transport: 'deep-link', - ...successText(`Triggered app event: ${eventName}`), - }; - } - case 'screenshot': { - const positionalPath = positionals[0]; - const screenshotPath = positionalPath ?? outPath ?? `./screenshot-${Date.now()}.png`; - await fs.mkdir(pathModule.dirname(screenshotPath), { recursive: true }); - const screenshotOptions = screenshotOptionsFromFlags(context); - await interactor.screenshot(screenshotPath, { - appBundleId: context?.appBundleId, - fullscreen: screenshotOptions.fullscreen, - stabilize: screenshotOptions.stabilize, - surface: context?.surface, - }); - return { path: screenshotPath, ...successText(`Saved screenshot: ${screenshotPath}`) }; - } - case 'back': - await interactor.back(context?.backMode); - return { action: 'back', mode: context?.backMode ?? 'in-app', ...successText('Back') }; - case 'home': - await interactor.home(); - return { action: 'home', ...successText('Home') }; - case 'rotate': { - const orientation = parseDeviceRotation(positionals[0]); - await interactor.rotate(orientation); - return { - action: 'rotate', - orientation, - ...successText(`Rotated to ${orientation}`), - }; - } - case 'app-switcher': - await interactor.appSwitcher(); - return { action: 'app-switcher', ...successText('Opened app switcher') }; - case 'clipboard': - return handleClipboardCommand(interactor, positionals); - case 'keyboard': - return handleKeyboardCommand(device, positionals, context, runnerCtx); - case 'settings': - return handleSettingsCommand(device, interactor, positionals, context); - case 'push': - return handlePushCommand(device, positionals, context); - case 'snapshot': - return await handleSnapshotCommand(interactor, context); - case 'read': - return handleReadCommand(device, positionals, context); - default: - throw new AppError('INVALID_ARGS', `Unknown command: ${command}`); - } + const handler = DISPATCH_COMMAND_HANDLERS[command]; + if (!handler) throw new AppError('INVALID_ARGS', `Unknown command: ${command}`); + return await handler({ device, interactor, positionals, outPath, context, runnerCtx }); }, { command, @@ -290,13 +299,7 @@ async function handleKeyboardCommand( runnerCtx: RunnerContext, ): Promise> { const action = (positionals[0] ?? 'status').toLowerCase(); - if ( - action !== 'status' && - action !== 'get' && - action !== 'dismiss' && - action !== 'enter' && - action !== 'return' - ) { + if (!isKeyboardAction(action)) { throw new AppError( 'INVALID_ARGS', 'keyboard requires a subcommand: status, get, dismiss, enter, or return', @@ -306,80 +309,108 @@ async function handleKeyboardCommand( throw new AppError('INVALID_ARGS', 'keyboard accepts at most one subcommand argument'); } if (device.platform === 'android') { - if (action === 'enter' || action === 'return') { - await pressAndroidEnter(device); - return { - platform: 'android', - action: 'enter', - ...successText('Keyboard enter pressed'), - }; - } - if (action === 'dismiss') { - const result = await dismissAndroidKeyboard(device); - return { - platform: 'android', - action: 'dismiss', - attempts: result.attempts, - wasVisible: result.wasVisible, - dismissed: result.dismissed, - visible: result.visible, - inputType: result.inputType, - type: result.type, - inputMethodPackage: result.inputMethodPackage, - focusedPackage: result.focusedPackage, - focusedResourceId: result.focusedResourceId, - inputOwner: result.inputOwner, - }; - } - const state = await getAndroidKeyboardState(device); + return await handleAndroidKeyboardCommand(device, action); + } + if (device.platform === 'ios') { + return await handleIosKeyboardCommand(device, action, context, runnerCtx); + } + throw new AppError('UNSUPPORTED_OPERATION', 'keyboard is supported only on Android and iOS'); +} + +function isKeyboardAction( + action: string, +): action is 'status' | 'get' | 'dismiss' | 'enter' | 'return' { + return ( + action === 'status' || + action === 'get' || + action === 'dismiss' || + action === 'enter' || + action === 'return' + ); +} + +async function handleAndroidKeyboardCommand( + device: DeviceInfo, + action: 'status' | 'get' | 'dismiss' | 'enter' | 'return', +): Promise> { + if (action === 'enter' || action === 'return') { + await pressAndroidEnter(device); return { platform: 'android', - action: 'status', - visible: state.visible, - inputType: state.inputType, - type: state.type, - inputMethodPackage: state.inputMethodPackage, - focusedPackage: state.focusedPackage, - focusedResourceId: state.focusedResourceId, - inputOwner: state.inputOwner, + action: 'enter', + ...successText('Keyboard enter pressed'), }; } - if (device.platform === 'ios') { - if (action !== 'dismiss' && action !== 'enter' && action !== 'return') { - throw new AppError( - 'UNSUPPORTED_OPERATION', - 'keyboard status/get is currently supported only on Android; use keyboard dismiss or enter on iOS', - ); - } - if (action === 'enter' || action === 'return') { - const result = await runIosRunnerCommand( - device, - { command: 'keyboardReturn', appBundleId: context?.appBundleId }, - runnerCtx, - ); - return { - platform: 'ios', - action: 'enter', - visible: result.visible, - wasVisible: result.wasVisible, - ...successText('Keyboard enter pressed'), - }; - } + if (action === 'dismiss') { + const result = await dismissAndroidKeyboard(device); + return { + platform: 'android', + action: 'dismiss', + attempts: result.attempts, + wasVisible: result.wasVisible, + dismissed: result.dismissed, + visible: result.visible, + inputType: result.inputType, + type: result.type, + inputMethodPackage: result.inputMethodPackage, + focusedPackage: result.focusedPackage, + focusedResourceId: result.focusedResourceId, + inputOwner: result.inputOwner, + }; + } + const state = await getAndroidKeyboardState(device); + return { + platform: 'android', + action: 'status', + visible: state.visible, + inputType: state.inputType, + type: state.type, + inputMethodPackage: state.inputMethodPackage, + focusedPackage: state.focusedPackage, + focusedResourceId: state.focusedResourceId, + inputOwner: state.inputOwner, + }; +} + +async function handleIosKeyboardCommand( + device: DeviceInfo, + action: 'status' | 'get' | 'dismiss' | 'enter' | 'return', + context: DispatchContext | undefined, + runnerCtx: RunnerContext, +): Promise> { + if (action !== 'dismiss' && action !== 'enter' && action !== 'return') { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'keyboard status/get is currently supported only on Android; use keyboard dismiss or enter on iOS', + ); + } + if (action === 'enter' || action === 'return') { const result = await runIosRunnerCommand( device, - { command: 'keyboardDismiss', appBundleId: context?.appBundleId }, + { command: 'keyboardReturn', appBundleId: context?.appBundleId }, runnerCtx, ); return { platform: 'ios', - action: 'dismiss', - wasVisible: result.wasVisible, - dismissed: result.dismissed, + action: 'enter', visible: result.visible, - ...successText(result.dismissed ? 'Keyboard dismissed' : 'Keyboard already hidden'), + wasVisible: result.wasVisible, + ...successText('Keyboard enter pressed'), }; } - throw new AppError('UNSUPPORTED_OPERATION', 'keyboard is supported only on Android and iOS'); + const result = await runIosRunnerCommand( + device, + { command: 'keyboardDismiss', appBundleId: context?.appBundleId }, + runnerCtx, + ); + return { + platform: 'ios', + action: 'dismiss', + wasVisible: result.wasVisible, + dismissed: result.dismissed, + visible: result.visible, + ...successText(result.dismissed ? 'Keyboard dismissed' : 'Keyboard already hidden'), + }; } async function handleSettingsCommand( diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 878cd94df..a82003772 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -284,11 +284,44 @@ async function dispatchDirectIosSelectorTap( session: SessionState, selector: DirectIosSelectorTarget, ): Promise { + return await dispatchDirectIosSelectorInteraction({ + params, + session, + selector, + command: 'press', + positionals: [], + extra: { selector: selector.raw }, + fallbackPhase: 'ios_direct_selector_tap_fallback', + }); +} + +async function dispatchDirectIosSelectorInteraction(params: { + params: InteractionHandlerParams; + session: SessionState; + selector: DirectIosSelectorTarget; + command: 'press' | 'fill'; + positionals: string[]; + extra: Record; + fallbackPhase: string; +}): Promise { + const { + params: handlerParams, + session, + selector, + command, + positionals, + extra, + fallbackPhase, + } = params; const actionStartedAt = Date.now(); try { const data = - (await dispatchCommand(session.device, 'press', [], params.req.flags?.out, { - ...params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), + (await dispatchCommand(session.device, command, positionals, handlerParams.req.flags?.out, { + ...handlerParams.contextFromFlags( + handlerParams.req.flags, + session.appBundleId, + session.trace?.outPath, + ), directElementSelector: selector, surface: session.surface, })) ?? {}; @@ -299,16 +332,14 @@ async function dispatchDirectIosSelectorTap( fallbackX: point.x, fallbackY: point.y, referenceFrame: readReferenceFrameFromDirectSelectorTapResult(data), - extra: { - selector: selector.raw, - }, + extra, }); return finalizeTouchInteraction({ session, - sessionStore: params.sessionStore, - command: params.req.command, - positionals: params.req.positionals ?? [], - flags: params.req.flags, + sessionStore: handlerParams.sessionStore, + command: handlerParams.req.command, + positionals: handlerParams.req.positionals ?? [], + flags: handlerParams.req.flags, result: responseData, responseData, actionStartedAt, @@ -320,7 +351,7 @@ async function dispatchDirectIosSelectorTap( } emitDiagnostic({ level: 'debug', - phase: 'ios_direct_selector_tap_fallback', + phase: fallbackPhase, data: { selector: selector.raw, error: error instanceof Error ? error.message : String(error), @@ -452,51 +483,15 @@ async function dispatchDirectIosSelectorFill( selector: DirectIosSelectorTarget, text: string, ): Promise { - const actionStartedAt = Date.now(); - try { - const data = - (await dispatchCommand(session.device, 'fill', [text], params.req.flags?.out, { - ...params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), - directElementSelector: selector, - surface: session.surface, - })) ?? {}; - const actionFinishedAt = Date.now(); - const point = readPointFromDirectSelectorTapResult(data); - const responseData = buildTouchVisualizationResult({ - data, - fallbackX: point.x, - fallbackY: point.y, - referenceFrame: readReferenceFrameFromDirectSelectorTapResult(data), - extra: { - selector: selector.raw, - text, - }, - }); - return finalizeTouchInteraction({ - session, - sessionStore: params.sessionStore, - command: params.req.command, - positionals: params.req.positionals ?? [], - flags: params.req.flags, - result: responseData, - responseData, - actionStartedAt, - actionFinishedAt, - }); - } catch (error) { - if (!isDirectIosSelectorFallbackError(error)) { - return { ok: false, error: normalizeError(error) }; - } - emitDiagnostic({ - level: 'debug', - phase: 'ios_direct_selector_fill_fallback', - data: { - selector: selector.raw, - error: error instanceof Error ? error.message : String(error), - }, - }); - return null; - } + return await dispatchDirectIosSelectorInteraction({ + params, + session, + selector, + command: 'fill', + positionals: [text], + extra: { selector: selector.raw, text }, + fallbackPhase: 'ios_direct_selector_fill_fallback', + }); } async function dispatchRuntimeInteraction< diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts index 6d24c063e..8ec68ae15 100644 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -34,6 +34,18 @@ const MAESTRO_REPLAY_POLICY = { }, } as const; +const MAESTRO_TAP_TARGET_TYPE_RANK = new Map([ + ['button', 0], + ['link', 0], + ['textfield', 0], + ['textview', 0], + ['searchfield', 0], + ['switch', 0], + ['slider', 0], + ['cell', 1], + ['statictext', 2], +]); + type ReplayBaseRequest = Omit; type MaestroReplayInvoker = (params: { @@ -163,29 +175,10 @@ async function invokeMaestroWaitForAnimationToEnd(params: { let lastResponse: DaemonResponse | undefined; while (Date.now() - startedAt < timeoutMs) { - const response = await params.invoke({ - ...params.baseReq, - command: 'snapshot', - positionals: [], - flags: { - ...params.baseReq.flags, - noRecord: true, - snapshotRaw: true, - snapshotForceFull: true, - }, - }); - if (!response.ok) { - lastResponse = response; - await sleep(MAESTRO_REPLAY_POLICY.animationPollMs); - continue; - } - const snapshot = readSnapshotState(response.data); - if (!snapshot) return response; - const signature = snapshotStabilitySignature(snapshot); - if (previousSignature === signature) { - return { ok: true, data: { stable: true, timeoutMs } }; - } - previousSignature = signature; + const response = await captureMaestroRawSnapshot(params); + const poll = readAnimationPollResult(response, previousSignature, timeoutMs); + if (poll.done) return poll.response; + previousSignature = poll.signature ?? previousSignature; lastResponse = response; await sleep(MAESTRO_REPLAY_POLICY.animationPollMs); } @@ -195,6 +188,43 @@ async function invokeMaestroWaitForAnimationToEnd(params: { : { ok: true, data: { stable: false, timeoutMs } }; } +function readAnimationPollResult( + response: DaemonResponse, + previousSignature: string | undefined, + timeoutMs: number, +): { done: true; response: DaemonResponse } | { done: false; signature?: string } { + const signature = readSnapshotStabilitySignature(response); + if (!response.ok) return { done: false }; + if (!signature) return { done: true, response }; + if (previousSignature === signature) { + return { done: true, response: { ok: true, data: { stable: true, timeoutMs } } }; + } + return { done: false, signature }; +} + +async function captureMaestroRawSnapshot(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; +}): Promise { + return await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, + }); +} + +function readSnapshotStabilitySignature(response: DaemonResponse): string | null { + if (!response.ok) return null; + const snapshot = readSnapshotState(response.data); + return snapshot ? snapshotStabilitySignature(snapshot) : null; +} + async function invokeMaestroScrollUntilVisible( params: MaestroScrollUntilVisibleParams, ): Promise { @@ -356,23 +386,34 @@ async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { + const attempt = await invokeMaestroSnapshotTapOn(params, selector, options); + if (attempt.ok) return { retry: false, response: attempt }; + if (!fuzzyTextQuery) return { retry: true, response: attempt }; + return await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); +} + async function invokeMaestroSnapshotTapOn( params: MaestroTapOnParams, selector: string, @@ -423,7 +478,9 @@ async function invokeMaestroSwipeOn(params: { async function invokeMaestroFuzzyTapOn( params: MaestroTapOnParams, query: string, -): Promise<{ retry: boolean; response: DaemonResponse }> { +): Promise< + { retry: false; response: DaemonResponse } | { retry: true; response: FailedDaemonResponse } +> { const findResponse = await params.invoke({ ...params.baseReq, command: 'find', @@ -544,14 +601,11 @@ function findMaestroSelectorMatches( function resolveNodeRect(nodes: SnapshotState['nodes'], node: SnapshotNode): Rect | null { if (node.rect && node.rect.width > 0 && node.rect.height > 0) return node.rect; - let current: SnapshotNode | undefined = node; - const byIndex = new Map(nodes.map((candidate) => [candidate.index, candidate])); - while (typeof current.parentIndex === 'number') { - current = byIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; - if (!current) return null; - if (current.rect && current.rect.width > 0 && current.rect.height > 0) return current.rect; - } - return null; + return ( + findSnapshotAncestor(nodes, node, (ancestor) => + ancestor.rect && ancestor.rect.width > 0 && ancestor.rect.height > 0 ? ancestor : null, + )?.rect ?? null + ); } function selectMaestroSnapshotMatch( @@ -601,22 +655,7 @@ function compareMaestroSnapshotMatches( } function maestroTapTargetTypeRank(node: SnapshotNode): number { - switch (node.type?.toLowerCase()) { - case 'button': - case 'link': - case 'textfield': - case 'textview': - case 'searchfield': - case 'switch': - case 'slider': - return 0; - case 'cell': - return 1; - case 'statictext': - return 2; - default: - return 3; - } + return MAESTRO_TAP_TARGET_TYPE_RANK.get(node.type?.toLowerCase() ?? '') ?? 3; } function isDescendantOfSnapshotNode( @@ -624,14 +663,27 @@ function isDescendantOfSnapshotNode( node: SnapshotNode, ancestor: SnapshotNode, ): boolean { + return Boolean( + findSnapshotAncestor(nodes, node, (candidate) => + candidate === ancestor || candidate.index === ancestor.index ? candidate : null, + ), + ); +} + +function findSnapshotAncestor( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + resolve: (ancestor: SnapshotNode) => T | null, +): T | null { let current: SnapshotNode | undefined = node; const byIndex = new Map(nodes.map((candidate) => [candidate.index, candidate])); while (typeof current.parentIndex === 'number') { current = byIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; - if (!current) return false; - if (current === ancestor || current.index === ancestor.index) return true; + if (!current) return null; + const result = resolve(current); + if (result) return result; } - return false; + return null; } function readMaestroTapOnOptions( From dc030ae0da0a4ce83eccd204808d1a36a517d829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 27 May 2026 19:39:50 +0200 Subject: [PATCH 7/8] refactor: satisfy Maestro replay fallow audit --- android-snapshot-helper/README.md | 2 + .../SnapshotInstrumentation.java | 136 ++- .../RunnerTests+Interaction.swift | 2 + src/cli/commands/client-command.ts | 9 +- src/cli/commands/generic.ts | 1 + src/client-types.ts | 22 +- src/client.ts | 1 + src/commands/system.ts | 92 +- .../maestro/__tests__/replay-flow.test.ts | 181 +++- .../maestro/__tests__/runtime-targets.test.ts | 202 ++++ src/compat/maestro/command-mapper.ts | 21 +- src/compat/maestro/flow-control.ts | 311 +++++- src/compat/maestro/interactions.ts | 90 +- src/compat/maestro/points.ts | 20 - src/compat/maestro/replay-flow.ts | 9 + src/compat/maestro/run-script.ts | 33 +- src/compat/maestro/runtime-assertions.ts | 212 ++++ src/compat/maestro/runtime-commands.ts | 3 + src/compat/maestro/runtime-flow.ts | 157 +++ src/compat/maestro/runtime-geometry.ts | 129 +++ src/compat/maestro/runtime-interactions.ts | 598 +++++++++++ src/compat/maestro/runtime-support.ts | 120 +++ src/compat/maestro/runtime-targets.ts | 454 ++++++++ src/compat/maestro/runtime.ts | 108 ++ src/compat/maestro/support.ts | 11 - src/core/__tests__/app-events.test.ts | 1 - .../__tests__/dispatch-interactions.test.ts | 5 +- src/core/__tests__/dispatch-series.test.ts | 1 - src/core/dispatch.ts | 245 ++--- src/core/interactors/android.ts | 7 +- src/daemon-client.ts | 384 ++++--- .../post-gesture-stabilization.test.ts | 24 +- .../__tests__/request-lock-policy.test.ts | 1 - src/daemon/__tests__/selectors.test.ts | 1 - src/daemon/__tests__/session-routing.test.ts | 1 - .../__tests__/interaction-flags.test.ts | 1 - .../handlers/__tests__/interaction.test.ts | 11 +- .../__tests__/session-open-target.test.ts | 1 - .../__tests__/session-replay-vars.test.ts | 844 ++++++++++++++- .../__tests__/session-test-discovery.test.ts | 24 +- .../__tests__/session-test-runtime.test.ts | 1 - src/daemon/handlers/__tests__/session.test.ts | 2 +- .../handlers/interaction-touch-targets.ts | 2 + src/daemon/handlers/interaction-touch.ts | 18 +- src/daemon/handlers/session-close.ts | 25 +- .../handlers/session-replay-action-runtime.ts | 4 +- .../session-replay-maestro-runtime.ts | 976 ------------------ src/daemon/handlers/session-replay-runtime.ts | 2 +- src/daemon/handlers/session-test-discovery.ts | 54 +- src/daemon/handlers/session-test.ts | 1 + src/daemon/handlers/snapshot-capture.ts | 5 +- src/daemon/post-gesture-stabilization.ts | 8 +- src/daemon/runtime-session.ts | 2 +- src/mcp/server.ts | 4 +- .../__tests__/adb-provider-scope.test.ts | 5 +- src/platforms/android/__tests__/index.test.ts | 32 + src/platforms/android/__tests__/perf.test.ts | 5 +- .../android/__tests__/snapshot-helper.test.ts | 57 +- .../android/__tests__/snapshot.test.ts | 96 +- src/platforms/android/app-lifecycle.ts | 196 +++- src/platforms/android/perf.ts | 2 +- .../android/snapshot-helper-capture.ts | 184 +++- .../android/snapshot-helper-types.ts | 4 + src/platforms/android/snapshot-types.ts | 1 + src/platforms/android/snapshot.ts | 358 +++++-- src/platforms/ios/runner-client.ts | 1 - src/replay/__tests__/script.test.ts | 1 - src/replay/vars.ts | 4 +- src/utils/__tests__/args.test.ts | 33 +- src/utils/__tests__/daemon-client.test.ts | 35 + src/utils/__tests__/device.test.ts | 1 - .../__tests__/selector-is-predicates.test.ts | 98 ++ src/utils/command-schema.ts | 13 +- src/utils/keyboard-actions.ts | 7 + src/utils/selector-is-predicates.ts | 9 +- src/utils/snapshot-label-signals.ts | 2 +- .../android-test-suite.test.ts | 69 ++ .../provider-scenarios/android-world.ts | 23 + .../suites/agent-device-smoke-suite.ts | 2 +- website/docs/docs/replay-e2e.md | 4 +- 80 files changed, 5072 insertions(+), 1749 deletions(-) create mode 100644 src/compat/maestro/__tests__/runtime-targets.test.ts create mode 100644 src/compat/maestro/runtime-assertions.ts create mode 100644 src/compat/maestro/runtime-flow.ts create mode 100644 src/compat/maestro/runtime-geometry.ts create mode 100644 src/compat/maestro/runtime-interactions.ts create mode 100644 src/compat/maestro/runtime-support.ts create mode 100644 src/compat/maestro/runtime-targets.ts create mode 100644 src/compat/maestro/runtime.ts delete mode 100644 src/daemon/handlers/session-replay-maestro-runtime.ts create mode 100644 src/utils/__tests__/selector-is-predicates.test.ts create mode 100644 src/utils/keyboard-actions.ts diff --git a/android-snapshot-helper/README.md b/android-snapshot-helper/README.md index f5be62591..2808a6800 100644 --- a/android-snapshot-helper/README.md +++ b/android-snapshot-helper/README.md @@ -31,6 +31,7 @@ VERSION="$(node -p 'require("./package.json").version')" adb install -r -t ".tmp/android-snapshot-helper/agent-device-android-snapshot-helper-$VERSION.apk" adb shell am instrument -w \ -e waitForIdleTimeoutMs 500 \ + -e waitForIdleQuietMs 100 \ -e timeoutMs 8000 \ -e maxDepth 128 \ -e maxNodes 5000 \ @@ -59,6 +60,7 @@ The final instrumentation result includes: - `ok=true` - `helperApiVersion=1` - `waitForIdleTimeoutMs` +- `waitForIdleQuietMs` - `timeoutMs` - `maxDepth` - `maxNodes` diff --git a/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index 629b7291b..ad211a2ec 100644 --- a/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -8,6 +8,10 @@ import android.util.Base64; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityWindowInfo; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Locale; @@ -17,8 +21,9 @@ public final class SnapshotInstrumentation extends Instrumentation { private static final String PROTOCOL = "android-snapshot-helper-v1"; private static final String OUTPUT_FORMAT = "uiautomator-xml"; private static final String HELPER_API_VERSION = "1"; - private static final int CHUNK_SIZE = 8 * 1024; + private static final int CHUNK_SIZE = 2 * 1024; private static final long DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS = 500; + private static final long DEFAULT_WAIT_FOR_IDLE_QUIET_MS = 100; private static final long DEFAULT_TIMEOUT_MS = 8_000; private static final int DEFAULT_MAX_DEPTH = 128; private static final int DEFAULT_MAX_NODES = 5_000; @@ -36,21 +41,27 @@ public void onStart() { super.onStart(); long waitForIdleTimeoutMs = readLongArgument(arguments, "waitForIdleTimeoutMs", DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS); + long waitForIdleQuietMs = + readLongArgument(arguments, "waitForIdleQuietMs", DEFAULT_WAIT_FOR_IDLE_QUIET_MS); long timeoutMs = readLongArgument(arguments, "timeoutMs", DEFAULT_TIMEOUT_MS); int maxDepth = readIntArgument(arguments, "maxDepth", DEFAULT_MAX_DEPTH); int maxNodes = readIntArgument(arguments, "maxNodes", DEFAULT_MAX_NODES); + String outputPath = readStringArgument(arguments, "outputPath"); Bundle result = new Bundle(); result.putString("agentDeviceProtocol", PROTOCOL); result.putString("helperApiVersion", HELPER_API_VERSION); result.putString("outputFormat", OUTPUT_FORMAT); result.putString("waitForIdleTimeoutMs", Long.toString(waitForIdleTimeoutMs)); + result.putString("waitForIdleQuietMs", Long.toString(waitForIdleQuietMs)); result.putString("timeoutMs", Long.toString(timeoutMs)); result.putString("maxDepth", Integer.toString(maxDepth)); result.putString("maxNodes", Integer.toString(maxNodes)); try { long startedAtMs = System.currentTimeMillis(); - CaptureResult capture = captureXml(waitForIdleTimeoutMs, maxDepth, maxNodes); + CaptureResult capture = + captureXml(waitForIdleQuietMs, waitForIdleTimeoutMs, timeoutMs, maxDepth, maxNodes); + writeOutputFile(outputPath, capture.xml); emitChunks(capture.xml); result.putString("ok", "true"); result.putString("rootPresent", Boolean.toString(capture.rootPresent)); @@ -59,26 +70,111 @@ public void onStart() { result.putString("nodeCount", Integer.toString(capture.nodeCount)); result.putString("truncated", Boolean.toString(capture.truncated)); result.putString("elapsedMs", Long.toString(System.currentTimeMillis() - startedAtMs)); - finish(0, result); + finishSafely(0, result); } catch (Throwable error) { result.putString("ok", "false"); result.putString("errorType", error.getClass().getName()); result.putString( "message", error.getMessage() == null ? error.getClass().getName() : error.getMessage()); - finish(1, result); + finishSafely(1, result); + } + } + + private static String readStringArgument(Bundle arguments, String key) { + if (arguments == null || !arguments.containsKey(key)) { + return null; + } + String value = arguments.getString(key); + return value == null || value.trim().isEmpty() ? null : value.trim(); + } + + private static void writeOutputFile(String outputPath, String xml) throws IOException { + if (outputPath == null) { + return; + } + File file = new File(outputPath); + File parent = file.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + try (FileOutputStream stream = new FileOutputStream(file, false)) { + stream.write(xml.getBytes(StandardCharsets.UTF_8)); + } + } + + private void finishSafely(int resultCode, Bundle result) { + RuntimeException lastError = null; + for (int attempt = 0; attempt < 100; attempt += 1) { + try { + finish(resultCode, result); + return; + } catch (IllegalStateException error) { + if (!isUiAutomationConnectingError(error)) { + throw error; + } + lastError = error; + sleep(100); + } + } + detachUiAutomationBeforeFinish(); + try { + finish(resultCode, result); + return; + } catch (IllegalStateException error) { + if (!isUiAutomationConnectingError(error)) { + throw error; + } + lastError = error; + } + throw lastError; + } + + private void detachUiAutomationBeforeFinish() { + try { + Field field = Instrumentation.class.getDeclaredField("mUiAutomation"); + field.setAccessible(true); + field.set(this, null); + } catch (ReflectiveOperationException | RuntimeException ignored) { + // If the platform blocks reflection, preserve the original finish failure below. + } + } + + private static boolean isUiAutomationConnectingError(IllegalStateException error) { + String message = error.getMessage(); + return message != null && message.contains("while connecting"); + } + + private static boolean isUiAutomationNotConnectedError(IllegalStateException error) { + String message = error.getMessage(); + return message != null && message.toLowerCase(Locale.ROOT).contains("not connected"); + } + + private static void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); } } @SuppressWarnings("deprecation") - private CaptureResult captureXml(long waitForIdleTimeoutMs, int maxDepth, int maxNodes) + private CaptureResult captureXml( + long waitForIdleQuietMs, + long waitForIdleTimeoutMs, + long timeoutMs, + int maxDepth, + int maxNodes) throws TimeoutException { - UiAutomation automation = getUiAutomation(); + UiAutomation automation = getConnectedUiAutomation(timeoutMs); enableInteractiveWindowRetrieval(automation); if (waitForIdleTimeoutMs > 0) { try { - // Best-effort settle: avoids empty roots without inheriting UIAutomator's long idle wait. - automation.waitForIdle(waitForIdleTimeoutMs, waitForIdleTimeoutMs); + // Best-effort settle: wait for the accessibility stream to become idle, but require only + // a short quiet window once it does. Using the full timeout as the quiet window made every + // stable snapshot pay a fixed 500 ms tax. + long quietMs = Math.min(waitForIdleQuietMs, waitForIdleTimeoutMs); + automation.waitForIdle(quietMs, waitForIdleTimeoutMs); } catch (TimeoutException ignored) { // Busy or animated apps can still expose a usable root; capture whatever is available. } @@ -109,6 +205,30 @@ private CaptureResult captureXml(long waitForIdleTimeoutMs, int maxDepth, int ma xml.toString(), windowCount > 0, captureMode, windowCount, stats.nodeCount, stats.truncated); } + private UiAutomation getConnectedUiAutomation(long timeoutMs) throws TimeoutException { + long deadlineMs = System.currentTimeMillis() + Math.max(1, timeoutMs); + UiAutomation automation = getUiAutomation(); + RuntimeException lastError = null; + while (System.currentTimeMillis() <= deadlineMs) { + try { + automation.getServiceInfo(); + return automation; + } catch (IllegalStateException error) { + if (!isUiAutomationConnectingError(error) && !isUiAutomationNotConnectedError(error)) { + throw error; + } + lastError = error; + } + sleep(50); + } + TimeoutException timeout = + new TimeoutException("Timed out waiting for Android UiAutomation to connect"); + if (lastError != null) { + timeout.initCause(lastError); + } + throw timeout; + } + private static void enableInteractiveWindowRetrieval(UiAutomation automation) { AccessibilityServiceInfo serviceInfo; try { diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index b07e4145f..cd231a953 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -375,6 +375,8 @@ extension RunnerTests { func focusedTextInput(app: XCUIApplication) -> XCUIElement? { #if os(iOS) + // iOS focus predicates can return stale or misleading text-input matches + // under XCUITest, so text entry readiness is driven by tap/keyboard state. return nil #else var focused: XCUIElement? diff --git a/src/cli/commands/client-command.ts b/src/cli/commands/client-command.ts index e52bddbf9..871c37da0 100644 --- a/src/cli/commands/client-command.ts +++ b/src/cli/commands/client-command.ts @@ -10,6 +10,7 @@ import type { import type { CliFlags } from '../../utils/command-schema.ts'; import { AppError } from '../../utils/errors.ts'; import { readCommandMessage } from '../../utils/success-text.ts'; +import { isKeyboardAction } from '../../utils/keyboard-actions.ts'; import { waitCommandCodec } from '../../command-codecs.ts'; import { parseDeviceRotation } from '../../core/device-rotation.ts'; import { buildSelectionOptions, writeCommandMessage, writeCommandOutput } from './shared.ts'; @@ -173,13 +174,7 @@ function readKeyboardAction( ): KeyboardCommandOptions['action'] | undefined { const action = value?.toLowerCase(); if (action === 'get') return 'status'; - if ( - action === undefined || - action === 'status' || - action === 'dismiss' || - action === 'enter' || - action === 'return' - ) { + if (action === undefined || (isKeyboardAction(action) && action !== 'get')) { return action; } throw new AppError( diff --git a/src/cli/commands/generic.ts b/src/cli/commands/generic.ts index 8fa93ad0e..e8c2be820 100644 --- a/src/cli/commands/generic.ts +++ b/src/cli/commands/generic.ts @@ -69,6 +69,7 @@ const genericClientCommandRunners = { ...buildSelectionOptions(flags), paths: positionals, update: flags.replayUpdate, + backend: flags.replayMaestro ? 'maestro' : undefined, env: flags.replayEnv, failFast: flags.failFast, timeoutMs: flags.timeoutMs, diff --git a/src/client-types.ts b/src/client-types.ts index 8d28aab89..dddbc47d9 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -674,20 +674,24 @@ export type FindOptions = | (FindBaseOptions & { action: 'wait'; timeoutMs?: number }) | (FindBaseOptions & { action: 'fill' | 'type'; value: string }); -export type ReplayRunOptions = AgentDeviceRequestOverrides & { - path: string; - update?: boolean; - /** @deprecated Use backend: 'maestro'. */ - maestro?: boolean; - backend?: string; - env?: string[]; - timeoutMs?: number; -}; +export type ReplayRunOptions = AgentDeviceRequestOverrides & + AgentDeviceSelectionOptions & { + path: string; + update?: boolean; + /** @deprecated Use backend: 'maestro'. */ + maestro?: boolean; + backend?: string; + env?: string[]; + timeoutMs?: number; + }; export type ReplayTestOptions = AgentDeviceRequestOverrides & AgentDeviceSelectionOptions & { paths: string[]; update?: boolean; + /** @deprecated Use backend: 'maestro'. */ + maestro?: boolean; + backend?: string; env?: string[]; failFast?: boolean; timeoutMs?: number; diff --git a/src/client.ts b/src/client.ts index 3d2f5e1ad..a9272b526 100644 --- a/src/client.ts +++ b/src/client.ts @@ -456,6 +456,7 @@ export function createAgentDeviceClient( await executeCommandRequest(PUBLIC_COMMANDS.test, options.paths, { ...options, replayUpdate: options.update, + replayBackend: options.backend ?? (options.maestro === true ? 'maestro' : undefined), replayEnv: options.env, replayShellEnv: collectReplayClientShellEnv(process.env), }), diff --git a/src/commands/system.ts b/src/commands/system.ts index a8cbc5116..9ab4ece5a 100644 --- a/src/commands/system.ts +++ b/src/commands/system.ts @@ -9,6 +9,7 @@ import type { CommandContext } from '../runtime-contract.ts'; import { AppError } from '../utils/errors.ts'; import { successText } from '../utils/success-text.ts'; import { requireIntInRange } from '../utils/validation.ts'; +import { isKeyboardAction } from '../utils/keyboard-actions.ts'; import type { RuntimeCommand } from './runtime-types.ts'; import { toBackendContext } from './selector-read-utils.ts'; import { normalizeOptionalText } from './text.ts'; @@ -217,30 +218,12 @@ export const keyboardCommand: RuntimeCommand< const formattedBackendResult = toBackendResult(state); const keyboardState = isKeyboardResult(state) ? state : {}; if (action === 'enter' || action === 'return') { - return { - kind: 'keyboardEnterPressed', - action: 'enter', - state: keyboardState, - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - ...successText('Keyboard enter pressed'), - }; + return normalizeKeyboardEnterResult(keyboardState, formattedBackendResult); } if (action === 'dismiss') { - const dismissed = isKeyboardResult(state) ? state.dismissed : undefined; - return { - kind: 'keyboardDismissed', - action, - state: keyboardState, - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - ...successText(dismissed === false ? 'Keyboard already hidden' : 'Keyboard dismissed'), - }; + return normalizeKeyboardDismissResult(action, keyboardState, formattedBackendResult); } - return { - kind: 'keyboardState', - action, - state: keyboardState, - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - }; + return normalizeKeyboardStateResult(action, keyboardState, formattedBackendResult); }; export const clipboardCommand: RuntimeCommand< @@ -368,14 +351,53 @@ function normalizeAlertResult( action: BackendAlertAction, result: BackendAlertResult, ): SystemAlertCommandResult { - switch (action) { - case 'get': - return normalizeAlertStatusResult(result); - case 'wait': - return normalizeAlertWaitResult(result); - default: - return normalizeAlertHandledResult(action, result); + if (action === 'get') { + return normalizeAlertStatusResult(result); } + if (action === 'wait') { + return normalizeAlertWaitResult(result); + } + return normalizeAlertHandledResult(action, result); +} + +function normalizeKeyboardEnterResult( + state: BackendKeyboardResult, + backendResult: Record | undefined, +): SystemKeyboardCommandResult { + return { + kind: 'keyboardEnterPressed', + action: 'enter', + state, + ...(backendResult ? { backendResult } : {}), + ...successText('Keyboard enter pressed'), + }; +} + +function normalizeKeyboardDismissResult( + action: 'dismiss', + state: BackendKeyboardResult, + backendResult: Record | undefined, +): SystemKeyboardCommandResult { + return { + kind: 'keyboardDismissed', + action, + state, + ...(backendResult ? { backendResult } : {}), + ...successText(state.dismissed === false ? 'Keyboard already hidden' : 'Keyboard dismissed'), + }; +} + +function normalizeKeyboardStateResult( + action: 'status' | 'get', + state: BackendKeyboardResult, + backendResult: Record | undefined, +): SystemKeyboardCommandResult { + return { + kind: 'keyboardState', + action, + state, + ...(backendResult ? { backendResult } : {}), + }; } function normalizeAlertStatusResult(result: BackendAlertResult): SystemAlertCommandResult { @@ -400,7 +422,7 @@ function normalizeAlertWaitResult(result: BackendAlertResult): SystemAlertComman } function normalizeAlertHandledResult( - action: Extract, + action: Exclude, result: BackendAlertResult, ): SystemAlertCommandResult { if (result.kind !== 'alertHandled') { @@ -419,18 +441,6 @@ function normalizeAlertHandledResult( }; } -function isKeyboardAction( - action: string, -): action is 'status' | 'get' | 'dismiss' | 'enter' | 'return' { - return ( - action === 'status' || - action === 'get' || - action === 'dismiss' || - action === 'enter' || - action === 'return' - ); -} - function isKeyboardResult(value: unknown): value is BackendKeyboardResult { return Boolean(value && typeof value === 'object'); } diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index 30153fb6b..72ab6ebe5 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -58,16 +58,25 @@ env: ['__maestroTapOn', ['id="home-open-form"']], ['__maestroTapPointPercent', ['20', '20']], ['click', ['id="release-notice"']], - ['click', ['label="Agent Device Tester"']], + [ + 'click', + ['label="Agent Device Tester" || text="Agent Device Tester" || id="Agent Device Tester"'], + ], ['open', ['exp://localhost:8082']], ['__maestroTapOn', ['label="Full name" || text="Full name" || id="Full name"']], ['type', ['Ada Lovelace']], - ['wait', ['label="Checkout form"', '5000']], - ['__maestroAssertNotVisible', ['label="Missing banner"']], - ['wait', ['id="submit-order"', '7000']], + [ + '__maestroAssertVisible', + ['label="Checkout form" || text="Checkout form" || id="Checkout form"', '5000'], + ], + [ + '__maestroAssertNotVisible', + ['label="Missing banner" || text="Missing banner" || id="Missing banner"'], + ], + ['__maestroAssertVisible', ['id="submit-order"', '7000']], ['scroll', ['down']], - ['scroll', ['down', '0.4']], - ['scroll', ['right']], + ['__maestroSwipeScreen', ['percent', '50', '75', '50', '35', '300']], + ['__maestroSwipeScreen', ['direction', 'left']], [ '__maestroScrollUntilVisible', ['label="Discover" || text="Discover" || id="Discover"', '5000', 'up'], @@ -99,25 +108,40 @@ test('parseMaestroReplayFlow maps iOS openLink through the app id when available ); }); -test('parseMaestroReplayFlow converts Bluesky Maestro selector compatibility syntax', () => { +test('parseMaestroReplayFlow maps Android openLink like Maestro without package binding', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- openLink: exp://localhost:8082 +`, + { platform: 'android' }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [['open', ['exp://localhost:8082']]], + ); +}); + +test('parseMaestroReplayFlow converts Maestro nested selector compatibility syntax', () => { const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- - eraseText - eraseText: 12 - tapOn: - id: likeBtn + id: childActionButton childOf: - id: postThreadItem-by-bob.test + id: parent-row-secondary - tapOn: - id: postDropdownBtn + id: overflowButton index: 0 - tapOn: - label: Display name metadata - text: Display name + label: Profile name metadata + text: Profile name - swipe: - label: Drag feed down + label: Drag item down from: - id: feed-drag-handle + id: reorder-handle direction: UP duration: 350 `); @@ -129,11 +153,11 @@ test('parseMaestroReplayFlow converts Bluesky Maestro selector compatibility syn ['type', ['\b'.repeat(12)]], [ '__maestroTapOn', - ['id="likeBtn"', JSON.stringify({ childOf: 'id="postThreadItem-by-bob.test"' })], + ['id="childActionButton"', JSON.stringify({ childOf: 'id="parent-row-secondary"' })], ], - ['__maestroTapOn', ['id="postDropdownBtn"', JSON.stringify({ index: 0 })]], - ['__maestroTapOn', ['label="Display name"']], - ['__maestroSwipeOn', ['id="feed-drag-handle"', 'up', '350']], + ['__maestroTapOn', ['id="overflowButton"', JSON.stringify({ index: 0 })]], + ['__maestroTapOn', ['label="Profile name" || text="Profile name" || id="Profile name"']], + ['__maestroSwipeOn', ['id="reorder-handle"', 'up', '350']], ], ); }); @@ -189,15 +213,15 @@ test('parseMaestroReplayFlow marks tapOn before inputText for snapshot tap focus const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- - tapOn: - id: editListNameInput -- inputText: Muted Users + id: editableNameInput +- inputText: Saved list `); assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), [ - ['__maestroTapOn', ['id="editListNameInput"']], - ['type', ['Muted Users']], + ['__maestroTapOn', ['id="editableNameInput"']], + ['type', ['Saved list']], ], ); assert.equal(parsed.actions[0]?.flags?.maestro?.allowNonHittableCoordinateFallback, undefined); @@ -224,6 +248,25 @@ test('parseMaestroReplayFlow coalesces tapOn inputText while preserving pressKey assert.equal(parsed.actions[1]?.flags?.maestro?.allowNonHittableCoordinateFallback, true); }); +test('parseMaestroReplayFlow does not coalesce text entry for non-input-looking targets', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- tapOn: Continue +- inputText: unexpected +- pressKey: Enter +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['__maestroTapOn', ['label="Continue" || text="Continue" || id="Continue"']], + ['type', ['unexpected']], + ['__maestroPressEnter', []], + ], + ); + assert.equal(parsed.actions[0]?.flags?.maestro?.allowNonHittableCoordinateFallback, undefined); +}); + test('parseMaestroReplayFlow rejects relative runScript paths without source path', () => { assert.throws( () => @@ -266,7 +309,7 @@ test('parseMaestroReplayFlow preserves selector state and absolute swipe command assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), [ - ['wait', ['id="shipping-pickup" selected="true"', '5000']], + ['__maestroAssertVisible', ['id="shipping-pickup" selected="true"', '5000']], ['swipe', ['100', '500', '100', '200', '300']], ], ); @@ -408,6 +451,50 @@ test('parseMaestroReplayFlow skips platform-gated runFlow commands for other pla ); }); +test('parseMaestroReplayFlow treats Web platform gates as non-native branches', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runFlow: + when: + platform: Web + commands: + - tapOn: Web only +- tapOn: Native +`, + { platform: 'ios' }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [['__maestroTapOn', ['label="Native" || text="Native" || id="Native"']]], + ); +}); + +test('parseMaestroReplayFlow evaluates simple runFlow.when.true platform expressions', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runFlow: + when: + true: \${maestro.platform == 'android' || maestro.platform == 'ios'} + commands: + - tapOn: Native +- runFlow: + when: + true: \${maestro.platform == 'web' || maestro.platform == 'android'} + commands: + - tapOn: Not iOS +`, + { platform: 'ios' }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [['__maestroTapOn', ['label="Native" || text="Native" || id="Native"']]], + ); +}); + test('parseMaestroReplayFlow keeps visible-gated runFlow commands for runtime evaluation', () => { const parsed = parseMaestroReplayFlow( `appId: com.callstack.agentdevicelab @@ -435,6 +522,36 @@ test('parseMaestroReplayFlow keeps visible-gated runFlow commands for runtime ev ]); }); +test('parseMaestroReplayFlow keeps retry commands for runtime evaluation', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- retry: + maxRetries: 3 + commands: + - openLink: + link: \${APP_SCHEME}details + - assertVisible: Article +`, + { env: { APP_SCHEME: 'example://' } }, + ); + + assert.equal(parsed.actions[0]?.command, '__maestroRetry'); + assert.deepEqual(parsed.actions[0]?.positionals, ['3']); + assert.deepEqual(parsed.actions[0]?.flags.batchSteps, [ + { + command: 'open', + positionals: ['example://details'], + flags: {}, + }, + { + command: '__maestroAssertVisible', + positionals: ['label="Article" || text="Article" || id="Article"', '5000'], + flags: {}, + }, + ]); +}); + test('parseMaestroReplayFlow accepts launchApp reset options', () => { const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- @@ -524,24 +641,24 @@ test('parseMaestroReplayFlow parses the test-app Maestro suite fixture', () => { assert.deepEqual( parsed.actions.map((entry) => entry.command), [ - 'wait', + '__maestroAssertVisible', '__maestroTapOn', - 'wait', + '__maestroAssertVisible', '__maestroTapOn', 'type', '__maestroTapOn', 'type', '__maestroTapOn', - 'wait', - 'wait', - 'scroll', + '__maestroAssertVisible', + '__maestroAssertVisible', + '__maestroSwipeScreen', '__maestroTapOn', - 'wait', + '__maestroAssertVisible', '__maestroTapOn', - 'wait', + '__maestroAssertVisible', '__maestroTapOn', - 'wait', - 'wait', + '__maestroAssertVisible', + '__maestroAssertVisible', ], ); }); diff --git a/src/compat/maestro/__tests__/runtime-targets.test.ts b/src/compat/maestro/__tests__/runtime-targets.test.ts new file mode 100644 index 000000000..b77ff7aa0 --- /dev/null +++ b/src/compat/maestro/__tests__/runtime-targets.test.ts @@ -0,0 +1,202 @@ +import { test, expect } from 'vitest'; +import type { SnapshotState } from '../../../utils/snapshot.ts'; +import { + resolveMaestroNodeFromSnapshot, + resolveVisibleMaestroNodeFromSnapshot, +} from '../runtime-targets.ts'; + +test('resolveVisibleMaestroNodeFromSnapshot treats app content behind React Native overlays as hidden', () => { + const snapshot = makeReactNativeOverlaySnapshot(); + + const appContent = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + 'label="Article title" || text="Article title" || id="Article title"', + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + const overlayControl = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + 'label="Minimize" || text="Minimize" || id="Minimize"', + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + + expect(appContent).toMatchObject({ + ok: false, + message: expect.stringContaining('React Native overlay is covering app content'), + }); + expect(overlayControl).toMatchObject({ + ok: true, + node: expect.objectContaining({ label: 'Minimize' }), + }); +}); + +test('resolveMaestroNodeFromSnapshot blocks taps on app content behind React Native overlays', () => { + const snapshot = makeReactNativeOverlaySnapshot(); + + const appContent = resolveMaestroNodeFromSnapshot( + snapshot, + 'label="Article title" || text="Article title" || id="Article title"', + {}, + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + const overlayControl = resolveMaestroNodeFromSnapshot( + snapshot, + 'label="Dismiss" || text="Dismiss" || id="Dismiss"', + {}, + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + + expect(appContent).toMatchObject({ + ok: false, + message: expect.stringContaining('React Native overlay is covering app content'), + }); + expect(overlayControl).toMatchObject({ + ok: true, + node: expect.objectContaining({ label: 'Dismiss' }), + }); +}); + +test('resolveMaestroNodeFromSnapshot prefers foreground duplicate matches', () => { + const snapshot: SnapshotState = { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'button', + label: 'Show Dialog', + rect: { x: 24, y: 220, width: 240, height: 72 }, + depth: 8, + }, + { + index: 2, + ref: 'e2', + type: 'button', + label: 'Show Dialog', + rect: { x: 24, y: 220, width: 240, height: 72 }, + depth: 8, + }, + ], + }; + + const target = resolveMaestroNodeFromSnapshot( + snapshot, + 'label="Show Dialog" || text="Show Dialog" || id="Show Dialog"', + {}, + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + + expect(target).toMatchObject({ + ok: true, + node: expect.objectContaining({ index: 2 }), + }); +}); + +test('resolveMaestroNodeFromSnapshot preserves read order for duplicate matches in different rects', () => { + const snapshot: SnapshotState = { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'button', + label: 'Open details', + rect: { x: 24, y: 520, width: 240, height: 72 }, + depth: 8, + }, + { + index: 2, + ref: 'e2', + type: 'button', + label: 'Open details', + rect: { x: 24, y: 320, width: 240, height: 72 }, + depth: 8, + }, + ], + }; + + const target = resolveMaestroNodeFromSnapshot( + snapshot, + 'label="Open details" || text="Open details" || id="Open details"', + {}, + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + + expect(target).toMatchObject({ + ok: true, + node: expect.objectContaining({ index: 1 }), + }); +}); + +test('resolveVisibleMaestroNodeFromSnapshot requires visible text matches to be on screen', () => { + const snapshot: SnapshotState = { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'android.widget.TextView', + label: 'Library', + rect: { x: 0, y: 2340, width: 120, height: 48 }, + depth: 8, + }, + ], + }; + + const target = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + 'label="Library" || text="Library" || id="Library"', + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + + expect(target).toMatchObject({ + ok: false, + message: expect.stringContaining('none were visible'), + }); +}); + +function makeReactNativeOverlaySnapshot(): SnapshotState { + return { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'android.widget.TextView', + label: 'Article title', + rect: { x: 24, y: 420, width: 320, height: 54 }, + depth: 8, + }, + { + index: 2, + ref: 'e2', + type: 'android.widget.TextView', + label: 'AppStack.tsx (42:7)', + rect: { x: 28, y: 1304, width: 1025, height: 44 }, + depth: 8, + }, + { + index: 3, + ref: 'e3', + type: 'android.view.ViewGroup', + label: 'Dismiss', + rect: { x: 0, y: 2142, width: 540, height: 132 }, + depth: 6, + }, + { + index: 4, + ref: 'e4', + type: 'android.view.ViewGroup', + label: 'Minimize', + rect: { x: 540, y: 2142, width: 540, height: 132 }, + depth: 6, + }, + ], + }; +} diff --git a/src/compat/maestro/command-mapper.ts b/src/compat/maestro/command-mapper.ts index 30e36d60d..36f294b40 100644 --- a/src/compat/maestro/command-mapper.ts +++ b/src/compat/maestro/command-mapper.ts @@ -16,13 +16,15 @@ import { } from './interactions.ts'; import { action, + assertOnlyKeys, + isPlainRecord, readTimeoutMs, requireAppId, requireStringValue, resolveMaestroString, unsupportedCommand, } from './support.ts'; -import { convertRepeat, convertRunFlow } from './flow-control.ts'; +import { convertRepeat, convertRetry, convertRunFlow } from './flow-control.ts'; import { convertRunScript } from './run-script.ts'; import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import type { @@ -54,7 +56,10 @@ const MAP_COMMAND_HANDLERS: Record = { ], openLink: ({ value, config, context, name }) => [convertOpenLink(value, config, context, name)], assertVisible: ({ value, context, name }) => [ - action('wait', [maestroSelector(value, name, [], context), '5000']), + action(MAESTRO_RUNTIME_COMMAND.assertVisible, [ + maestroSelector(value, name, [], context), + '5000', + ]), ], assertNotVisible: ({ value, context, name }) => [ action(MAESTRO_RUNTIME_COMMAND.assertNotVisible, [maestroSelector(value, name, [], context)]), @@ -78,6 +83,8 @@ const MAP_COMMAND_HANDLERS: Record = { convertRunFlow(value, config, context, deps, convertCommandList), repeat: ({ value, config, context, deps }) => convertRepeat(value, config, context, deps, convertCommandList), + retry: ({ value, config, context, deps }) => + convertRetry(value, config, context, deps, convertCommandList), }; const SCALAR_COMMAND_HANDLERS: Record< @@ -145,13 +152,21 @@ function convertOpenLink( context: MaestroParseContext, name: string, ): SessionAction { - const url = resolveMaestroString(requireStringValue(name, value), context); + const rawLink = readOpenLink(value, name); + const url = resolveMaestroString(rawLink, context); if (context.platform === 'ios' && config.appId) { return action('open', [resolveMaestroString(requireAppId(config, name), context), url]); } return action('open', [url]); } +function readOpenLink(value: unknown, name: string): string { + if (typeof value === 'string') return value; + if (!isPlainRecord(value)) return requireStringValue(name, value); + assertOnlyKeys(value, name, ['link']); + return requireStringValue(`${name}.link`, value.link); +} + function convertCommandList( commands: MaestroCommand[], config: MaestroFlowConfig, diff --git a/src/compat/maestro/flow-control.ts b/src/compat/maestro/flow-control.ts index 513d0a1cb..0c526cb93 100644 --- a/src/compat/maestro/flow-control.ts +++ b/src/compat/maestro/flow-control.ts @@ -8,7 +8,6 @@ import { assertOnlyKeys, isPlainRecord, normalizeCommandList, - normalizePlatformValue, readEnvMap, resolveMaestroString, unsupportedMaestroSyntax, @@ -24,6 +23,7 @@ import type { // a guardrail until repeat can execute as a runtime loop without materializing // every child action. const MAX_REPEAT_EXPANSIONS = 1000; +type MaestroConditionPlatform = 'android' | 'ios' | 'web'; type ConvertCommandList = ( commands: MaestroCommand[], @@ -89,6 +89,30 @@ export function convertRepeat( ); } +export function convertRetry( + value: unknown, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'retry expects a map.'); + } + assertOnlyKeys(value, 'retry', ['maxRetries', 'commands']); + if (!Array.isArray(value.commands)) { + throw new AppError('INVALID_ARGS', 'retry requires a commands list.'); + } + const maxRetries = readRetryMaxRetries(value.maxRetries, context); + const commands = normalizeCommandList(value.commands); + const actions = convertCommandList(commands, config, context, deps); + return [ + action(MAESTRO_RUNTIME_COMMAND.retry, [String(maxRetries)], { + batchSteps: actions.map(sessionActionToBatchStep), + }), + ]; +} + function readRunFlowActions( value: Record, config: MaestroFlowConfig, @@ -117,19 +141,34 @@ function readRunFlowCondition(value: unknown, context: MaestroParseContext): Run throw new AppError('INVALID_ARGS', 'runFlow.when expects a map.'); } assertOnlyKeys(value, 'runFlow.when', ['platform', 'visible', 'notVisible', 'true']); - rejectUnsupportedCondition(value, 'true', 'when.true'); - if (value.platform !== undefined) { - const platform = normalizePlatformValue(value.platform, 'runFlow.when.platform'); - if (!context.platform) { - throw new AppError( - 'INVALID_ARGS', - 'Maestro runFlow.when.platform requires replay to be run with --platform ios|android.', - ); - } - if (platform !== context.platform) return { shouldRun: false }; - } + if (!matchesRunFlowStaticCondition(value, context)) return { shouldRun: false }; return { shouldRun: true, + ...readRunFlowVisibilityCondition(value, context), + }; +} + +function matchesRunFlowStaticCondition( + value: Record, + context: MaestroParseContext, +): boolean { + if (value.true !== undefined && !evaluateRunFlowTrueCondition(value.true, context)) return false; + if (value.platform === undefined) return true; + const platform = normalizeRunFlowPlatform(value.platform, 'runFlow.when.platform'); + if (!context.platform) { + throw new AppError( + 'INVALID_ARGS', + 'Maestro runFlow.when.platform requires replay to be run with --platform ios|android.', + ); + } + return platform === context.platform; +} + +function readRunFlowVisibilityCondition( + value: Record, + context: MaestroParseContext, +): Pick { + return { ...(value.visible !== undefined ? { visibleSelector: maestroSelector(value.visible, 'runFlow.when.visible', [], context) } : {}), @@ -146,6 +185,229 @@ function readRunFlowCondition(value: unknown, context: MaestroParseContext): Run }; } +function normalizeRunFlowPlatform(value: unknown, name: string): MaestroConditionPlatform { + if (typeof value !== 'string') { + throw new AppError('INVALID_ARGS', `${name} expects Android, iOS, or Web.`); + } + const normalized = value.trim().toLowerCase(); + if (normalized === 'android' || normalized === 'ios' || normalized === 'web') { + return normalized; + } + throw new AppError('INVALID_ARGS', `${name} expects Android, iOS, or Web.`); +} + +function evaluateRunFlowTrueCondition(value: unknown, context: MaestroParseContext): boolean { + if (typeof value === 'boolean') return value; + if (typeof value !== 'string') { + throw new AppError('INVALID_ARGS', 'runFlow.when.true expects a boolean or expression string.'); + } + const expression = unwrapMaestroExpression(resolveMaestroString(value, context)); + const parser = new MaestroBooleanExpressionParser(tokenizeMaestroBooleanExpression(expression), { + platform: context.platform, + }); + return parser.parse(); +} + +function unwrapMaestroExpression(value: string): string { + const trimmed = value.trim(); + return trimmed.startsWith('${') && trimmed.endsWith('}') ? trimmed.slice(2, -1).trim() : trimmed; +} + +type MaestroBooleanToken = + | { type: 'platform' } + | MaestroBooleanOperatorToken + | { type: 'paren'; value: '(' | ')' } + | { type: 'string'; value: string } + | { type: 'boolean'; value: boolean }; + +type MaestroBooleanOperatorToken = { type: 'operator'; value: '==' | '!=' | '&&' | '||' }; + +type MaestroBooleanTokenMatch = { + token: MaestroBooleanToken; + length: number; +}; + +function tokenizeMaestroBooleanExpression(expression: string): MaestroBooleanToken[] { + const tokens: MaestroBooleanToken[] = []; + let index = 0; + while (index < expression.length) { + const remaining = expression.slice(index); + const skipped = whitespaceLength(remaining); + if (skipped > 0) { + index += skipped; + continue; + } + const next = readMaestroBooleanToken(remaining); + if (next) { + tokens.push(next.token); + index += next.length; + continue; + } + throw new AppError( + 'INVALID_ARGS', + `Unsupported runFlow.when.true expression near "${remaining.slice(0, 24)}".`, + ); + } + return tokens; +} + +function whitespaceLength(value: string): number { + return /^\s+/.exec(value)?.[0].length ?? 0; +} + +function readMaestroBooleanToken(remaining: string): MaestroBooleanTokenMatch | null { + return ( + readPlatformToken(remaining) ?? + readOperatorToken(remaining) ?? + readParenToken(remaining) ?? + readStringToken(remaining) ?? + readBooleanToken(remaining) + ); +} + +function readPlatformToken(remaining: string): MaestroBooleanTokenMatch | null { + const name = 'maestro.platform'; + return remaining.startsWith(name) ? { token: { type: 'platform' }, length: name.length } : null; +} + +function readOperatorToken(remaining: string): MaestroBooleanTokenMatch | null { + const operator = /^(==|!=|&&|\|\|)/.exec(remaining)?.[1]; + return operator + ? { + token: { type: 'operator', value: operator as MaestroBooleanOperatorToken['value'] }, + length: operator.length, + } + : null; +} + +function readParenToken(remaining: string): MaestroBooleanTokenMatch | null { + const value = remaining[0]; + return value === '(' || value === ')' ? { token: { type: 'paren', value }, length: 1 } : null; +} + +function readStringToken(remaining: string): MaestroBooleanTokenMatch | null { + const quoted = /^(['"])(.*?)\1/.exec(remaining); + return quoted + ? { token: { type: 'string', value: quoted[2] ?? '' }, length: quoted[0].length } + : null; +} + +function readBooleanToken(remaining: string): MaestroBooleanTokenMatch | null { + const value = /^(true|false)\b/.exec(remaining)?.[1]; + return value + ? { token: { type: 'boolean', value: value === 'true' }, length: value.length } + : null; +} + +class MaestroBooleanExpressionParser { + private index = 0; + private readonly tokens: MaestroBooleanToken[]; + private readonly context: { platform?: 'android' | 'ios' }; + + constructor(tokens: MaestroBooleanToken[], context: { platform?: 'android' | 'ios' }) { + this.tokens = tokens; + this.context = context; + } + + parse(): boolean { + const result = this.parseOr(); + if (this.peek()) { + throw new AppError('INVALID_ARGS', 'Unsupported trailing runFlow.when.true expression.'); + } + return result; + } + + private parseOr(): boolean { + let result = this.parseAnd(); + while (this.consumeOperator('||')) { + result = this.parseAnd() || result; + } + return result; + } + + private parseAnd(): boolean { + let result = this.parsePrimary(); + while (this.consumeOperator('&&')) { + result = this.parsePrimary() && result; + } + return result; + } + + private parsePrimary(): boolean { + const token = this.peek(); + if (!token) { + throw new AppError('INVALID_ARGS', 'Incomplete runFlow.when.true expression.'); + } + if (token.type === 'boolean') { + this.index += 1; + return token.value; + } + if (token.type === 'paren' && token.value === '(') { + this.index += 1; + const result = this.parseOr(); + if (!this.consumeParen(')')) { + throw new AppError('INVALID_ARGS', 'Unclosed runFlow.when.true parenthesis.'); + } + return result; + } + return this.parsePlatformComparison(); + } + + private parsePlatformComparison(): boolean { + this.expectPlatform(); + const operator = this.expectEqualityOperator(); + const value = this.expectString().toLowerCase(); + const platform = this.context.platform; + return operator === '==' ? platform === value : platform !== value; + } + + private expectPlatform(): void { + if (this.peek()?.type !== 'platform') { + throw new AppError( + 'INVALID_ARGS', + 'runFlow.when.true supports maestro.platform comparisons.', + ); + } + this.index += 1; + } + + private expectEqualityOperator(): '==' | '!=' { + const token = this.peek(); + if (token?.type === 'operator' && (token.value === '==' || token.value === '!=')) { + this.index += 1; + return token.value; + } + throw new AppError('INVALID_ARGS', 'runFlow.when.true comparison requires == or !=.'); + } + + private expectString(): string { + const token = this.peek(); + if (token?.type === 'string') { + this.index += 1; + return token.value; + } + throw new AppError('INVALID_ARGS', 'runFlow.when.true comparison requires a string literal.'); + } + + private consumeOperator(value: '&&' | '||'): boolean { + const token = this.peek(); + if (token?.type !== 'operator' || token.value !== value) return false; + this.index += 1; + return true; + } + + private consumeParen(value: '(' | ')'): boolean { + const token = this.peek(); + if (token?.type !== 'paren' || token.value !== value) return false; + this.index += 1; + return true; + } + + private peek(): MaestroBooleanToken | undefined { + return this.tokens[this.index]; + } +} + function wrapRunFlowCondition( actions: SessionAction[], condition: RunFlowCondition, @@ -179,6 +441,19 @@ function sessionActionToBatchStep( } function readRepeatTimes(value: unknown, context: MaestroParseContext): number { + return readMaestroNonNegativeInteger(value, context, 'repeat.times'); +} + +function readRetryMaxRetries(value: unknown, context: MaestroParseContext): number { + if (value === undefined) return 1; + return readMaestroNonNegativeInteger(value, context, 'retry.maxRetries'); +} + +function readMaestroNonNegativeInteger( + value: unknown, + context: MaestroParseContext, + name: string, +): number { const resolved = typeof value === 'string' ? resolveMaestroString(value, context) : value; const numeric = typeof resolved === 'number' @@ -189,18 +464,8 @@ function readRepeatTimes(value: unknown, context: MaestroParseContext): number { if (numeric === undefined || !Number.isInteger(numeric) || numeric < 0) { throw new AppError( 'INVALID_ARGS', - 'repeat.times must be a non-negative integer or ${VAR} resolving to one.', + `${name} must be a non-negative integer or \${VAR} resolving to one.`, ); } return numeric; } - -function rejectUnsupportedCondition( - value: Record, - key: string, - label: string, -): void { - if (value[key] !== undefined) { - throw unsupportedMaestroSyntax(`Maestro ${label} is not supported yet.`); - } -} diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index 897ec824c..6448d2c26 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -9,11 +9,7 @@ import { resolveMaestroString, unsupportedMaestroSyntax, } from './support.ts'; -import { - parseAbsolutePoint, - parseMaestroPoint, - readScrollPositionalsFromPercentSwipe, -} from './points.ts'; +import { parseAbsolutePoint, parseMaestroPoint } from './points.ts'; import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import type { MaestroParseContext } from './types.ts'; @@ -148,7 +144,7 @@ export function convertExtendedWaitUntil( if (value.notVisible !== undefined) { return [action('wait', [timeoutMs]), action('is', ['hidden', selector])]; } - return [action('wait', [selector, timeoutMs])]; + return [action(MAESTRO_RUNTIME_COMMAND.assertVisible, [selector, timeoutMs])]; } export function convertScroll(value: unknown): SessionAction { @@ -177,7 +173,9 @@ export function convertScrollUntilVisible( assertOnlyKeys(value, 'scrollUntilVisible', ['element', 'direction', 'timeout']); const selector = maestroSelector(value.element, 'scrollUntilVisible.element', [], context); const direction = - typeof value.direction === 'string' ? readScrollUntilVisibleDirection(value.direction) : 'down'; + typeof value.direction === 'string' + ? readMaestroDirection(value.direction, 'scrollUntilVisible.direction') + : 'down'; const timeoutMs = String(readTimeoutMs(value, 5000)); return [action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [selector, timeoutMs, direction])]; } @@ -192,7 +190,11 @@ export function convertSwipe(value: unknown, context: MaestroParseContext): Sess return convertTargetedSwipe(value, from, context); } if (typeof value.direction === 'string') { - return action('scroll', readScrollPositionalsFromDirectionSwipe(value.direction)); + return action(MAESTRO_RUNTIME_COMMAND.swipeScreen, [ + 'direction', + readSwipeDirection(value.direction), + ...swipeDurationPositionals(value), + ]); } return convertCoordinateSwipe(value); } @@ -249,27 +251,21 @@ function convertCoordinateSwipePoints( ]); } if (start.kind === 'percent' && end.kind === 'percent') { - return action('scroll', readScrollPositionalsFromPercentSwipe(start, end)); + return action(MAESTRO_RUNTIME_COMMAND.swipeScreen, [ + 'percent', + String(start.x), + String(start.y), + String(end.x), + String(end.y), + ...(durationMs ? [durationMs] : []), + ]); } throw unsupportedMaestroSyntax( 'Maestro swipe start/end must both be absolute pixels or both be percentages.', ); } -function readScrollPositionalsFromDirectionSwipe(direction: string): string[] { - switch (readSwipeDirection(direction)) { - case 'up': - return ['down']; - case 'down': - return ['up']; - case 'left': - return ['right']; - case 'right': - return ['left']; - } -} - -function readSwipeDirection(direction: string): SwipeDirection { +function readMaestroDirection(direction: string, name: string): SwipeDirection { const normalized = direction.toLowerCase(); switch (normalized) { case 'up': @@ -278,22 +274,12 @@ function readSwipeDirection(direction: string): SwipeDirection { case 'right': return normalized; default: - throw unsupportedMaestroSyntax('Maestro swipe direction must be UP, DOWN, LEFT, or RIGHT.'); + throw unsupportedMaestroSyntax(`Maestro ${name} must be UP, DOWN, LEFT, or RIGHT.`); } } -function readScrollUntilVisibleDirection(direction: string): string { - switch (direction.toLowerCase()) { - case 'up': - case 'down': - case 'left': - case 'right': - return direction.toLowerCase(); - default: - throw unsupportedMaestroSyntax( - 'Maestro scrollUntilVisible.direction must be UP, DOWN, LEFT, or RIGHT.', - ); - } +function readSwipeDirection(direction: string): SwipeDirection { + return readMaestroDirection(direction, 'swipe direction'); } export function convertPressKey(value: unknown): SessionAction { @@ -317,16 +303,19 @@ export function maestroSelector( assertOnlyKeys(value, command, ['id', 'text', 'enabled', 'selected', ...allowedExtraKeys]); const terms: string[] = []; - if (typeof value.id === 'string') - terms.push(selectorTerm('id', resolveMaestroString(value.id, context))); - if (typeof value.text === 'string') - terms.push(selectorTerm('label', resolveMaestroString(value.text, context))); - if (typeof value.label === 'string' && terms.length === 0) - terms.push(selectorTerm('label', resolveMaestroString(value.label, context))); + const stateTerms: string[] = []; if (typeof value.enabled === 'boolean') - terms.push(selectorTerm('enabled', String(value.enabled))); + stateTerms.push(selectorTerm('enabled', String(value.enabled))); if (typeof value.selected === 'boolean') - terms.push(selectorTerm('selected', String(value.selected))); + stateTerms.push(selectorTerm('selected', String(value.selected))); + if (typeof value.id === 'string') + terms.push(selectorTerm('id', resolveMaestroString(value.id, context)), ...stateTerms); + if (typeof value.text === 'string' && terms.length === 0) { + return visibleTextSelector(resolveMaestroString(value.text, context), stateTerms); + } + if (typeof value.label === 'string' && terms.length === 0) + terms.push(selectorTerm('label', resolveMaestroString(value.label, context)), ...stateTerms); + if (terms.length === 0 && stateTerms.length > 0) terms.push(...stateTerms); if (terms.length === 0) { throw new AppError( 'INVALID_ARGS', @@ -336,11 +325,11 @@ export function maestroSelector( return terms.join(' '); } -function visibleTextSelector(value: string): string { +function visibleTextSelector(value: string, extraTerms: readonly string[] = []): string { return [ - selectorTerm('label', value), - selectorTerm('text', value), - selectorTerm('id', value), + [selectorTerm('label', value), ...extraTerms].join(' '), + [selectorTerm('text', value), ...extraTerms].join(' '), + [selectorTerm('id', value), ...extraTerms].join(' '), ].join(' || '); } @@ -360,9 +349,8 @@ function maestroTapOnRuntimeOptions(value: unknown, context: MaestroParseContext } function swipeDurationPositionals(value: Record): string[] { - return typeof value.duration === 'number' && Number.isFinite(value.duration) - ? [String(Math.max(16, Math.floor(value.duration)))] - : []; + const durationMs = readSwipeDurationMs(value.duration); + return durationMs ? [durationMs] : []; } function selectorTerm(key: string, value: string): string { diff --git a/src/compat/maestro/points.ts b/src/compat/maestro/points.ts index c0e25ffb4..571e692a5 100644 --- a/src/compat/maestro/points.ts +++ b/src/compat/maestro/points.ts @@ -1,4 +1,3 @@ -import { AppError } from '../../utils/errors.ts'; import { unsupportedMaestroSyntax } from './support.ts'; export type MaestroPoint = @@ -36,22 +35,3 @@ export function parseMaestroPoint(value: string): MaestroPoint { 'Only Maestro swipe coordinates like "100,200" or "50%,75%" are supported.', ); } - -export function readScrollPositionalsFromPercentSwipe( - start: Extract, - end: Extract, -): string[] { - const deltaX = end.x - start.x; - const deltaY = end.y - start.y; - if (Math.abs(deltaX) === 0 && Math.abs(deltaY) === 0) { - throw new AppError('INVALID_ARGS', 'swipe start and end cannot be the same point.'); - } - const vertical = Math.abs(deltaY) >= Math.abs(deltaX); - const direction = vertical ? (deltaY < 0 ? 'down' : 'up') : deltaX < 0 ? 'right' : 'left'; - const amount = Math.min(1, Math.max(0.01, Math.abs(vertical ? deltaY : deltaX) / 100)); - return [direction, formatAmount(amount)]; -} - -function formatAmount(value: number): string { - return value.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); -} diff --git a/src/compat/maestro/replay-flow.ts b/src/compat/maestro/replay-flow.ts index eab80c650..cc247d22c 100644 --- a/src/compat/maestro/replay-flow.ts +++ b/src/compat/maestro/replay-flow.ts @@ -110,6 +110,9 @@ function optimizeTypedAfterTap( const tapSelector = readPlainMaestroTapSelector(action); if (typedAfterTap === null || tapSelector === null) return null; const line = actionLines[index] ?? 1; + if (!isLikelyTextEntrySelector(tapSelector)) { + return { actions: [clearMaestroNonHittableTap(action)], actionLines: [line], consumed: 1 }; + } if (actions[index + 2]?.command !== MAESTRO_RUNTIME_COMMAND.pressEnter) { return { actions: [clearMaestroNonHittableTap(action)], actionLines: [line], consumed: 1 }; } @@ -162,6 +165,12 @@ function readPlainTypeText(action: SessionAction | undefined): string | null { return text; } +function isLikelyTextEntrySelector(selector: string): boolean { + return /\b(input|textfield|textarea|field|email|password|username|search|query)\b/i.test( + selector.replace(/([a-z])([A-Z])/g, '$1 $2'), + ); +} + function parseYamlDocuments(script: string): unknown[] { const documents = parseAllDocuments(script); for (const document of documents) { diff --git a/src/compat/maestro/run-script.ts b/src/compat/maestro/run-script.ts index 49f9f242e..2b6602690 100644 --- a/src/compat/maestro/run-script.ts +++ b/src/compat/maestro/run-script.ts @@ -62,7 +62,7 @@ export function executeRunScriptFile(params: { }): Record { const { scriptPath, env } = params; const script = fs.readFileSync(scriptPath, 'utf8'); - const output: Record = {}; + const output: Record = Object.create(null) as Record; try { // Compatibility note: node:vm is not a security sandbox. Maestro runScript @@ -128,7 +128,7 @@ function buildScriptGlobals( return { ...env, output, - json: (value: string) => JSON.parse(value) as unknown, + json: parseRunScriptJson, http: { post: (url: string, options?: { headers?: Record; body?: string }) => runHttpRequestSync('POST', url, options), @@ -136,6 +136,35 @@ function buildScriptGlobals( }; } +function parseRunScriptJson(value: unknown): unknown { + if (typeof value !== 'string') { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript json() expected a string body, received ${typeof value}.`, + ); + } + if (value.trim().length === 0) { + throw new AppError( + 'COMMAND_FAILED', + 'Maestro runScript json() received an empty body. Check the preceding http response status and setup server output.', + ); + } + try { + return JSON.parse(value, safeRunScriptJsonReviver) as unknown; + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript json() could not parse response body: ${error instanceof Error ? error.message : String(error)}`, + { bodyPreview: value.slice(0, 1000) }, + error instanceof Error ? error : undefined, + ); + } +} + +function safeRunScriptJsonReviver(key: string, value: unknown): unknown { + return key === '__proto__' || key === 'constructor' || key === 'prototype' ? undefined : value; +} + function runHttpRequestSync( method: string, url: string, diff --git a/src/compat/maestro/runtime-assertions.ts b/src/compat/maestro/runtime-assertions.ts new file mode 100644 index 000000000..788434662 --- /dev/null +++ b/src/compat/maestro/runtime-assertions.ts @@ -0,0 +1,212 @@ +import { getSnapshotReferenceFrame } from '../../daemon/touch-reference-frame.ts'; +import type { DaemonResponse } from '../../daemon/types.ts'; +import type { ReplayVarScope } from '../../replay/vars.ts'; +import type { SnapshotState } from '../../utils/snapshot.ts'; +import { sleep } from '../../utils/timeouts.ts'; +import { + captureMaestroRawSnapshot, + errorResponse, + rememberMaestroSnapshot, + readSnapshotState, + type MaestroRuntimeInvoke, + type ReplayBaseRequest, +} from './runtime-support.ts'; +import { + readMaestroSelectorPlatform, + resolveVisibleMaestroNodeFromSnapshot, +} from './runtime-targets.ts'; + +const MAESTRO_ASSERTION_POLICY = { + animationPollMs: 250, + assertVisibleGraceMs: 1000, + assertVisiblePollMs: 250, + assertNotVisiblePollMs: 250, + assertNotVisibleTimeoutMs: 3000, +} as const; + +export async function invokeMaestroAssertVisible(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; +}): Promise { + const [selector, timeoutValue = '5000'] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'assertVisible requires a selector.'); + } + const timeoutMs = Number(timeoutValue); + if (!Number.isFinite(timeoutMs) || timeoutMs < 0) { + return errorResponse('INVALID_ARGS', 'assertVisible timeout must be a non-negative number.'); + } + + const startedAt = Date.now(); + const deadlineMs = timeoutMs + MAESTRO_ASSERTION_POLICY.assertVisibleGraceMs; + let lastResponse: DaemonResponse | undefined; + do { + const response = await captureMaestroRawSnapshot(params); + lastResponse = response; + if (response.ok) { + const snapshot = readSnapshotState(response.data); + if (!snapshot) { + return errorResponse('COMMAND_FAILED', 'Unable to read snapshot data for assertVisible.'); + } + const target = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + selector, + readMaestroSelectorPlatform(params.baseReq.flags), + getSnapshotReferenceFrame(snapshot), + ); + if (target.ok) { + rememberMaestroSnapshot(params.scope, response.data, selector); + return { + ok: true, + data: { + selector, + matches: target.matches, + nodeIndex: target.node.index, + nodeType: target.node.type, + nodeLabel: target.node.label, + nodeIdentifier: target.node.identifier, + rect: target.rect, + waitedMs: Date.now() - startedAt, + }, + }; + } + lastResponse = errorResponse('COMMAND_FAILED', target.message, { selector }); + } + + if (Date.now() - startedAt >= deadlineMs) break; + await sleep(MAESTRO_ASSERTION_POLICY.assertVisiblePollMs); + } while (Date.now() - startedAt <= deadlineMs); + + return ( + lastResponse ?? + errorResponse('COMMAND_FAILED', `Expected visible but did not match: ${selector}`, { + selector, + timeoutMs, + }) + ); +} + +export async function invokeMaestroAssertNotVisible(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const [selector] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'assertNotVisible requires a selector.'); + } + const startedAt = Date.now(); + let hiddenSamples = 0; + let lastVisibleResponse: DaemonResponse | undefined; + while (Date.now() - startedAt <= MAESTRO_ASSERTION_POLICY.assertNotVisibleTimeoutMs) { + const response = await params.invoke({ + ...params.baseReq, + command: 'is', + positionals: ['visible', selector], + flags: { ...params.baseReq.flags, noRecord: true }, + }); + if (response.ok) { + hiddenSamples = 0; + lastVisibleResponse = response; + } else if (isMaestroVisibilityMiss(response)) { + hiddenSamples += 1; + if (hiddenSamples >= 2) { + return { + ok: true, + data: { + pass: true, + selector, + stableSamples: hiddenSamples, + timeoutMs: MAESTRO_ASSERTION_POLICY.assertNotVisibleTimeoutMs, + }, + }; + } + } else { + return response; + } + await sleep(MAESTRO_ASSERTION_POLICY.assertNotVisiblePollMs); + } + return errorResponse('COMMAND_FAILED', `Expected not visible but matched: ${selector}`, { + selector, + timeoutMs: MAESTRO_ASSERTION_POLICY.assertNotVisibleTimeoutMs, + lastResponse: lastVisibleResponse, + }); +} + +export async function invokeMaestroWaitForAnimationToEnd(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const timeoutMs = Number(params.positionals[0] ?? 15000); + if (!Number.isFinite(timeoutMs) || timeoutMs < 0) { + return errorResponse('INVALID_ARGS', 'waitForAnimationToEnd timeout must be a number.'); + } + const startedAt = Date.now(); + let previousSignature: string | undefined; + let lastResponse: DaemonResponse | undefined; + + while (Date.now() - startedAt < timeoutMs) { + const response = await captureMaestroRawSnapshot(params); + const poll = readAnimationPollResult(response, previousSignature, timeoutMs); + if (poll.done) return poll.response; + previousSignature = poll.signature ?? previousSignature; + lastResponse = response; + await sleep(MAESTRO_ASSERTION_POLICY.animationPollMs); + } + + return lastResponse?.ok === false + ? lastResponse + : { ok: true, data: { stable: false, timeoutMs } }; +} + +function isMaestroVisibilityMiss(response: Extract): boolean { + const details = response.error.details; + return ( + details?.command === 'is' && + (details.reason === 'selector_not_found' || details.reason === 'predicate_failed') + ); +} + +function readAnimationPollResult( + response: DaemonResponse, + previousSignature: string | undefined, + timeoutMs: number, +): { done: true; response: DaemonResponse } | { done: false; signature?: string } { + const signature = readSnapshotStabilitySignature(response); + if (!response.ok) return { done: false }; + if (!signature) return { done: true, response }; + if (previousSignature === signature) { + return { done: true, response: { ok: true, data: { stable: true, timeoutMs } } }; + } + return { done: false, signature }; +} + +function readSnapshotStabilitySignature(response: DaemonResponse): string | null { + if (!response.ok) return null; + const snapshot = readSnapshotState(response.data); + return snapshot ? snapshotStabilitySignature(snapshot) : null; +} + +function snapshotStabilitySignature(snapshot: SnapshotState): string { + return JSON.stringify( + snapshot.nodes.map((node) => ({ + index: node.index, + parentIndex: node.parentIndex, + type: node.type, + identifier: node.identifier, + label: node.label, + value: node.value, + rect: node.rect + ? { + x: Math.round(node.rect.x), + y: Math.round(node.rect.y), + width: Math.round(node.rect.width), + height: Math.round(node.rect.height), + } + : undefined, + })), + ); +} diff --git a/src/compat/maestro/runtime-commands.ts b/src/compat/maestro/runtime-commands.ts index 8e354f362..d15292434 100644 --- a/src/compat/maestro/runtime-commands.ts +++ b/src/compat/maestro/runtime-commands.ts @@ -1,10 +1,13 @@ export const MAESTRO_RUNTIME_COMMAND = { runFlowWhen: '__maestroRunFlowWhen', + retry: '__maestroRetry', runScript: '__maestroRunScript', + assertVisible: '__maestroAssertVisible', assertNotVisible: '__maestroAssertNotVisible', pressEnter: '__maestroPressEnter', waitForAnimationToEnd: '__maestroWaitForAnimationToEnd', scrollUntilVisible: '__maestroScrollUntilVisible', + swipeScreen: '__maestroSwipeScreen', swipeOn: '__maestroSwipeOn', tapOn: '__maestroTapOn', tapPointPercent: '__maestroTapPointPercent', diff --git a/src/compat/maestro/runtime-flow.ts b/src/compat/maestro/runtime-flow.ts new file mode 100644 index 000000000..4feb4f962 --- /dev/null +++ b/src/compat/maestro/runtime-flow.ts @@ -0,0 +1,157 @@ +import { type CommandFlags } from '../../core/dispatch.ts'; +import type { DaemonRequest, DaemonResponse, SessionAction } from '../../daemon/types.ts'; +import { getSnapshotReferenceFrame } from '../../daemon/touch-reference-frame.ts'; +import { + batchStepToSessionAction, + captureMaestroRawSnapshot, + errorResponse, + readSnapshotState, + type MaestroReplayInvoker, + type ReplayBaseRequest, +} from './runtime-support.ts'; +import { + readMaestroSelectorPlatform, + resolveVisibleMaestroNodeFromSnapshot, +} from './runtime-targets.ts'; + +type MaestroRunFlowWhenCondition = + | { ok: true; mode: string; predicate: string; selector: string } + | { ok: false; response: DaemonResponse }; + +export async function invokeMaestroRunFlowWhen(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: MaestroReplayInvoker; +}): Promise { + const condition = readMaestroRunFlowWhenCondition(params.positionals); + if (!condition.ok) return condition.response; + const conditionResult = await evaluateMaestroRunFlowWhenCondition(params, condition); + if (!conditionResult.ok) return conditionResult.response; + if (!conditionResult.matched) { + return { + ok: true, + data: { skipped: true, condition: condition.mode, selector: condition.selector }, + }; + } + return await invokeMaestroRunFlowWhenSteps(params, condition); +} + +export async function invokeMaestroRetry(params: { + positionals: string[]; + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invokeReplayAction: MaestroReplayInvoker; +}): Promise { + const [maxRetriesValue = '1'] = params.positionals; + const maxRetries = Number(maxRetriesValue); + if (!Number.isInteger(maxRetries) || maxRetries < 0) { + return errorResponse('INVALID_ARGS', 'retry.maxRetries must be a non-negative integer.'); + } + + const steps = (params.batchSteps ?? []).map(batchStepToSessionAction); + let lastResponse: DaemonResponse | undefined; + for (let attempt = 0; attempt <= maxRetries; attempt += 1) { + const response = await invokeMaestroRetryAttempt(params, steps, attempt); + if (response.ok) { + return { ok: true, data: { attempts: attempt + 1, retried: attempt > 0 } }; + } + lastResponse = response; + } + return lastResponse ?? errorResponse('COMMAND_FAILED', 'retry commands failed.'); +} + +function readMaestroRunFlowWhenCondition(positionals: string[]): MaestroRunFlowWhenCondition { + const [mode, selector] = positionals; + if ((mode !== 'visible' && mode !== 'notVisible') || !selector) { + return { + ok: false, + response: errorResponse( + 'INVALID_ARGS', + 'runFlow.when requires visible/notVisible and a selector.', + ), + }; + } + return { + ok: true, + mode, + predicate: mode === 'visible' ? 'visible' : 'hidden', + selector, + }; +} + +async function evaluateMaestroRunFlowWhenCondition( + params: { + baseReq: ReplayBaseRequest; + invoke: (req: DaemonRequest) => Promise; + }, + condition: Extract, +): Promise<{ ok: true; matched: boolean } | { ok: false; response: DaemonResponse }> { + const response = await captureMaestroRawSnapshot(params); + if (!response.ok) return { ok: false, response }; + const snapshot = readSnapshotState(response.data); + if (!snapshot) { + return { + ok: false, + response: errorResponse('COMMAND_FAILED', 'Unable to read snapshot data for runFlow.when.'), + }; + } + const visible = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + condition.selector, + readMaestroSelectorPlatform(params.baseReq.flags), + getSnapshotReferenceFrame(snapshot), + ).ok; + return { ok: true, matched: condition.mode === 'visible' ? visible : !visible }; +} + +async function invokeMaestroRunFlowWhenSteps( + params: { + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invokeReplayAction: MaestroReplayInvoker; + }, + condition: Extract, +): Promise { + const steps = (params.batchSteps ?? []).map(batchStepToSessionAction); + for (const [index, action] of steps.entries()) { + // Preserve stable parent-step ordering for nested runtime commands while + // keeping the substep distinguishable in traces. + const response = await params.invokeReplayAction({ + action, + line: params.line, + step: params.step + index / 1000, + }); + if (!response.ok) return response; + } + + return { + ok: true, + data: { ran: steps.length, condition: condition.mode, selector: condition.selector }, + }; +} + +async function invokeMaestroRetryAttempt( + params: { + line: number; + step: number; + invokeReplayAction: MaestroReplayInvoker; + }, + steps: SessionAction[], + attempt: number, +): Promise { + for (const [index, action] of steps.entries()) { + const response = await params.invokeReplayAction({ + action, + line: params.line, + step: params.step + attempt + index / 1000, + }); + if (!response.ok) return response; + } + return { ok: true, data: { ran: steps.length } }; +} diff --git a/src/compat/maestro/runtime-geometry.ts b/src/compat/maestro/runtime-geometry.ts new file mode 100644 index 000000000..50396b0a5 --- /dev/null +++ b/src/compat/maestro/runtime-geometry.ts @@ -0,0 +1,129 @@ +import type { Rect, SnapshotNode } from '../../utils/snapshot.ts'; +import type { MaestroSnapshotTarget } from './runtime-targets.ts'; + +const MAESTRO_GEOMETRY_POLICY = { + swipe: { + screenRatio: 0.35, + minDistancePx: 120, + maxDistancePx: 360, + marginPx: 8, + }, + largeTextContainerBias: { + minWidth: 120, + minHeight: 70, + width: 168, + height: 48, + }, +} as const; + +export function swipeCoordinatesFromTarget( + target: MaestroSnapshotTarget, + direction: string, +): + | { ok: true; start: { x: number; y: number }; end: { x: number; y: number } } + | { ok: false; message: string } { + const center = pointInsideRect(target.rect); + const frame = target.frame; + const horizontalDistance = swipeDistance(frame?.referenceWidth, target.rect.width); + const verticalDistance = swipeDistance(frame?.referenceHeight, target.rect.height); + const margin = MAESTRO_GEOMETRY_POLICY.swipe.marginPx; + const minX = margin; + const minY = margin; + const maxX = frame ? frame.referenceWidth - margin : center.x + horizontalDistance; + const maxY = frame ? frame.referenceHeight - margin : center.y + verticalDistance; + switch (direction.toLowerCase()) { + case 'up': + return { + ok: true, + start: center, + end: { x: center.x, y: clampCoordinate(center.y - verticalDistance, minY, maxY) }, + }; + case 'down': + return { + ok: true, + start: center, + end: { x: center.x, y: clampCoordinate(center.y + verticalDistance, minY, maxY) }, + }; + case 'left': + return { + ok: true, + start: center, + end: { x: clampCoordinate(center.x - horizontalDistance, minX, maxX), y: center.y }, + }; + case 'right': + return { + ok: true, + start: center, + end: { x: clampCoordinate(center.x + horizontalDistance, minX, maxX), y: center.y }, + }; + default: + return { ok: false, message: 'swipe.label direction must be up, down, left, or right.' }; + } +} + +export function pointForMaestroTapOnTarget( + target: MaestroSnapshotTarget, + isVisibleTextSelector: boolean, +): { x: number; y: number } { + if (!shouldBiasMaestroVisibleTextTap(target.node, isVisibleTextSelector, target.rect)) { + return pointInsideRect(target.rect); + } + return { + x: interiorCoordinate( + target.rect.x, + Math.min(target.rect.width, MAESTRO_GEOMETRY_POLICY.largeTextContainerBias.width), + ), + y: interiorCoordinate( + target.rect.y, + Math.min(target.rect.height, MAESTRO_GEOMETRY_POLICY.largeTextContainerBias.height), + ), + }; +} + +function swipeDistance(frameSize: number | undefined, rectSize: number): number { + const screenRelative = + typeof frameSize === 'number' ? frameSize * MAESTRO_GEOMETRY_POLICY.swipe.screenRatio : 0; + return Math.round( + Math.min( + MAESTRO_GEOMETRY_POLICY.swipe.maxDistancePx, + Math.max(MAESTRO_GEOMETRY_POLICY.swipe.minDistancePx, screenRelative, rectSize * 1.5), + ), + ); +} + +function clampCoordinate(value: number, min: number, max: number): number { + return Math.round(Math.min(max, Math.max(min, value))); +} + +function pointInsideRect(rect: Rect): { x: number; y: number } { + return { + x: interiorCoordinate(rect.x, rect.width), + y: interiorCoordinate(rect.y, rect.height), + }; +} + +function shouldBiasMaestroVisibleTextTap( + node: SnapshotNode, + isVisibleTextSelector: boolean, + rect: Rect, +): boolean { + if (!isVisibleTextSelector) return false; + if ( + rect.height < MAESTRO_GEOMETRY_POLICY.largeTextContainerBias.minHeight || + rect.width < MAESTRO_GEOMETRY_POLICY.largeTextContainerBias.minWidth + ) { + return false; + } + const type = node.type?.toLowerCase(); + return type === 'cell' || type === 'other' || type === 'scrollview'; +} + +function interiorCoordinate(origin: number, size: number): number { + // Maestro flows often expose hidden E2E controls as 1x1 views at the screen + // edge. Preserve zero-origin taps for those controls instead of nudging them + // outside their tiny rect by applying normal center/bounds clamping. + if (size <= 1) return Math.floor(origin); + const min = Math.ceil(origin); + const max = Math.floor(origin + size - 1); + return clampCoordinate(origin + size / 2, min, max); +} diff --git a/src/compat/maestro/runtime-interactions.ts b/src/compat/maestro/runtime-interactions.ts new file mode 100644 index 000000000..8300b7d32 --- /dev/null +++ b/src/compat/maestro/runtime-interactions.ts @@ -0,0 +1,598 @@ +import { getSnapshotReferenceFrame } from '../../daemon/touch-reference-frame.ts'; +import type { DaemonRequest, DaemonResponse } from '../../daemon/types.ts'; +import type { ReplayVarScope } from '../../replay/vars.ts'; +import { sleep } from '../../utils/timeouts.ts'; +import { pointForMaestroTapOnTarget, swipeCoordinatesFromTarget } from './runtime-geometry.ts'; +import { + captureMaestroRawSnapshot, + consumeMaestroSnapshot, + errorResponse, + readCachedMaestroReferenceFrame, + readSnapshotState, + type FailedDaemonResponse, + type MaestroRuntimeInvoke, + type ReplayBaseRequest, +} from './runtime-support.ts'; +import { + extractMaestroVisibleTextQuery, + readMaestroSelectorPlatform, + resolveMaestroFuzzyTextNodeFromSnapshot, + resolveMaestroNodeFromSnapshot, + type MaestroSnapshotTarget, + type MaestroTapOnOptions, +} from './runtime-targets.ts'; + +const MAESTRO_INTERACTION_POLICY = { + scrollUntilVisibleProbeMs: 500, + tapOnRetryMs: 250, + tapOnTimeoutMs: 30000, + optionalTapOnTimeoutMs: 3000, +} as const; + +type MaestroScrollUntilVisibleParams = { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}; + +type MaestroTapOnParams = { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; +}; + +export async function invokeMaestroScrollUntilVisible( + params: MaestroScrollUntilVisibleParams, +): Promise { + const [selector, timeoutValue = '5000', direction = 'down'] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'scrollUntilVisible requires a selector.'); + } + const timeoutMs = Number(timeoutValue); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + return errorResponse('INVALID_ARGS', 'scrollUntilVisible timeout must be a positive number.'); + } + const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + const attempts = Math.max( + 1, + Math.ceil(timeoutMs / MAESTRO_INTERACTION_POLICY.scrollUntilVisibleProbeMs), + ); + let lastWaitResponse: FailedDaemonResponse | null = null; + + for (let index = 0; index < attempts; index += 1) { + const probeResponse = await probeMaestroScrollVisibility( + params, + selector, + fuzzyTextQuery, + scrollProbeMs(timeoutMs, index), + ); + if (probeResponse.ok) return probeResponse; + lastWaitResponse = probeResponse; + + if (index === attempts - 1) break; + + const scrollResponse = await params.invoke({ + ...params.baseReq, + command: 'scroll', + positionals: [direction], + }); + if (!scrollResponse.ok) return scrollResponse; + } + + return withMaestroScrollTimeoutContext(lastWaitResponse, selector, timeoutMs); +} + +export async function invokeMaestroTapPointPercent(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const [xValue, yValue] = params.positionals; + const xPercent = Number(xValue); + const yPercent = Number(yValue); + if (!Number.isFinite(xPercent) || !Number.isFinite(yPercent)) { + return errorResponse('INVALID_ARGS', 'tapOn percentage point requires numeric x/y values.'); + } + + const snapshotResponse = await captureMaestroRawSnapshot(params); + if (!snapshotResponse.ok) return snapshotResponse; + + const snapshot = readSnapshotState(snapshotResponse.data); + if (!snapshot) { + return errorResponse( + 'COMMAND_FAILED', + 'Unable to read snapshot data for Maestro percentage point tap.', + ); + } + + const frame = getSnapshotReferenceFrame(snapshot); + if (!frame) { + return errorResponse( + 'COMMAND_FAILED', + 'Unable to resolve screen size for Maestro percentage point tap.', + ); + } + + return await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [ + String(Math.round((frame.referenceWidth * xPercent) / 100)), + String(Math.round((frame.referenceHeight * yPercent) / 100)), + ], + }); +} + +export async function invokeMaestroSwipeScreen(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; +}): Promise { + const swipe = await resolveMaestroScreenSwipe(params); + if (!swipe.ok) return swipe.response; + + return await invokeSwipeGesture(params, swipe, swipe.durationMs); +} + +export async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { + const [selector, rawOptions] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'tapOn requires a selector.'); + } + const options = readMaestroTapOnOptions(rawOptions); + if (!options.ok) return options.response; + const startedAt = Date.now(); + const timeoutMs = maestroTapOnTimeoutMs(params); + let lastResponse: DaemonResponse | undefined; + while (Date.now() - startedAt < timeoutMs) { + const attempt = await attemptMaestroTapOn(params, selector, options.value ?? {}); + if (!attempt.retry) return attempt.response; + lastResponse = attempt.response; + await sleep(MAESTRO_INTERACTION_POLICY.tapOnRetryMs); + } + + return maestroTapOnTimeoutResponse(params, selector, lastResponse); +} + +export async function invokeMaestroSwipeOn(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const [selector, direction = 'up', durationMs] = params.positionals; + if (!selector) return errorResponse('INVALID_ARGS', 'swipe.label requires a label selector.'); + const target = await resolveMaestroSnapshotTarget(params, selector, {}, 'swipe.label'); + if (!target.ok) return target.response; + const swipe = swipeCoordinatesFromTarget(target.target, direction); + if (!swipe.ok) return errorResponse('INVALID_ARGS', swipe.message); + return await invokeSwipeGesture(params, swipe, durationMs); +} + +async function invokeSwipeGesture( + params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + }, + swipe: { + start: { x: number; y: number }; + end: { x: number; y: number }; + }, + durationMs: string | undefined, +): Promise { + return await params.invoke({ + ...params.baseReq, + command: 'swipe', + positionals: [ + String(swipe.start.x), + String(swipe.start.y), + String(swipe.end.x), + String(swipe.end.y), + ...(durationMs ? [durationMs] : []), + ], + }); +} + +async function resolveMaestroScreenSwipe(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; +}): Promise< + | { + ok: true; + start: { x: number; y: number }; + end: { x: number; y: number }; + durationMs?: string; + } + | { ok: false; response: DaemonResponse } +> { + const cachedFrame = readCachedMaestroReferenceFrame(params.scope); + const frame = cachedFrame ?? (await captureFrameForMaestroScreenSwipe(params)); + if (!frame) { + return { + ok: false, + response: errorResponse('COMMAND_FAILED', 'Unable to resolve screen size for Maestro swipe.'), + }; + } + + const [mode, ...args] = params.positionals; + if (mode === 'direction') return resolveDirectionalScreenSwipe(args, frame); + if (mode === 'percent') { + return resolvePercentScreenSwipe( + args, + frame, + readMaestroSelectorPlatform(params.baseReq.flags), + ); + } + return { + ok: false, + response: errorResponse('INVALID_ARGS', 'Maestro screen swipe requires direction or percent.'), + }; +} + +async function captureFrameForMaestroScreenSwipe(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; +}): Promise<{ referenceWidth: number; referenceHeight: number } | undefined> { + const snapshotResponse = await captureMaestroRawSnapshot(params); + if (!snapshotResponse.ok) return undefined; + const snapshot = readSnapshotState(snapshotResponse.data); + return getSnapshotReferenceFrame(snapshot); +} + +function resolveDirectionalScreenSwipe( + args: string[], + frame: { referenceWidth: number; referenceHeight: number }, +): + | { + ok: true; + start: { x: number; y: number }; + end: { x: number; y: number }; + durationMs?: string; + } + | { ok: false; response: DaemonResponse } { + const [direction, durationMs] = args; + if (!direction) { + return { + ok: false, + response: errorResponse('INVALID_ARGS', 'Maestro direction swipe requires a direction.'), + }; + } + const point = (xPercent: number, yPercent: number) => percentPoint(frame, xPercent, yPercent, 8); + switch (direction) { + case 'up': + return { ok: true, start: point(50, 80), end: point(50, 20), durationMs }; + case 'down': + return { ok: true, start: point(50, 20), end: point(50, 80), durationMs }; + case 'left': + return { ok: true, start: point(80, 50), end: point(20, 50), durationMs }; + case 'right': + return { ok: true, start: point(20, 50), end: point(80, 50), durationMs }; + default: + return { + ok: false, + response: errorResponse( + 'INVALID_ARGS', + 'Maestro swipe direction must be UP, DOWN, LEFT, or RIGHT.', + ), + }; + } +} + +function resolvePercentScreenSwipe( + args: string[], + frame: { referenceWidth: number; referenceHeight: number }, + platform: string, +): + | { + ok: true; + start: { x: number; y: number }; + end: { x: number; y: number }; + durationMs?: string; + } + | { ok: false; response: DaemonResponse } { + const [startX, startY, endX, endY, durationMs] = args; + const values = [startX, startY, endX, endY].map(Number); + if (values.some((value) => !Number.isFinite(value))) { + return { + ok: false, + response: errorResponse('INVALID_ARGS', 'Maestro percentage swipe requires numeric points.'), + }; + } + const [x1, y1, x2, y2] = values as [number, number, number, number]; + const adjustedY = androidHorizontalContentSwipeY(platform, x1, y1, x2, y2); + return { + ok: true, + start: percentPoint(frame, x1, adjustedY, 1), + end: percentPoint(frame, x2, adjustedY, 1), + durationMs, + }; +} + +function androidHorizontalContentSwipeY( + platform: string, + x1: number, + y1: number, + x2: number, + y2: number, +): number { + if (platform !== 'android') return y2; + if (y1 !== y2 || y1 !== 50) return y2; + if (Math.abs(x2 - x1) < 50) return y2; + // Maestro's Android driver treats 50% horizontal swipes as content swipes. + // Raw `adb input swipe` at the physical screen midpoint can land above + // horizontally paged content in React Native layouts, so use a lower content + // lane for full-width horizontal Maestro percentage swipes. + return 65; +} + +function percentPoint( + frame: { referenceWidth: number; referenceHeight: number }, + xPercent: number, + yPercent: number, + marginPx: number, +): { x: number; y: number } { + return { + x: clampPoint( + Math.round((frame.referenceWidth * xPercent) / 100), + marginPx, + frame.referenceWidth, + ), + y: clampPoint( + Math.round((frame.referenceHeight * yPercent) / 100), + marginPx, + frame.referenceHeight, + ), + }; +} + +function clampPoint(value: number, marginPx: number, size: number): number { + const max = Math.max(marginPx, size - marginPx); + return Math.min(max, Math.max(marginPx, value)); +} + +async function probeMaestroScrollVisibility( + params: MaestroScrollUntilVisibleParams, + selector: string, + fuzzyTextQuery: string | null, + probeMs: number, +): Promise { + const waitResponse = await params.invoke({ + ...params.baseReq, + command: 'wait', + positionals: [selector, String(probeMs)], + }); + if (waitResponse.ok || !fuzzyTextQuery) return waitResponse; + + const fuzzyResponse = await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [fuzzyTextQuery, 'wait', String(probeMs)], + }); + return fuzzyResponse; +} + +function scrollProbeMs(timeoutMs: number, index: number): number { + return Math.min( + MAESTRO_INTERACTION_POLICY.scrollUntilVisibleProbeMs, + Math.max(1, timeoutMs - index * MAESTRO_INTERACTION_POLICY.scrollUntilVisibleProbeMs), + ); +} + +function maestroTapOnTimeoutMs(params: MaestroTapOnParams): number { + return params.baseReq.flags?.maestro?.optional === true + ? MAESTRO_INTERACTION_POLICY.optionalTapOnTimeoutMs + : MAESTRO_INTERACTION_POLICY.tapOnTimeoutMs; +} + +function maestroTapOnTimeoutResponse( + params: MaestroTapOnParams, + selector: string, + lastResponse: DaemonResponse | undefined, +): DaemonResponse { + if (params.baseReq.flags?.maestro?.optional === true) { + return { ok: true, data: { skipped: true, optional: true, selector } }; + } + return ( + lastResponse ?? errorResponse('COMMAND_FAILED', `tapOn timed out for selector: ${selector}`) + ); +} + +async function attemptMaestroTapOn( + params: MaestroTapOnParams, + selector: string, + options: MaestroTapOnOptions, +): Promise< + { retry: false; response: DaemonResponse } | { retry: true; response: FailedDaemonResponse } +> { + const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + const attempt = await invokeMaestroSnapshotTapOn(params, selector, options); + if (attempt.response.ok) return { retry: false, response: attempt.response }; + if (attempt.targetResolved && fuzzyTextQuery) { + return await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); + } + return { retry: true, response: attempt.response }; +} + +async function invokeMaestroSnapshotTapOn( + params: MaestroTapOnParams, + selector: string, + options: MaestroTapOnOptions, +): Promise<{ response: DaemonResponse; targetResolved: boolean }> { + const target = await resolveMaestroSnapshotTarget(params, selector, options, 'tapOn'); + if (!target.ok) return { response: target.response, targetResolved: false }; + const point = pointForMaestroTapOnTarget( + target.target, + extractMaestroVisibleTextQuery(selector) !== null, + ); + const response = await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [String(point.x), String(point.y)], + }); + return { + response, + targetResolved: true, + }; +} + +async function invokeMaestroFuzzyTapOn( + params: MaestroTapOnParams, + query: string, +): Promise< + { retry: false; response: DaemonResponse } | { retry: true; response: FailedDaemonResponse } +> { + const findResponse = await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [query, 'click'], + flags: { + ...params.baseReq.flags, + findFirst: true, + }, + }); + if (findResponse.ok) return { retry: false, response: findResponse }; + return { retry: true, response: findResponse }; +} + +async function resolveMaestroSnapshotTarget( + params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; + }, + selector: string, + options: MaestroTapOnOptions, + commandLabel: string, +): Promise<{ ok: true; target: MaestroSnapshotTarget } | { ok: false; response: DaemonResponse }> { + const cachedTarget = resolveCachedMaestroSnapshotTarget(params, selector, options); + if (cachedTarget.ok) return cachedTarget; + + const snapshotResponse = await captureMaestroRawSnapshot(params); + if (!snapshotResponse.ok) return { ok: false, response: snapshotResponse }; + + const snapshot = readSnapshotState(snapshotResponse.data); + if (!snapshot) { + return { + ok: false, + response: errorResponse( + 'COMMAND_FAILED', + `Unable to read snapshot data for ${commandLabel}.`, + ), + }; + } + + const frame = getSnapshotReferenceFrame(snapshot); + const resolution = resolveMaestroNodeFromSnapshot( + snapshot, + selector, + options, + readMaestroSelectorPlatform(params.baseReq.flags), + frame, + ); + if (!resolution.ok) { + const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + if (fuzzyTextQuery) { + const fuzzyResolution = resolveMaestroFuzzyTextNodeFromSnapshot( + snapshot, + fuzzyTextQuery, + frame, + ); + if (fuzzyResolution.ok) { + return { + ok: true, + target: { + node: fuzzyResolution.node, + rect: fuzzyResolution.rect, + frame, + }, + }; + } + } + } + if (!resolution.ok) { + return { + ok: false, + response: errorResponse('ELEMENT_NOT_FOUND', resolution.message, { + selector, + options, + command: commandLabel, + }), + }; + } + return { + ok: true, + target: { + node: resolution.node, + rect: resolution.rect, + frame, + }, + }; +} + +function resolveCachedMaestroSnapshotTarget( + params: { + baseReq: ReplayBaseRequest; + scope?: ReplayVarScope; + }, + selector: string, + options: MaestroTapOnOptions, +): { ok: true; target: MaestroSnapshotTarget } | { ok: false } { + const cached = consumeMaestroSnapshot(params.scope, selector); + if (!cached) return { ok: false }; + const resolution = resolveMaestroNodeFromSnapshot( + cached.snapshot, + selector, + options, + readMaestroSelectorPlatform(params.baseReq.flags), + cached.frame, + ); + return resolution.ok + ? { + ok: true, + target: { + node: resolution.node, + rect: resolution.rect, + frame: cached.frame, + }, + } + : { ok: false }; +} + +function readMaestroTapOnOptions( + rawOptions: string | undefined, +): { ok: true; value: MaestroTapOnOptions | null } | { ok: false; response: DaemonResponse } { + if (!rawOptions) return { ok: true, value: null }; + try { + const value = JSON.parse(rawOptions) as MaestroTapOnOptions; + return { ok: true, value }; + } catch { + return { + ok: false, + response: errorResponse('INVALID_ARGS', 'tapOn runtime options must be valid JSON.'), + }; + } +} + +function withMaestroScrollTimeoutContext( + response: FailedDaemonResponse | null, + selector: string, + timeoutMs: number, +): DaemonResponse { + if (!response) { + return errorResponse( + 'COMMAND_FAILED', + `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}`, + ); + } + return { + ok: false, + error: { + ...response.error, + message: `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}. Last wait: ${response.error.message}`, + }, + }; +} diff --git a/src/compat/maestro/runtime-support.ts b/src/compat/maestro/runtime-support.ts new file mode 100644 index 000000000..576da0ab7 --- /dev/null +++ b/src/compat/maestro/runtime-support.ts @@ -0,0 +1,120 @@ +import type { CommandFlags } from '../../core/dispatch.ts'; +import { + getSnapshotReferenceFrame, + type TouchReferenceFrame, +} from '../../daemon/touch-reference-frame.ts'; +import type { DaemonRequest, DaemonResponse, SessionAction } from '../../daemon/types.ts'; +import type { ReplayVarScope } from '../../replay/vars.ts'; +import type { SnapshotState } from '../../utils/snapshot.ts'; + +export type ReplayBaseRequest = Omit; + +export type MaestroReplayInvoker = (params: { + action: SessionAction; + line: number; + step: number; +}) => Promise; + +export type MaestroRuntimeInvoke = (req: DaemonRequest) => Promise; + +export type FailedDaemonResponse = Extract; + +const maestroReferenceFrameCache = new WeakMap(); +const maestroSnapshotCache = new WeakMap< + ReplayVarScope, + { snapshot: SnapshotState; frame: TouchReferenceFrame | undefined; selector: string } +>(); + +export function errorResponse( + code: string, + message: string, + details?: Record, +): FailedDaemonResponse { + return { + ok: false, + error: { code, message, ...(details ? { details } : {}) }, + }; +} + +export async function captureMaestroRawSnapshot(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; +}): Promise { + const response = await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, + }); + if (response.ok && params.scope) rememberMaestroReferenceFrame(params.scope, response.data); + return response; +} + +export function readSnapshotState(data: unknown): SnapshotState | undefined { + if ( + typeof data === 'object' && + data !== null && + Array.isArray((data as { nodes?: unknown }).nodes) + ) { + return data as SnapshotState; + } + return undefined; +} + +export function readCachedMaestroReferenceFrame( + scope: ReplayVarScope | undefined, +): TouchReferenceFrame | undefined { + return scope ? maestroReferenceFrameCache.get(scope) : undefined; +} + +export function rememberMaestroSnapshot( + scope: ReplayVarScope | undefined, + data: unknown, + selector: string, +): void { + if (!scope) return; + const snapshot = readSnapshotState(data); + if (!snapshot) return; + maestroSnapshotCache.set(scope, { + snapshot, + frame: getSnapshotReferenceFrame(snapshot), + selector, + }); +} + +export function consumeMaestroSnapshot( + scope: ReplayVarScope | undefined, + selector: string, +): { snapshot: SnapshotState; frame: TouchReferenceFrame | undefined } | undefined { + if (!scope) return undefined; + const cached = maestroSnapshotCache.get(scope); + maestroSnapshotCache.delete(scope); + return cached?.selector === selector ? cached : undefined; +} + +function rememberMaestroReferenceFrame(scope: ReplayVarScope, data: unknown): void { + const snapshot = readSnapshotState(data); + const frame = getSnapshotReferenceFrame(snapshot); + if (frame) maestroReferenceFrameCache.set(scope, frame); +} + +export function batchStepToSessionAction( + step: NonNullable[number], +): SessionAction { + const action: SessionAction = { + ts: Date.now(), + command: step.command, + positionals: step.positionals ?? [], + flags: step.flags ?? {}, + }; + if (step.runtime && typeof step.runtime === 'object') { + action.runtime = step.runtime as SessionAction['runtime']; + } + return action; +} diff --git a/src/compat/maestro/runtime-targets.ts b/src/compat/maestro/runtime-targets.ts new file mode 100644 index 000000000..7170b5b03 --- /dev/null +++ b/src/compat/maestro/runtime-targets.ts @@ -0,0 +1,454 @@ +import type { Platform } from '../../utils/device.ts'; +import type { Rect, SnapshotNode, SnapshotState } from '../../utils/snapshot.ts'; +import { parseSelectorChain } from '../../daemon/selectors.ts'; +import { matchesSelector } from '../../daemon/selectors-match.ts'; +import { evaluateIsPredicate } from '../../utils/selector-is-predicates.ts'; +import { normalizeText } from '../../utils/finders.ts'; +import { extractNodeText } from '../../utils/snapshot-processing.ts'; +import type { TouchReferenceFrame } from '../../daemon/touch-reference-frame.ts'; +import type { DaemonRequest } from '../../daemon/types.ts'; +import type { Selector, SelectorTerm } from '../../daemon/selectors-parse.ts'; +import { detectReactNativeOverlay } from '../../commands/react-native/overlay.ts'; + +const MAESTRO_TAP_TARGET_TYPE_RANK = new Map([ + ['button', 0], + ['link', 0], + ['textfield', 0], + ['textview', 0], + ['searchfield', 0], + ['switch', 0], + ['slider', 0], + ['cell', 1], + ['statictext', 2], +]); + +export type MaestroTapOnOptions = { + childOf?: string; + index?: number; +}; + +export type MaestroSnapshotTarget = { + node: SnapshotNode; + rect: Rect; + frame?: TouchReferenceFrame; +}; + +type MaestroResolvedSnapshotMatch = { + node: SnapshotNode; + rect: Rect; + inheritedRect: boolean; +}; + +type ReactNativeOverlayFilterResult = { + matches: SnapshotNode[]; + blockedByReactNativeOverlay: boolean; +}; + +type SnapshotNodeByIndex = Map; + +export function resolveMaestroNodeFromSnapshot( + snapshot: SnapshotState, + selector: string, + options: MaestroTapOnOptions, + platform: Platform, + frame: TouchReferenceFrame | undefined, +): { ok: true; node: SnapshotNode; rect: Rect } | { ok: false; message: string } { + let matches = findMaestroSelectorMatches(snapshot, selector, platform); + if (options.childOf) { + const parents = findMaestroSelectorMatches(snapshot, options.childOf, platform); + if (parents.length === 0) { + return { ok: false, message: `Maestro childOf parent did not match: ${options.childOf}` }; + } + const nodeByIndex = buildSnapshotNodeByIndex(snapshot.nodes); + matches = matches.filter((node) => + parents.some((parent) => + isDescendantOfSnapshotNode(snapshot.nodes, node, parent, nodeByIndex), + ), + ); + } + const filteredMatches = filterReactNativeOverlayBlockedMatches(snapshot.nodes, matches); + + const target = selectMaestroSnapshotMatch( + snapshot.nodes, + filteredMatches.matches, + options.index, + extractMaestroVisibleTextQuery(selector), + frame, + ); + if (!target) { + const index = options.index ?? 0; + return { + ok: false, + message: filteredMatches.blockedByReactNativeOverlay + ? `Maestro selector matched ${matches.length} element(s), but React Native overlay is covering app content: ${selector}` + : `Maestro selector did not match index ${index}: ${selector}`, + }; + } + return { ok: true, node: target.node, rect: target.rect }; +} + +export function resolveMaestroFuzzyTextNodeFromSnapshot( + snapshot: SnapshotState, + query: string, + frame: TouchReferenceFrame | undefined, +): { ok: true; node: SnapshotNode; rect: Rect } | { ok: false; message: string } { + const matches = findMaestroFuzzyTextMatches(snapshot, query); + const target = selectMaestroSnapshotMatch(snapshot.nodes, matches, undefined, query, frame); + if (!target) { + return { ok: false, message: `Maestro fuzzy text did not match: ${query}` }; + } + return { ok: true, node: target.node, rect: target.rect }; +} + +export function resolveVisibleMaestroNodeFromSnapshot( + snapshot: SnapshotState, + selector: string, + platform: Platform, + frame: TouchReferenceFrame | undefined, +): { ok: true; node: SnapshotNode; rect: Rect; matches: number } | { ok: false; message: string } { + const matches = findMaestroSelectorMatches(snapshot, selector, platform); + const visibleMatchesResult = filterVisibleMaestroMatches({ + nodes: snapshot.nodes, + matches, + platform, + }); + const target = selectMaestroSnapshotMatch( + snapshot.nodes, + visibleMatchesResult.matches, + undefined, + extractMaestroVisibleTextQuery(selector), + frame, + true, + ); + if (!target) { + return { + ok: false, + message: + matches.length > 0 + ? visibleMatchesResult.blockedByReactNativeOverlay + ? `Maestro selector matched ${matches.length} element(s), but React Native overlay is covering app content: ${selector}` + : `Maestro selector matched ${matches.length} element(s), but none were visible: ${selector}` + : `Maestro selector did not match: ${selector}`, + }; + } + return { + ok: true, + node: target.node, + rect: target.rect, + matches: visibleMatchesResult.matches.length, + }; +} + +function filterVisibleMaestroMatches(params: { + nodes: SnapshotState['nodes']; + matches: SnapshotNode[]; + platform: Platform; +}): { matches: SnapshotNode[]; blockedByReactNativeOverlay: boolean } { + const visibleMatches = params.matches.filter( + (node) => + evaluateIsPredicate({ + predicate: 'visible', + node, + nodes: params.nodes, + platform: params.platform, + }).pass, + ); + const overlayFilter = filterReactNativeOverlayBlockedMatches(params.nodes, visibleMatches); + return { + matches: overlayFilter.matches, + blockedByReactNativeOverlay: overlayFilter.blockedByReactNativeOverlay, + }; +} + +function filterReactNativeOverlayBlockedMatches( + nodes: SnapshotState['nodes'], + matches: SnapshotNode[], +): ReactNativeOverlayFilterResult { + const overlay = detectReactNativeOverlay(nodes); + if (!overlay.detected) { + return { matches, blockedByReactNativeOverlay: false }; + } + const overlayNodeIndexes = new Set( + [...overlay.dismissNodes, ...overlay.minimizeNodes, ...overlay.collapsedNodes].map( + (node) => node.index, + ), + ); + const overlayMatches = matches.filter((node) => overlayNodeIndexes.has(node.index)); + return { + matches: overlayMatches, + blockedByReactNativeOverlay: matches.length > 0 && overlayMatches.length === 0, + }; +} + +export function readMaestroSelectorPlatform(flags: DaemonRequest['flags']): Platform { + return flags?.platform === 'android' ? 'android' : 'ios'; +} + +export function extractMaestroVisibleTextQuery(selectorExpression: string): string | null { + const chain = parseSelectorChain(selectorExpression); + const terms = chain.selectors.flatMap((selector) => selector.terms); + if (terms.length === 0) return null; + // Mixed selectors may encode more than a visible-text lookup, so they keep + // the exact selector path instead of fuzzy text fallback. + if (!terms.some((term) => term.key === 'label' || term.key === 'text')) return null; + if (!terms.every((term) => ['label', 'text', 'id'].includes(term.key))) return null; + const values = terms.map((term) => (typeof term.value === 'string' ? term.value : '')); + const first = values[0]; + if (!first || !values.every((value) => value === first)) return null; + return first; +} + +function findMaestroSelectorMatches( + snapshot: SnapshotState, + selectorExpression: string, + platform: Platform, +): SnapshotNode[] { + const chain = parseSelectorChain(selectorExpression); + for (const selector of chain.selectors) { + const matches = snapshot.nodes.filter((node) => + matchesMaestroSelector(node, selector, platform), + ); + if (matches.length > 0) return matches; + } + return []; +} + +function findMaestroFuzzyTextMatches(snapshot: SnapshotState, query: string): SnapshotNode[] { + const normalizedQuery = normalizeText(query); + if (!normalizedQuery) return []; + const exact: SnapshotNode[] = []; + const partial: SnapshotNode[] = []; + for (const node of snapshot.nodes) { + const values = [node.label, extractNodeText(node), node.identifier, node.value].filter( + (value): value is string => Boolean(value), + ); + const normalizedValues = values.map((value) => normalizeText(value)); + if (normalizedValues.some((value) => value === normalizedQuery)) { + exact.push(node); + } else if (normalizedValues.some((value) => value.includes(normalizedQuery))) { + partial.push(node); + } + } + return exact.length > 0 ? exact : partial; +} + +function matchesMaestroSelector( + node: SnapshotNode, + selector: Selector, + platform: Platform, +): boolean { + if (matchesSelector(node, selector, platform)) return true; + return selector.terms.every((term) => matchesMaestroTerm(node, term, platform)); +} + +function matchesMaestroTerm(node: SnapshotNode, term: SelectorTerm, platform: Platform): boolean { + if (typeof term.value !== 'string' || !isMaestroRegexTextKey(term.key)) { + return matchesSelector(node, { raw: term.key, terms: [term] }, platform); + } + const value = readMaestroTextTermValue(node, term.key); + return textEqualsOrRegex(value, term.value); +} + +function isMaestroRegexTextKey(key: SelectorTerm['key']): key is 'id' | 'label' | 'text' | 'value' { + return key === 'id' || key === 'label' || key === 'text' || key === 'value'; +} + +function readMaestroTextTermValue( + node: SnapshotNode, + key: 'id' | 'label' | 'text' | 'value', +): string | undefined { + if (key === 'id') return node.identifier; + if (key === 'label') return node.label; + if (key === 'value') return node.value; + return extractNodeText(node); +} + +function textEqualsOrRegex(value: string | undefined, query: string): boolean { + const text = value ?? ''; + if (normalizeText(text) === normalizeText(query)) return true; + try { + return new RegExp(query).test(text); + } catch { + return false; + } +} + +function resolveNodeRect( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + nodeByIndex: SnapshotNodeByIndex, +): { rect: Rect; inherited: boolean } | null { + if (node.rect && node.rect.width > 0 && node.rect.height > 0) { + return { rect: node.rect, inherited: false }; + } + if (node.rect) return null; + const rect = resolveRectlessNodeAncestorRect(nodes, node, nodeByIndex); + return rect ? { rect, inherited: true } : null; +} + +function resolveRectlessNodeAncestorRect( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + nodeByIndex: SnapshotNodeByIndex, +): Rect | null { + let current: SnapshotNode | undefined = node; + while (typeof current.parentIndex === 'number') { + current = nodeByIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; + if (!current) return null; + if (!current.rect) continue; + return current.rect.width > 0 && current.rect.height > 0 ? current.rect : null; + } + return null; +} + +function selectMaestroSnapshotMatch( + nodes: SnapshotState['nodes'], + matches: SnapshotNode[], + index: number | undefined, + visibleTextQuery: string | null, + frame: TouchReferenceFrame | undefined, + requireOnScreen = false, +): { node: SnapshotNode; rect: Rect } | null { + const nodeByIndex = buildSnapshotNodeByIndex(nodes); + const resolved = matches + .map((node) => { + const match = resolveNodeRect(nodes, node, nodeByIndex); + return match ? { node, rect: match.rect, inheritedRect: match.inherited } : null; + }) + .filter((candidate): candidate is MaestroResolvedSnapshotMatch => Boolean(candidate)); + const candidates = + visibleTextQuery && index === undefined + ? preferOnScreenMatches(resolved, frame, requireOnScreen) + : resolved; + if (index !== undefined) { + return candidates[index] ?? null; + } + const sorted = candidates.sort((left, right) => + compareMaestroSnapshotMatches(left, right, visibleTextQuery), + ); + return sorted[0] ?? null; +} + +function preferOnScreenMatches( + matches: MaestroResolvedSnapshotMatch[], + frame: TouchReferenceFrame | undefined, + requireOnScreen: boolean, +): MaestroResolvedSnapshotMatch[] { + const onScreen = matches.filter((match) => isRectOnScreen(match.rect, frame)); + if (requireOnScreen) return onScreen; + return onScreen.length > 0 ? onScreen : matches; +} + +function isRectOnScreen(rect: Rect, frame: TouchReferenceFrame | undefined): boolean { + const maxX = frame?.referenceWidth ?? Number.POSITIVE_INFINITY; + const maxY = frame?.referenceHeight ?? Number.POSITIVE_INFINITY; + return rect.x < maxX && rect.y < maxY && rect.x + rect.width > 0 && rect.y + rect.height > 0; +} + +function compareMaestroSnapshotMatches( + left: MaestroResolvedSnapshotMatch, + right: MaestroResolvedSnapshotMatch, + visibleTextQuery: string | null, +): number { + const priorityRank = compareMaestroSnapshotMatchPriority(left, right, visibleTextQuery); + if (priorityRank !== 0) return priorityRank; + + if (!sameRoundedRect(left.rect, right.rect)) { + return left.node.index - right.node.index; + } + + const depthRank = (right.node.depth ?? 0) - (left.node.depth ?? 0); + if (depthRank !== 0) return depthRank; + + // Android transparent stacks can expose both the background screen and the + // foreground screen at the same coordinates. UIAutomator reports the + // foreground duplicate later in the snapshot, which matches Maestro's + // practical tap target for overlapping duplicates. + return right.node.index - left.node.index; +} + +function sameRoundedRect(left: Rect, right: Rect): boolean { + return ( + Math.round(left.x) === Math.round(right.x) && + Math.round(left.y) === Math.round(right.y) && + Math.round(left.width) === Math.round(right.width) && + Math.round(left.height) === Math.round(right.height) + ); +} + +function compareMaestroSnapshotMatchPriority( + left: MaestroResolvedSnapshotMatch, + right: MaestroResolvedSnapshotMatch, + visibleTextQuery: string | null, +): number { + if (visibleTextQuery) { + const textRank = + maestroVisibleTextMatchRank(left.node, visibleTextQuery) - + maestroVisibleTextMatchRank(right.node, visibleTextQuery); + if (textRank !== 0) return textRank; + } + + const typeRank = maestroTapTargetTypeRank(left.node) - maestroTapTargetTypeRank(right.node); + if (typeRank !== 0) return typeRank; + + const rectSourceRank = Number(left.inheritedRect) - Number(right.inheritedRect); + if (rectSourceRank !== 0) return rectSourceRank; + + const areaRank = + visibleTextQuery && maestroTapTargetTypeRank(left.node) === maestroTapTargetTypeRank(right.node) + ? rectArea(right.rect) - rectArea(left.rect) + : rectArea(left.rect) - rectArea(right.rect); + if (areaRank !== 0) return areaRank; + return 0; +} + +function rectArea(rect: Rect): number { + return rect.width * rect.height; +} + +function maestroTapTargetTypeRank(node: SnapshotNode): number { + return MAESTRO_TAP_TARGET_TYPE_RANK.get(node.type?.toLowerCase() ?? '') ?? 3; +} + +function maestroVisibleTextMatchRank(node: SnapshotNode, query: string): number { + const values = [node.label, extractNodeText(node), node.identifier, node.value].filter( + (value): value is string => Boolean(value), + ); + if (values.some((value) => value === query)) return 0; + if (values.some((value) => normalizeText(value) === normalizeText(query))) return 1; + if (values.some((value) => textEqualsOrRegex(value, query))) return 2; + return 3; +} + +function isDescendantOfSnapshotNode( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + ancestor: SnapshotNode, + nodeByIndex: SnapshotNodeByIndex, +): boolean { + return Boolean( + findSnapshotAncestor(nodes, node, nodeByIndex, (candidate) => + candidate === ancestor || candidate.index === ancestor.index ? candidate : null, + ), + ); +} + +function findSnapshotAncestor( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + nodeByIndex: SnapshotNodeByIndex, + resolve: (ancestor: SnapshotNode) => T | null, +): T | null { + let current: SnapshotNode | undefined = node; + while (typeof current.parentIndex === 'number') { + current = nodeByIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; + if (!current) return null; + const result = resolve(current); + if (result) return result; + } + return null; +} + +function buildSnapshotNodeByIndex(nodes: SnapshotState['nodes']): SnapshotNodeByIndex { + return new Map(nodes.map((candidate) => [candidate.index, candidate])); +} diff --git a/src/compat/maestro/runtime.ts b/src/compat/maestro/runtime.ts new file mode 100644 index 000000000..42cd0950f --- /dev/null +++ b/src/compat/maestro/runtime.ts @@ -0,0 +1,108 @@ +import { type CommandFlags } from '../../core/dispatch.ts'; +import { asAppError } from '../../utils/errors.ts'; +import type { ReplayVarScope } from '../../replay/vars.ts'; +import type { DaemonRequest, DaemonResponse } from '../../daemon/types.ts'; +import { executeRunScriptFile } from './run-script.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; +import { + invokeMaestroAssertNotVisible, + invokeMaestroAssertVisible, + invokeMaestroWaitForAnimationToEnd, +} from './runtime-assertions.ts'; +import { invokeMaestroRetry, invokeMaestroRunFlowWhen } from './runtime-flow.ts'; +import { + errorResponse, + type MaestroReplayInvoker, + type MaestroRuntimeInvoke, + type ReplayBaseRequest, +} from './runtime-support.ts'; +import { + invokeMaestroScrollUntilVisible, + invokeMaestroSwipeScreen, + invokeMaestroSwipeOn, + invokeMaestroTapOn, + invokeMaestroTapPointPercent, +} from './runtime-interactions.ts'; + +export async function invokeMaestroRuntimeCommand(params: { + command: string; + baseReq: ReplayBaseRequest; + positionals: string[]; + batchSteps: CommandFlags['batchSteps'] | undefined; + scope: ReplayVarScope; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: MaestroReplayInvoker; +}): Promise { + switch (params.command) { + case MAESTRO_RUNTIME_COMMAND.assertVisible: + return await invokeMaestroAssertVisible(params); + case MAESTRO_RUNTIME_COMMAND.assertNotVisible: + return await invokeMaestroAssertNotVisible(params); + case MAESTRO_RUNTIME_COMMAND.retry: + return await invokeMaestroRetry(params); + case MAESTRO_RUNTIME_COMMAND.pressEnter: + return await invokeMaestroPressEnter(params); + case MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd: + return await invokeMaestroWaitForAnimationToEnd(params); + case MAESTRO_RUNTIME_COMMAND.scrollUntilVisible: + return await invokeMaestroScrollUntilVisible(params); + case MAESTRO_RUNTIME_COMMAND.swipeScreen: + return await invokeMaestroSwipeScreen(params); + case MAESTRO_RUNTIME_COMMAND.swipeOn: + return await invokeMaestroSwipeOn(params); + case MAESTRO_RUNTIME_COMMAND.tapOn: + return await invokeMaestroTapOn(params); + case MAESTRO_RUNTIME_COMMAND.tapPointPercent: + return await invokeMaestroTapPointPercent(params); + case MAESTRO_RUNTIME_COMMAND.runFlowWhen: + return await invokeMaestroRunFlowWhen(params); + case MAESTRO_RUNTIME_COMMAND.runScript: + return invokeMaestroRunScript(params); + default: + return undefined; + } +} + +async function invokeMaestroPressEnter(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; +}): Promise { + const keyboardResponse = await params.invoke({ + ...params.baseReq, + command: 'keyboard', + positionals: ['enter'], + }); + if (keyboardResponse.ok) return keyboardResponse; + + return await params.invoke({ + ...params.baseReq, + command: 'type', + positionals: ['\n'], + }); +} + +function invokeMaestroRunScript(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + scope: ReplayVarScope; +}): DaemonResponse { + const [scriptPath] = params.positionals; + if (!scriptPath) { + return errorResponse('INVALID_ARGS', 'runScript requires a file path.'); + } + try { + const outputEnv = executeRunScriptFile({ + scriptPath, + env: { + ...params.scope.values, + ...(params.baseReq.flags?.maestro?.runScriptEnv ?? {}), + }, + }); + return { ok: true, data: { outputEnv } }; + } catch (error) { + const appError = asAppError(error); + return errorResponse(appError.code, appError.message, appError.details); + } +} diff --git a/src/compat/maestro/support.ts b/src/compat/maestro/support.ts index 997a5c70d..39d3c20b8 100644 --- a/src/compat/maestro/support.ts +++ b/src/compat/maestro/support.ts @@ -52,17 +52,6 @@ export function normalizePlatform(value: string | undefined): 'android' | 'ios' return normalizePlatformName(value); } -export function normalizePlatformValue(value: unknown, name: string): 'android' | 'ios' { - if (typeof value !== 'string') { - throw new AppError('INVALID_ARGS', `${name} expects Android or iOS.`); - } - const platform = normalizePlatformName(value); - if (!platform) { - throw new AppError('INVALID_ARGS', `${name} expects Android or iOS.`); - } - return platform; -} - export function readEnvMap(value: unknown, name: string): Record { if (value === undefined || value === null) return {}; if (!isPlainRecord(value)) { diff --git a/src/core/__tests__/app-events.test.ts b/src/core/__tests__/app-events.test.ts index aa8c4e89d..a2ebee940 100644 --- a/src/core/__tests__/app-events.test.ts +++ b/src/core/__tests__/app-events.test.ts @@ -9,4 +9,3 @@ test('parseTriggerAppEventArgs validates event name format', () => { (error) => error instanceof AppError && error.code === 'INVALID_ARGS', ); }); - diff --git a/src/core/__tests__/dispatch-interactions.test.ts b/src/core/__tests__/dispatch-interactions.test.ts index eba5b2eff..e5f4716f1 100644 --- a/src/core/__tests__/dispatch-interactions.test.ts +++ b/src/core/__tests__/dispatch-interactions.test.ts @@ -5,10 +5,7 @@ import { handleTransformGestureCommand, } from '../dispatch-interactions.ts'; import type { Interactor } from '../interactor-types.ts'; -import { - ANDROID_EMULATOR, - IOS_SIMULATOR, -} from '../../__tests__/test-utils/device-fixtures.ts'; +import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; vi.mock('../../platforms/ios/macos-helper.ts', async (importOriginal) => { const actual = await importOriginal(); diff --git a/src/core/__tests__/dispatch-series.test.ts b/src/core/__tests__/dispatch-series.test.ts index b8b5e0b18..e7630d63f 100644 --- a/src/core/__tests__/dispatch-series.test.ts +++ b/src/core/__tests__/dispatch-series.test.ts @@ -93,4 +93,3 @@ test('shouldUseIosDragSeries returns false when count is 1', () => { // --- computeDeterministicJitter --- // --- runRepeatedSeries --- - diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 05888e718..4367157ab 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -22,6 +22,7 @@ import { emitDiagnostic, withDiagnosticTimer } from '../utils/diagnostics.ts'; import { readLocationCoordinate } from '../utils/location-coordinates.ts'; import { successText, withSuccessText } from '../utils/success-text.ts'; import { screenshotOptionsFromFlags } from '../commands/capture-screenshot-options.ts'; +import { isKeyboardAction, type KeyboardAction } from '../utils/keyboard-actions.ts'; import type { DispatchContext } from './dispatch-context.ts'; import { handleFillCommand, @@ -44,105 +45,6 @@ import { parseDeviceRotation } from './device-rotation.ts'; export { resolveTargetDevice } from './dispatch-resolve.ts'; export type { BatchStep, CommandFlags, DispatchContext } from './dispatch-context.ts'; -type DispatchCommandHandlerParams = { - device: DeviceInfo; - interactor: Interactor; - positionals: string[]; - outPath?: string; - context?: DispatchContext; - runnerCtx: RunnerContext; -}; - -type DispatchCommandHandler = ( - params: DispatchCommandHandlerParams, -) => Promise | void> | Record | void; - -const DISPATCH_COMMAND_HANDLERS: Record = { - open: ({ device, interactor, positionals, context }) => - handleOpenCommand(device, interactor, positionals, context), - close: async ({ interactor, positionals }) => { - const app = positionals[0]; - if (!app) { - return { closed: 'session', ...successText('Closed session') }; - } - await interactor.close(app); - return { app, ...successText(`Closed: ${app}`) }; - }, - press: ({ device, interactor, positionals, context }) => - handlePressCommand(device, interactor, positionals, context), - swipe: ({ device, interactor, positionals, context }) => - handleSwipeCommand(device, interactor, positionals, context), - pan: ({ interactor, positionals }) => handlePanCommand(interactor, positionals), - fling: ({ interactor, positionals }) => handleFlingCommand(interactor, positionals), - longpress: ({ interactor, positionals }) => handleLongPressCommand(interactor, positionals), - focus: ({ interactor, positionals }) => handleFocusCommand(interactor, positionals), - type: ({ interactor, positionals, context }) => - handleTypeCommand(interactor, positionals, context), - fill: ({ interactor, positionals, context }) => - handleFillCommand(interactor, positionals, context), - scroll: ({ interactor, positionals, context }) => - handleScrollCommand(interactor, positionals, context), - pinch: ({ device, interactor, positionals, context }) => - handlePinchCommand(device, interactor, positionals, context), - 'rotate-gesture': ({ device, interactor, positionals }) => - handleRotateGestureCommand(device, interactor, positionals), - 'transform-gesture': ({ device, interactor, positionals }) => - handleTransformGestureCommand(device, interactor, positionals), - 'trigger-app-event': async ({ device, interactor, positionals, context }) => { - const { eventName, payload } = parseTriggerAppEventArgs(positionals); - const eventUrl = resolveAppEventUrl(device.platform, eventName, payload); - await interactor.open(eventUrl, { appBundleId: context?.appBundleId }); - return { - event: eventName, - eventUrl, - transport: 'deep-link', - ...successText(`Triggered app event: ${eventName}`), - }; - }, - screenshot: async ({ interactor, positionals, outPath, context }) => { - const positionalPath = positionals[0]; - const screenshotPath = positionalPath ?? outPath ?? `./screenshot-${Date.now()}.png`; - await fs.mkdir(pathModule.dirname(screenshotPath), { recursive: true }); - const screenshotOptions = screenshotOptionsFromFlags(context); - await interactor.screenshot(screenshotPath, { - appBundleId: context?.appBundleId, - fullscreen: screenshotOptions.fullscreen, - stabilize: screenshotOptions.stabilize, - surface: context?.surface, - }); - return { path: screenshotPath, ...successText(`Saved screenshot: ${screenshotPath}`) }; - }, - back: async ({ interactor, context }) => { - await interactor.back(context?.backMode); - return { action: 'back', mode: context?.backMode ?? 'in-app', ...successText('Back') }; - }, - home: async ({ interactor }) => { - await interactor.home(); - return { action: 'home', ...successText('Home') }; - }, - rotate: async ({ interactor, positionals }) => { - const orientation = parseDeviceRotation(positionals[0]); - await interactor.rotate(orientation); - return { - action: 'rotate', - orientation, - ...successText(`Rotated to ${orientation}`), - }; - }, - 'app-switcher': async ({ interactor }) => { - await interactor.appSwitcher(); - return { action: 'app-switcher', ...successText('Opened app switcher') }; - }, - clipboard: ({ interactor, positionals }) => handleClipboardCommand(interactor, positionals), - keyboard: ({ device, positionals, context, runnerCtx }) => - handleKeyboardCommand(device, positionals, context, runnerCtx), - settings: ({ device, interactor, positionals, context }) => - handleSettingsCommand(device, interactor, positionals, context), - push: ({ device, positionals, context }) => handlePushCommand(device, positionals, context), - snapshot: ({ interactor, context }) => handleSnapshotCommand(interactor, context), - read: ({ device, positionals, context }) => handleReadCommand(device, positionals, context), -}; - export async function dispatchCommand( device: DeviceInfo, command: string, @@ -170,9 +72,15 @@ export async function dispatchCommand( return await withDiagnosticTimer( 'platform_command', async () => { - const handler = DISPATCH_COMMAND_HANDLERS[command]; - if (!handler) throw new AppError('INVALID_ARGS', `Unknown command: ${command}`); - return await handler({ device, interactor, positionals, outPath, context, runnerCtx }); + return await dispatchKnownCommand( + device, + interactor, + command, + positionals, + outPath, + context, + runnerCtx, + ); }, { command, @@ -181,6 +89,84 @@ export async function dispatchCommand( ); } +// fallow-ignore-next-line complexity +async function dispatchKnownCommand( + device: DeviceInfo, + interactor: Interactor, + command: string, + positionals: string[], + outPath: string | undefined, + context: DispatchContext | undefined, + runnerCtx: RunnerContext, +): Promise | void> { + switch (command) { + case 'open': + return await handleOpenCommand(device, interactor, positionals, context); + case 'close': { + const app = positionals[0]; + if (!app) return { closed: 'session', ...successText('Closed session') }; + await interactor.close(app); + return { app, ...successText(`Closed: ${app}`) }; + } + case 'press': + return await handlePressCommand(device, interactor, positionals, context); + case 'swipe': + return await handleSwipeCommand(device, interactor, positionals, context); + case 'pan': + return await handlePanCommand(interactor, positionals); + case 'fling': + return await handleFlingCommand(interactor, positionals); + case 'longpress': + return await handleLongPressCommand(interactor, positionals); + case 'focus': + return await handleFocusCommand(interactor, positionals); + case 'type': + return await handleTypeCommand(interactor, positionals, context); + case 'fill': + return await handleFillCommand(interactor, positionals, context); + case 'scroll': + return await handleScrollCommand(interactor, positionals, context); + case 'pinch': + return await handlePinchCommand(device, interactor, positionals, context); + case 'rotate-gesture': + return await handleRotateGestureCommand(device, interactor, positionals); + case 'transform-gesture': + return await handleTransformGestureCommand(device, interactor, positionals); + case 'trigger-app-event': + return await handleTriggerAppEventCommand(device, interactor, positionals, context); + case 'screenshot': + return await handleScreenshotCommand(interactor, positionals, outPath, context); + case 'back': + await interactor.back(context?.backMode); + return { action: 'back', mode: context?.backMode ?? 'in-app', ...successText('Back') }; + case 'home': + await interactor.home(); + return { action: 'home', ...successText('Home') }; + case 'rotate': { + const orientation = parseDeviceRotation(positionals[0]); + await interactor.rotate(orientation); + return { action: 'rotate', orientation, ...successText(`Rotated to ${orientation}`) }; + } + case 'app-switcher': + await interactor.appSwitcher(); + return { action: 'app-switcher', ...successText('Opened app switcher') }; + case 'clipboard': + return await handleClipboardCommand(interactor, positionals); + case 'keyboard': + return await handleKeyboardCommand(device, positionals, context, runnerCtx); + case 'settings': + return await handleSettingsCommand(device, interactor, positionals, context); + case 'push': + return await handlePushCommand(device, positionals, context); + case 'snapshot': + return await handleSnapshotCommand(interactor, context); + case 'read': + return await handleReadCommand(device, positionals, context); + default: + throw new AppError('INVALID_ARGS', `Unknown command: ${command}`); + } +} + // --------------------------------------------------------------------------- // Command handlers // --------------------------------------------------------------------------- @@ -209,9 +195,6 @@ async function handleOpenCommand( throw new AppError('UNSUPPORTED_OPERATION', LAUNCH_CONSOLE_IOS_SIMULATOR_ONLY_MESSAGE); } if (url !== undefined) { - if (device.platform === 'android') { - throw new AppError('INVALID_ARGS', 'open is supported only on Apple platforms'); - } if (isDeepLinkTarget(app)) { throw new AppError( 'INVALID_ARGS', @@ -265,6 +248,42 @@ async function handleOpenCommand( return { app, ...(launchConsole ? { launchConsole } : {}), ...successText(`Opened: ${app}`) }; } +async function handleTriggerAppEventCommand( + device: DeviceInfo, + interactor: Interactor, + positionals: string[], + context: DispatchContext | undefined, +): Promise> { + const { eventName, payload } = parseTriggerAppEventArgs(positionals); + const eventUrl = resolveAppEventUrl(device.platform, eventName, payload); + await interactor.open(eventUrl, { appBundleId: context?.appBundleId }); + return { + event: eventName, + eventUrl, + transport: 'deep-link', + ...successText(`Triggered app event: ${eventName}`), + }; +} + +async function handleScreenshotCommand( + interactor: Interactor, + positionals: string[], + outPath: string | undefined, + context: DispatchContext | undefined, +): Promise> { + const positionalPath = positionals[0]; + const screenshotPath = positionalPath ?? outPath ?? `./screenshot-${Date.now()}.png`; + await fs.mkdir(pathModule.dirname(screenshotPath), { recursive: true }); + const screenshotOptions = screenshotOptionsFromFlags(context); + await interactor.screenshot(screenshotPath, { + appBundleId: context?.appBundleId, + fullscreen: screenshotOptions.fullscreen, + stabilize: screenshotOptions.stabilize, + surface: context?.surface, + }); + return { path: screenshotPath, ...successText(`Saved screenshot: ${screenshotPath}`) }; +} + async function handleClipboardCommand( interactor: Interactor, positionals: string[], @@ -317,21 +336,9 @@ async function handleKeyboardCommand( throw new AppError('UNSUPPORTED_OPERATION', 'keyboard is supported only on Android and iOS'); } -function isKeyboardAction( - action: string, -): action is 'status' | 'get' | 'dismiss' | 'enter' | 'return' { - return ( - action === 'status' || - action === 'get' || - action === 'dismiss' || - action === 'enter' || - action === 'return' - ); -} - async function handleAndroidKeyboardCommand( device: DeviceInfo, - action: 'status' | 'get' | 'dismiss' | 'enter' | 'return', + action: KeyboardAction, ): Promise> { if (action === 'enter' || action === 'return') { await pressAndroidEnter(device); @@ -374,7 +381,7 @@ async function handleAndroidKeyboardCommand( async function handleIosKeyboardCommand( device: DeviceInfo, - action: 'status' | 'get' | 'dismiss' | 'enter' | 'return', + action: KeyboardAction, context: DispatchContext | undefined, runnerCtx: RunnerContext, ): Promise> { diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index 3dc7f2993..8ebb77220 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -34,7 +34,12 @@ import type { Interactor } from '../interactor-types.ts'; export function createAndroidInteractor(device: DeviceInfo): Interactor { return { - open: (app, options) => openAndroidApp(device, app, options?.activity), + open: (app, options) => + openAndroidApp(device, app, { + activity: options?.activity, + appBundleId: options?.appBundleId, + url: options?.url, + }), openDevice: () => openAndroidDevice(device), close: (app) => closeAndroidApp(device, app), tap: (x, y) => pressAndroid(device, x, y), diff --git a/src/daemon-client.ts b/src/daemon-client.ts index 2faf868c8..4dce1e4fc 100644 --- a/src/daemon-client.ts +++ b/src/daemon-client.ts @@ -2,6 +2,7 @@ import net from 'node:net'; import http from 'node:http'; import https from 'node:https'; import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; import { sleep } from './utils/timeouts.ts'; @@ -26,6 +27,7 @@ import { } from './daemon/config.ts'; import { uploadArtifact } from './upload-client.ts'; import { computeDaemonCodeSignature } from './daemon/code-signature.ts'; +import { PUBLIC_COMMANDS } from './command-catalog.ts'; export { computeDaemonCodeSignature } from './daemon/code-signature.ts'; export type DaemonRequest = SharedDaemonRequest; export type DaemonResponse = SharedDaemonResponse; @@ -90,10 +92,16 @@ type DaemonClientSettings = { paths: DaemonPaths; transportPreference: DaemonTransportPreference; serverMode: DaemonServerMode; + ownedStateDir?: boolean; remoteBaseUrl?: string; remoteAuthToken?: string; }; +type EnsuredDaemon = { + info: DaemonInfo; + startedByClient: boolean; +}; + type ResolvedDaemonTransport = 'socket' | 'http'; const REQUEST_TIMEOUT_MS = 90_000; @@ -118,11 +126,12 @@ export async function sendToDaemon(req: Omit): Promise await ensureDaemon(settings), { requestId, session: req.session }, ); + const info = daemon.info; const preparedRemoteRequest = await prepareRemoteRequest(req, info); const request = { @@ -161,16 +170,20 @@ export async function sendToDaemon(req: Omit): Promise await sendRequest(info, request, settings.transportPreference, requestTimeoutMs), - { requestId, command: req.command }, - ); + try { + return await withDiagnosticTimer( + 'daemon_request', + async () => await sendRequest(info, request, settings.transportPreference, requestTimeoutMs), + { requestId, command: req.command }, + ); + } finally { + await cleanupDaemonAfterRequest(req, daemon, settings); + } } function resolveDaemonRequestTimeoutMs(req: Omit): number | undefined { - if (req.command === 'test') return undefined; - if (req.command === 'replay' && typeof req.flags?.timeoutMs === 'number') { + if (req.command === PUBLIC_COMMANDS.test) return undefined; + if (req.command === PUBLIC_COMMANDS.replay && typeof req.flags?.timeoutMs === 'number') { return req.flags.timeoutMs; } return REQUEST_TIMEOUT_MS; @@ -235,18 +248,7 @@ async function prepareRemoteRequest( let uploadedArtifactId: string | undefined; if (isRemoteDaemon(info)) { - const remoteArtifact = prepareRemoteArtifactCommand(req, positionals); - if (remoteArtifact) { - if (remoteArtifact.positionalPath !== undefined) { - positionals[remoteArtifact.positionalIndex] = remoteArtifact.positionalPath; - } - if (remoteArtifact.flagPath !== undefined) { - flags ??= {}; - flags.out = remoteArtifact.flagPath; - } - clientArtifactPaths[remoteArtifact.field] = remoteArtifact.localPath; - } - + flags = applyRemoteArtifactCommand(req, positionals, flags, clientArtifactPaths); const remoteInstallSource = await prepareRemoteInstallSource(req, info); if (remoteInstallSource) { installSource = remoteInstallSource.installSource; @@ -277,10 +279,8 @@ async function prepareRemoteRequest( return createPreparedRemoteRequest({ positionals, flags, clientArtifactPaths }); } - const localPath = path.isAbsolute(rawPath) - ? rawPath - : path.resolve(req.meta?.cwd ?? process.cwd(), rawPath); - if (!fs.existsSync(localPath)) { + const localPath = resolveLocalInstallPath(rawPath, req.meta?.cwd); + if (!localPath) { return createPreparedRemoteRequest({ positionals, flags, clientArtifactPaths }); } @@ -293,6 +293,37 @@ async function prepareRemoteRequest( return baseResult(); } +function applyRemoteArtifactCommand( + req: Omit, + positionals: string[], + flags: DaemonRequest['flags'] | undefined, + clientArtifactPaths: Record, +): DaemonRequest['flags'] | undefined { + const remoteArtifact = prepareRemoteArtifactCommand(req, positionals); + if (!remoteArtifact) return flags; + if (remoteArtifact.positionalPath !== undefined) { + positionals[remoteArtifact.positionalIndex] = remoteArtifact.positionalPath; + } + const nextFlags = applyRemoteArtifactOutFlag(flags, remoteArtifact.flagPath); + clientArtifactPaths[remoteArtifact.field] = remoteArtifact.localPath; + return nextFlags; +} + +function applyRemoteArtifactOutFlag( + flags: DaemonRequest['flags'] | undefined, + flagPath: string | undefined, +): DaemonRequest['flags'] | undefined { + if (flagPath === undefined) return flags; + return { ...(flags ?? {}), out: flagPath }; +} + +function resolveLocalInstallPath(rawPath: string, cwd: string | undefined): string | undefined { + const localPath = path.isAbsolute(rawPath) + ? rawPath + : path.resolve(cwd ?? process.cwd(), rawPath); + return fs.existsSync(localPath) ? localPath : undefined; +} + type PreparedRemoteRequest = { positionals: string[]; flags?: DaemonRequest['flags']; @@ -427,10 +458,11 @@ function buildRemoteTempArtifactPath(prefix: string, extension: string): string } function resolveClientSettings(req: Omit): DaemonClientSettings { - const stateDir = req.flags?.stateDir ?? process.env.AGENT_DEVICE_STATE_DIR; - const remoteBaseUrl = resolveRemoteDaemonBaseUrl( - req.flags?.daemonBaseUrl ?? process.env.AGENT_DEVICE_DAEMON_BASE_URL, - ); + const explicitStateDir = req.flags?.stateDir ?? process.env.AGENT_DEVICE_STATE_DIR; + const rawRemoteBaseUrl = req.flags?.daemonBaseUrl ?? process.env.AGENT_DEVICE_DAEMON_BASE_URL; + const useOwnedReplayStateDir = + isOneShotReplayCommand(req.command) && !explicitStateDir && !rawRemoteBaseUrl; + const remoteBaseUrl = resolveRemoteDaemonBaseUrl(rawRemoteBaseUrl); const remoteAuthToken = req.flags?.daemonAuthToken ?? process.env.AGENT_DEVICE_DAEMON_AUTH_TOKEN; validateRemoteDaemonTrust(remoteBaseUrl, remoteAuthToken); const rawTransport = req.flags?.daemonTransport ?? process.env.AGENT_DEVICE_DAEMON_TRANSPORT; @@ -447,57 +479,69 @@ function resolveClientSettings(req: Omit): DaemonClientS process.env.AGENT_DEVICE_DAEMON_SERVER_MODE ?? (rawTransport === 'dual' ? 'dual' : undefined); const serverMode = resolveDaemonServerMode(rawServerMode); + const stateDir = useOwnedReplayStateDir + ? fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-daemon-')) + : explicitStateDir; return { paths: resolveDaemonPaths(stateDir), transportPreference, serverMode, + ownedStateDir: useOwnedReplayStateDir, remoteBaseUrl, remoteAuthToken, }; } -async function ensureDaemon(settings: DaemonClientSettings): Promise { +async function ensureDaemon(settings: DaemonClientSettings): Promise { if (settings.remoteBaseUrl) { - const remoteInfo: DaemonInfo = { - transport: 'http', - // Remote mode reuses the auth token as the daemon token so the existing JSON-RPC contract still works. - token: settings.remoteAuthToken ?? '', - pid: 0, - baseUrl: settings.remoteBaseUrl, - }; - if (await canConnect(remoteInfo, 'http')) return remoteInfo; - throw new AppError('COMMAND_FAILED', 'Remote daemon is unavailable', { - daemonBaseUrl: settings.remoteBaseUrl, - hint: 'Verify AGENT_DEVICE_DAEMON_BASE_URL points to a reachable daemon with GET /health and POST /rpc.', - }); + return await ensureRemoteDaemon(settings); } - const existing = readDaemonInfo(settings.paths.infoPath); - const localVersion = readVersion(); - const localCodeSignature = resolveLocalDaemonCodeSignature(); - const existingReachable = existing - ? await canConnect(existing, settings.transportPreference) - : false; - if ( - existing && - existing.version === localVersion && - existing.codeSignature === localCodeSignature && - existingReachable - ) { - return existing; - } - if ( - existing && - (existing.version !== localVersion || - existing.codeSignature !== localCodeSignature || - !existingReachable) - ) { - await stopDaemonProcessForTakeover(existing); - removeDaemonInfo(settings.paths.infoPath); - } + const reusable = await readReusableLocalDaemon(settings); + if (reusable) return { info: reusable, startedByClient: false }; cleanupStaleDaemonLockIfSafe(settings.paths); + return await startLocalDaemon(settings); +} + +async function ensureRemoteDaemon(settings: DaemonClientSettings): Promise { + const remoteInfo: DaemonInfo = { + transport: 'http', + // Remote mode reuses the auth token as the daemon token so the existing JSON-RPC contract still works. + token: settings.remoteAuthToken ?? '', + pid: 0, + baseUrl: settings.remoteBaseUrl, + }; + if (await canConnect(remoteInfo, 'http')) { + return { info: remoteInfo, startedByClient: false }; + } + throw new AppError('COMMAND_FAILED', 'Remote daemon is unavailable', { + daemonBaseUrl: settings.remoteBaseUrl, + hint: 'Verify AGENT_DEVICE_DAEMON_BASE_URL points to a reachable daemon with GET /health and POST /rpc.', + }); +} +async function readReusableLocalDaemon(settings: DaemonClientSettings): Promise { + const existing = readDaemonInfo(settings.paths.infoPath); + if (!existing) return null; + + const existingReachable = await canConnect(existing, settings.transportPreference); + if (isReusableDaemonInfo(existing, existingReachable)) return existing; + + await stopDaemonProcessForTakeover(existing); + removeDaemonInfo(settings.paths.infoPath); + return null; +} + +function isReusableDaemonInfo(info: DaemonInfo, reachable: boolean): boolean { + return ( + info.version === readVersion() && + info.codeSignature === resolveLocalDaemonCodeSignature() && + reachable + ); +} + +async function startLocalDaemon(settings: DaemonClientSettings): Promise { let lockRecoveryCount = 0; const cleanupResults: DaemonStartupCleanupResult[] = []; let startError: string | undefined; @@ -515,7 +559,7 @@ async function ensureDaemon(settings: DaemonClientSettings): Promise } const started = await waitForDaemonInfo(DAEMON_STARTUP_TIMEOUT_MS, settings); - if (started) return started; + if (started) return { info: started, startedByClient: true }; if (await recoverDaemonLockHolder(settings.paths)) { lockRecoveryCount += 1; @@ -530,7 +574,7 @@ async function ensureDaemon(settings: DaemonClientSettings): Promise cleanupResults.push(cleanup); if (cleanup.retainedInfoProcess || cleanup.retainedLockProcess) { const extended = await waitForDaemonInfo(DAEMON_STARTUP_TIMEOUT_MS, settings); - if (extended) return extended; + if (extended) return { info: extended, startedByClient: true }; break; } if (!hasAnotherAttempt) break; @@ -554,6 +598,57 @@ async function ensureDaemon(settings: DaemonClientSettings): Promise }); } +async function cleanupDaemonAfterRequest( + req: Omit, + daemon: EnsuredDaemon, + settings: DaemonClientSettings, +): Promise { + if ( + !isOneShotReplayCommand(req.command) || + (!daemon.startedByClient && !settings.ownedStateDir) || + isRemoteDaemon(daemon.info) + ) { + return; + } + + const result = { + pid: daemon.info.pid, + removedInfo: false, + removedLock: false, + removedStateDir: false, + error: undefined as string | undefined, + }; + + try { + await stopDaemonProcessForTakeover(daemon.info); + } catch (error) { + result.error = error instanceof Error ? error.message : String(error); + } finally { + const infoExists = fs.existsSync(settings.paths.infoPath); + removeDaemonInfo(settings.paths.infoPath); + result.removedInfo = infoExists && !fs.existsSync(settings.paths.infoPath); + + const lockExists = fs.existsSync(settings.paths.lockPath); + removeDaemonLock(settings.paths.lockPath); + result.removedLock = lockExists && !fs.existsSync(settings.paths.lockPath); + + if (settings.ownedStateDir) { + fs.rmSync(settings.paths.baseDir, { recursive: true, force: true }); + result.removedStateDir = !fs.existsSync(settings.paths.baseDir); + } + } + + emitDiagnostic({ + level: result.error ? 'warn' : 'info', + phase: 'daemon_replay_cleanup', + data: result, + }); +} + +function isOneShotReplayCommand(command: string | undefined): boolean { + return command === PUBLIC_COMMANDS.replay || command === PUBLIC_COMMANDS.test; +} + async function waitForDaemonInfo( timeoutMs: number, settings: DaemonClientSettings, @@ -594,32 +689,46 @@ function readDaemonInfo(infoPath: string): DaemonInfo | null { const data = readJsonFile(infoPath); if (!data || typeof data !== 'object') return null; const parsed = data as Partial; - const token = typeof parsed.token === 'string' && parsed.token.length > 0 ? parsed.token : null; + const token = readRequiredDaemonToken(parsed); if (!token) return null; - const hasSocket = Number.isInteger(parsed.port) && Number(parsed.port) > 0; - const hasHttp = Number.isInteger(parsed.httpPort) && Number(parsed.httpPort) > 0; - if (!hasSocket && !hasHttp) return null; - const transport = parsed.transport; - const version = typeof parsed.version === 'string' ? parsed.version : undefined; - const codeSignature = typeof parsed.codeSignature === 'string' ? parsed.codeSignature : undefined; - const processStartTime = - typeof parsed.processStartTime === 'string' ? parsed.processStartTime : undefined; - const hasPid = Number.isInteger(parsed.pid) && Number(parsed.pid) > 0; + const ports = readDaemonInfoPorts(parsed); + if (!ports) return null; return { token, - port: hasSocket ? Number(parsed.port) : undefined, - httpPort: hasHttp ? Number(parsed.httpPort) : undefined, - transport: - transport === 'socket' || transport === 'http' || transport === 'dual' - ? transport - : undefined, - pid: hasPid ? Number(parsed.pid) : 0, - version, - codeSignature, - processStartTime, + ...ports, + transport: readDaemonInfoTransport(parsed.transport), + pid: readPositiveInteger(parsed.pid) ?? 0, + version: readOptionalString(parsed.version), + codeSignature: readOptionalString(parsed.codeSignature), + processStartTime: readOptionalString(parsed.processStartTime), }; } +function readRequiredDaemonToken(parsed: Partial): string | null { + return typeof parsed.token === 'string' && parsed.token.length > 0 ? parsed.token : null; +} + +function readDaemonInfoPorts( + parsed: Partial, +): Pick | null { + const port = readPositiveInteger(parsed.port); + const httpPort = readPositiveInteger(parsed.httpPort); + if (port === undefined && httpPort === undefined) return null; + return { port, httpPort }; +} + +function readDaemonInfoTransport(value: unknown): DaemonInfo['transport'] { + return value === 'socket' || value === 'http' || value === 'dual' ? value : undefined; +} + +function readOptionalString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +function readPositiveInteger(value: unknown): number | undefined { + return Number.isInteger(value) && Number(value) > 0 ? Number(value) : undefined; +} + function readDaemonLockInfo(lockPath: string): DaemonLockInfo | null { const data = readJsonFile(lockPath); if (!data || typeof data !== 'object') return null; @@ -761,15 +870,25 @@ async function canConnect( return await canConnectSocket(info.port); } -function canConnectSocket(port: number | undefined): Promise { +export function canConnectSocket(port: number | undefined): Promise { if (!port) return Promise.resolve(false); return new Promise((resolve) => { + let settled = false; const socket = net.createConnection({ host: '127.0.0.1', port }, () => { + finish(true); + }); + const finish = (reachable: boolean) => { + if (settled) return; + settled = true; socket.destroy(); - resolve(true); + resolve(reachable); + }; + socket.setTimeout(LOCAL_DAEMON_HEALTHCHECK_TIMEOUT_MS); + socket.on('timeout', () => { + finish(false); }); socket.on('error', () => { - resolve(false); + finish(false); }); }); } @@ -1158,28 +1277,9 @@ function handleDaemonHttpResponseBody( ): void { const { info, req, resolve, reject } = options; try { - const parsed = JSON.parse(body) as { - result?: DaemonResponse; - error?: { - message?: string; - data?: Record; - }; - }; + const parsed = parseDaemonHttpResponseBody(body); if (parsed.error) { - const data = parsed.error.data ?? {}; - reject( - new AppError( - toAppErrorCode(data.code != null ? String(data.code) : undefined, 'COMMAND_FAILED'), - String(data.message ?? parsed.error.message ?? 'Daemon RPC request failed'), - { - ...(typeof data.details === 'object' && data.details ? data.details : {}), - hint: typeof data.hint === 'string' ? data.hint : undefined, - diagnosticId: typeof data.diagnosticId === 'string' ? data.diagnosticId : undefined, - logPath: typeof data.logPath === 'string' ? data.logPath : undefined, - requestId: req.meta?.requestId, - }, - ), - ); + reject(toDaemonHttpRpcError(parsed.error, req.meta?.requestId)); return; } if (!parsed.result || typeof parsed.result !== 'object') { @@ -1190,11 +1290,7 @@ function handleDaemonHttpResponseBody( ); return; } - if (info.baseUrl && parsed.result.ok) { - void materializeRemoteArtifacts(info, req, parsed.result).then(resolve).catch(reject); - return; - } - resolve(parsed.result); + void resolveDaemonHttpResult(info, req, parsed.result, resolve, reject); } catch (err) { reject( new AppError( @@ -1210,6 +1306,50 @@ function handleDaemonHttpResponseBody( } } +function parseDaemonHttpResponseBody(body: string): { + result?: DaemonResponse; + error?: { message?: string; data?: Record }; +} { + return JSON.parse(body) as { + result?: DaemonResponse; + error?: { message?: string; data?: Record }; + }; +} + +function toDaemonHttpRpcError( + error: { message?: string; data?: Record }, + requestId: string | undefined, +): AppError { + const data = error.data ?? {}; + return new AppError( + toAppErrorCode(data.code != null ? String(data.code) : undefined, 'COMMAND_FAILED'), + String(data.message ?? error.message ?? 'Daemon RPC request failed'), + { + ...(typeof data.details === 'object' && data.details ? data.details : {}), + hint: typeof data.hint === 'string' ? data.hint : undefined, + diagnosticId: typeof data.diagnosticId === 'string' ? data.diagnosticId : undefined, + logPath: typeof data.logPath === 'string' ? data.logPath : undefined, + requestId, + }, + ); +} + +async function resolveDaemonHttpResult( + info: DaemonInfo, + req: DaemonRequest, + result: DaemonResponse, + resolve: (response: DaemonResponse | PromiseLike) => void, + reject: (error: unknown) => void, +): Promise { + try { + resolve( + info.baseUrl && result.ok ? await materializeRemoteArtifacts(info, req, result) : result, + ); + } catch (error) { + reject(error); + } +} + function buildHttpRpcPayload( req: DaemonRequest, options: { includeTokenParam: boolean }, @@ -1506,13 +1646,13 @@ export async function downloadRemoteArtifact(params: { }, ); const timeoutHandle = setTimeout(() => { - request.destroy( - new AppError('COMMAND_FAILED', 'Remote artifact download timed out', { - artifactId: params.artifactId, - requestId: params.requestId, - timeoutMs, - }), - ); + const timeoutError = new AppError('COMMAND_FAILED', 'Remote artifact download timed out', { + artifactId: params.artifactId, + requestId: params.requestId, + timeoutMs, + }); + settle(timeoutError); + request.destroy(timeoutError); }, timeoutMs); request.on('error', (error) => { if (error instanceof AppError) { diff --git a/src/daemon/__tests__/post-gesture-stabilization.test.ts b/src/daemon/__tests__/post-gesture-stabilization.test.ts index a96b4a498..77270fb78 100644 --- a/src/daemon/__tests__/post-gesture-stabilization.test.ts +++ b/src/daemon/__tests__/post-gesture-stabilization.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { afterEach, test, vi } from 'vitest'; -import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; +import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; import type { SnapshotState } from '../../utils/snapshot.ts'; import { capturePostGestureStabilizedSnapshot, @@ -12,6 +12,22 @@ afterEach(() => { vi.useRealTimers(); }); +test('markPostGestureStabilization marks iOS swipe sessions', () => { + const session = makeSession(); + + markPostGestureStabilization(session, 'swipe'); + + assert.equal(session.postGestureStabilization?.action, 'swipe'); +}); + +test('markPostGestureStabilization marks Android swipe sessions', () => { + const session = makeSession('android'); + + markPostGestureStabilization(session, 'swipe'); + + assert.equal(session.postGestureStabilization?.action, 'swipe'); +}); + test('capturePostGestureStabilizedSnapshot retries until rects stop moving', async () => { vi.useFakeTimers(); const session = makeSession(); @@ -30,10 +46,10 @@ test('capturePostGestureStabilizedSnapshot retries until rects stop moving', asy assert.equal(session.postGestureStabilization, undefined); }); -function makeSession(): SessionState { +function makeSession(platform: 'ios' | 'android' = 'ios'): SessionState { return { - name: 'ios', - device: IOS_SIMULATOR, + name: platform, + device: platform === 'android' ? ANDROID_EMULATOR : IOS_SIMULATOR, createdAt: Date.now(), actions: [], }; diff --git a/src/daemon/__tests__/request-lock-policy.test.ts b/src/daemon/__tests__/request-lock-policy.test.ts index 03e5d6f41..d6b54505d 100644 --- a/src/daemon/__tests__/request-lock-policy.test.ts +++ b/src/daemon/__tests__/request-lock-policy.test.ts @@ -291,4 +291,3 @@ test('strips only conflicting selectors for existing sessions', () => { assert.equal(req.flags?.device, 'iPhone 16'); assert.equal(req.flags?.serial, undefined); }); - diff --git a/src/daemon/__tests__/selectors.test.ts b/src/daemon/__tests__/selectors.test.ts index 1d42dbdb1..2849f19df 100644 --- a/src/daemon/__tests__/selectors.test.ts +++ b/src/daemon/__tests__/selectors.test.ts @@ -290,4 +290,3 @@ test('appName selector matches nodes with appName field', () => { assert.ok(match3); assert.equal(match3.matches, 1); }); - diff --git a/src/daemon/__tests__/session-routing.test.ts b/src/daemon/__tests__/session-routing.test.ts index 7bb5daea0..660d6dfdf 100644 --- a/src/daemon/__tests__/session-routing.test.ts +++ b/src/daemon/__tests__/session-routing.test.ts @@ -47,4 +47,3 @@ test('reuses lone active session for implicit default session', (t) => { assert.equal(resolved, 'android'); }); - diff --git a/src/daemon/handlers/__tests__/interaction-flags.test.ts b/src/daemon/handlers/__tests__/interaction-flags.test.ts index 5644dc84b..c88395aa7 100644 --- a/src/daemon/handlers/__tests__/interaction-flags.test.ts +++ b/src/daemon/handlers/__tests__/interaction-flags.test.ts @@ -9,4 +9,3 @@ test('unsupportedRefSnapshotFlags returns unsupported snapshot flags for @ref fl }); expect(unsupported).toEqual(['--depth', '--scope', '--raw']); }); - diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index db0f9b06c..a4e594141 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -478,7 +478,7 @@ test('click simple iOS selector forwards Maestro non-hittable coordinate fallbac token: 't', session: sessionName, command: 'click', - positionals: ['id="e2eSignInAlice"'], + positionals: ['id="hiddenTestLogin"'], flags: { maestro: { allowNonHittableCoordinateFallback: true } }, }, sessionName, @@ -491,10 +491,15 @@ test('click simple iOS selector forwards Maestro non-hittable coordinate fallbac expect(pressCalls.length).toBe(1); expect((pressCalls[0]?.[4] as Record)?.directElementSelector).toEqual({ key: 'id', - value: 'e2eSignInAlice', - raw: 'id="e2eSignInAlice"', + value: 'hiddenTestLogin', + raw: 'id="hiddenTestLogin"', allowNonHittableCoordinateFallback: true, }); + if (response?.ok) { + expect(response.data?.maestroNonHittableCoordinateFallbackAllowed).toBe(true); + expect(response.data?.maestroNonHittableCoordinateFallbackUsed).toBe(true); + expect(response.data?.maestroFallbackReason).toBe('non-hittable-coordinate'); + } }); test('click simple iOS id selector falls back to snapshot coordinates when direct tap fails', async () => { diff --git a/src/daemon/handlers/__tests__/session-open-target.test.ts b/src/daemon/handlers/__tests__/session-open-target.test.ts index d82605ad3..b07c6185e 100644 --- a/src/daemon/handlers/__tests__/session-open-target.test.ts +++ b/src/daemon/handlers/__tests__/session-open-target.test.ts @@ -31,4 +31,3 @@ test('inferAndroidPackageAfterOpen reads foreground package for Android URL open inferAndroidPackageAfterOpen(androidDevice, 'exp://127.0.0.1:8082', undefined), ).resolves.toBe('host.exp.exponent'); }); - diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index 66f01ad0d..3872a22d7 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -4,6 +4,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { AppError } from '../../../utils/errors.ts'; +import { runCmdBackground, type ExecBackgroundResult } from '../../../utils/exec.ts'; import type { DaemonRequest, DaemonResponse, SessionAction } from '../../types.ts'; import type { CommandFlags } from '../../../core/dispatch.ts'; import { SessionStore } from '../../session-store.ts'; @@ -70,6 +71,34 @@ async function runReplayFixture(params: { return { response, calls, root, scriptPath }; } +async function readFirstStdoutLine(process: ExecBackgroundResult): Promise { + return await new Promise((resolve, reject) => { + let stdout = ''; + const cleanup = (): void => { + clearTimeout(timer); + process.child.stdout?.off('data', onData); + process.child.off('exit', onExit); + }; + const timer = setTimeout(() => { + cleanup(); + reject(new Error('Timed out waiting for child process stdout.')); + }, 5000); + const onData = (chunk: Buffer | string): void => { + stdout += String(chunk); + const lineEnd = stdout.indexOf('\n'); + if (lineEnd === -1) return; + cleanup(); + resolve(stdout.slice(0, lineEnd)); + }; + const onExit = (): void => { + cleanup(); + reject(new Error('Child process exited before writing stdout.')); + }; + process.child.stdout?.on('data', onData); + process.child.on('exit', onExit); + }); +} + test('resolveReplayString substitutes variables', () => { const scope = buildReplayVarScope({ fileEnv: { APP: 'settings' } }); assert.equal(resolveReplayString('open ${APP}', scope, LOC), 'open settings'); @@ -466,6 +495,94 @@ output.result = SERVER_PATH + ':' + json(res.body).appviewDid ); }); +test('runReplayScriptFile supports successful Maestro runScript http.post calls', async () => { + const serverScript = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-runscript-http-')), + 'server.cjs', + ); + fs.writeFileSync( + serverScript, + ` +const http = require('node:http'); +const server = http.createServer((req, res) => { + let body = ''; + req.setEncoding('utf8'); + req.on('data', chunk => { body += chunk; }); + req.on('end', () => { + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({method: req.method, body})); + }); +}); +server.listen(0, '127.0.0.1', () => { + process.stdout.write(String(server.address().port) + '\\n'); +}); +`, + ); + const server = runCmdBackground(process.execPath, [serverScript], { allowFailure: true }); + const port = await readFirstStdoutLine(server); + + try { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-http-post', + files: { + 'setup.js': ` +var res = http.post('http://127.0.0.1:${port}/setup', {body: '{"ok":true}'}) +var parsed = json(res.body) +output.result = parsed.method + ':' + json(parsed.body).ok +`, + }, + script: [ + 'appId: demo.app', + '---', + '- runScript: ./setup.js', + '- inputText: ${output.result}', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['type', ['POST:true']]], + ); + } finally { + server.child.kill(); + await server.wait.catch(() => undefined); + } +}); + +test('runReplayScriptFile strips prototype pollution keys from runScript json()', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-json-prototype-keys', + files: { + 'setup.js': ` +var parsed = json('{"safe":1,"__proto__":{"polluted":true},"constructor":{"polluted":true},"nested":{"prototype":{"polluted":true},"ok":2}}') +output.result = [ + Object.prototype.hasOwnProperty.call(parsed, '__proto__'), + Object.prototype.hasOwnProperty.call(parsed, 'constructor'), + Object.prototype.hasOwnProperty.call(parsed.nested, 'prototype'), + parsed.nested.ok +].join(':') +`, + }, + script: [ + 'appId: demo.app', + '---', + '- runScript: ./setup.js', + '- inputText: ${output.result}', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['type', ['false:false:false:2']]], + ); +}); + test('runReplayScriptFile reports Maestro runScript failures at the runScript step', async () => { const { response, calls } = await runReplayFixture({ label: 'maestro-runscript-fail', @@ -487,6 +604,27 @@ test('runReplayScriptFile reports Maestro runScript failures at the runScript st assert.equal(calls.length, 0); }); +test('runReplayScriptFile explains empty Maestro runScript JSON bodies', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-empty-json', + files: { + 'setup.js': `output.result = json('').value`, + }, + script: ['appId: demo.app', '---', '- runScript: ./setup.js', '- inputText: never', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /json\(\) received an empty body/); + assert.match(response.error.message, /setup server output/); + } + assert.equal(calls.length, 0); +}); + test('runReplayScriptFile rejects Maestro runScript output keys containing dots', async () => { const { response, calls } = await runReplayFixture({ label: 'maestro-runscript-dotted-output', @@ -591,11 +729,21 @@ test('runReplayScriptFile lets Maestro tapOn use fuzzy visible text matching', a flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'find') return { ok: true, data: { found: true } }; - return { - ok: false, - error: { code: 'COMMAND_FAILED', message: 'Selector did not match' }, - }; + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Discover people', + rect: { x: 10, y: 600, width: 240, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; }, }); @@ -604,11 +752,62 @@ test('runReplayScriptFile lets Maestro tapOn use fuzzy visible text matching', a calls.map((call) => [call.command, call.positionals]), [ ['snapshot', []], - ['find', ['Discover', 'click']], + ['click', ['130', '622']], ], ); assert.equal(calls[0]?.flags?.noRecord, true); - assert.equal(calls[1]?.flags?.findFirst, true); +}); + +test('runReplayScriptFile reuses successful Maestro visibility snapshot for following tapOn', async () => { + let snapshots = 0; + const { response, calls } = await runReplayFixture({ + label: 'maestro-assert-visible-tap-cache', + script: ['appId: demo.app', '---', '- assertVisible: Open feed', '- tapOn: Open feed', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro', platform: 'android' }, + invoke: async (req) => { + if (req.command === 'snapshot') { + snapshots += 1; + return { + ok: true, + data: { + nodes: + snapshots === 1 + ? [ + { + index: 1, + label: 'Article', + rect: { x: 10, y: 100, width: 160, height: 44 }, + }, + { + index: 2, + label: 'Open feed', + rect: { x: 20, y: 180, width: 180, height: 48 }, + }, + ] + : [ + { + index: 1, + label: 'AppStack.tsx (42:7)', + rect: { x: 28, y: 1304, width: 1025, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['110', '204']], + ], + ); }); test('runReplayScriptFile lets exact Maestro text tapOn win before fuzzy matching', async () => { @@ -652,6 +851,241 @@ test('runReplayScriptFile lets exact Maestro text tapOn win before fuzzy matchin ); }); +test('runReplayScriptFile prefers exact Maestro text before actionable target type', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-exact-before-type-rank', + script: ['appId: demo.app', '---', '- tapOn: Search', ''].join('\n'), + flags: { platform: 'android', replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'android.widget.Button', + label: 'search', + rect: { x: 673, y: 165, width: 132, height: 132 }, + }, + { + index: 2, + type: 'android.widget.FrameLayout', + label: 'Search', + rect: { x: 810, y: 2054, width: 270, height: 220 }, + }, + ], + metadata: { referenceWidth: 1080, referenceHeight: 2340 }, + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['945', '2164']], + ], + ); +}); + +test('runReplayScriptFile prefers fuller same-type Maestro text matches', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-prefers-fuller-same-type', + script: ['appId: demo.app', '---', '- tapOn: Open details', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'android.widget.Button', + label: 'Open details', + rect: { x: 0, y: 352, width: 109, height: 110 }, + }, + { + index: 2, + type: 'android.widget.Button', + label: 'Open details', + rect: { x: 33, y: 433, width: 340, height: 110 }, + }, + ], + metadata: { referenceWidth: 1080, referenceHeight: 2340 }, + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['203', '488']], + ], + ); +}); + +test('runReplayScriptFile ignores zero-size Maestro tapOn matches', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-ignores-zero-size', + script: ['appId: demo.app', '---', '- tapOn: Retain', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'android.widget.ScrollView', + rect: { x: 0, y: 319, width: 816, height: 2021 }, + }, + { + index: 2, + parentIndex: 1, + type: 'android.widget.Button', + label: 'Retain', + rect: { x: 0, y: 2340, width: 771, height: 0 }, + }, + { + index: 3, + parentIndex: 2, + type: 'android.widget.TextView', + label: 'Retain', + }, + { + index: 4, + type: 'android.widget.Button', + label: 'Retain', + rect: { x: 418, y: 1385, width: 244, height: 110 }, + }, + ], + metadata: { referenceWidth: 1080, referenceHeight: 2340 }, + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['540', '1440']], + ], + ); +}); + +test('runReplayScriptFile prefers own-rect Maestro tapOn matches over inherited overlay rects', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-prefers-own-rect', + script: ['appId: demo.app', '---', '- tapOn: Library', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'android.view.View', + label: 'Library', + rect: { x: 585, y: 319, width: 222, height: 132 }, + }, + { + index: 2, + type: 'android.view.ViewGroup', + rect: { x: 0, y: 0, width: 816, height: 2340 }, + }, + { + index: 3, + parentIndex: 2, + type: 'android.view.ViewGroup', + }, + { + index: 4, + parentIndex: 3, + type: 'android.widget.TextView', + label: 'Library', + }, + ], + metadata: { referenceWidth: 1080, referenceHeight: 2340 }, + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['696', '385']], + ], + ); +}); + +test('runReplayScriptFile lets Maestro text tapOn match regex labels', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-regex', + script: ['appId: demo.app', '---', "- tapOn: '.*Featured.*'", ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'button', + label: 'Featured, selected tab', + rect: { x: 20, y: 720, width: 110, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['75', '742']], + ], + ); +}); + test('runReplayScriptFile prefers on-screen Maestro text tapOn matches', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ @@ -787,13 +1221,17 @@ test('runReplayScriptFile treats absent Maestro assertNotVisible targets as pass const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ label: 'maestro-assert-not-visible-absent', - script: ['appId: demo.app', '---', '- assertNotVisible: Feeds ✨', ''].join('\n'), + script: ['appId: demo.app', '---', '- assertNotVisible: Archived banner', ''].join('\n'), flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); return { ok: false, - error: { code: 'ELEMENT_NOT_FOUND', message: 'Selector did not match' }, + error: { + code: 'COMMAND_FAILED', + message: 'Selector did not match', + details: { command: 'is', reason: 'selector_not_found' }, + }, }; }, }); @@ -801,39 +1239,158 @@ test('runReplayScriptFile treats absent Maestro assertNotVisible targets as pass assert.equal(response.ok, true); assert.deepEqual( calls.map((call) => [call.command, call.positionals]), - [['is', ['visible', 'label="Feeds ✨" || text="Feeds ✨" || id="Feeds ✨"']]], + [ + [ + 'is', + ['visible', 'label="Archived banner" || text="Archived banner" || id="Archived banner"'], + ], + [ + 'is', + ['visible', 'label="Archived banner" || text="Archived banner" || id="Archived banner"'], + ], + ], ); assert.equal(calls[0]?.flags?.noRecord, true); }); -test('runReplayScriptFile retries Maestro fuzzy tapOn without raw selector fallback', async () => { +test('runReplayScriptFile propagates Maestro assertNotVisible infrastructure failures', async () => { const calls: CapturedInvocation[] = []; - let findAttempts = 0; const { response } = await runReplayFixture({ - label: 'maestro-tap-visible-text-fuzzy-retry', - script: ['appId: demo.app', '---', '- tapOn: Discover', ''].join('\n'), + label: 'maestro-assert-not-visible-infra-fail', + script: ['appId: demo.app', '---', '- assertNotVisible: Archived banner', ''].join('\n'), flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'find') { - findAttempts += 1; - if (findAttempts === 2) return { ok: true, data: { found: true } }; + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'Snapshot capture failed' }, + }; + }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /Snapshot capture failed/); + } + assert.equal(calls.length, 1); +}); + +test('runReplayScriptFile treats duplicate visible Maestro assertVisible matches as passing', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-assert-visible-duplicate-visible', + script: ['appId: demo.app', '---', '- assertVisible: Article', ''].join('\n'), + flags: { platform: 'android', replayBackend: 'maestro' }, + invoke: async (req) => { + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'statictext', + label: 'Article', + rect: { x: 16, y: 100, width: 120, height: 24 }, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'statictext', + label: 'Article', + rect: { x: 16, y: 140, width: 120, height: 24 }, + }, + ], + }, + }; } return { ok: false, - error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + error: { code: 'COMMAND_FAILED', message: 'unexpected command' }, }; }, }); + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['snapshot', []]], + ); + assert.equal(calls[0]?.flags?.snapshotRaw, true); +}); + +test('runReplayScriptFile waits briefly for Maestro assertNotVisible to stabilize', async () => { + const calls: CapturedInvocation[] = []; + let visibleChecks = 0; + const { response } = await runReplayFixture({ + label: 'maestro-assert-not-visible-stable', + script: ['appId: demo.app', '---', '- assertNotVisible: Archived banner', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + visibleChecks += 1; + if (visibleChecks === 1) return { ok: true, data: { pass: true } }; + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: 'is visible failed', + details: { command: 'is', reason: 'predicate_failed' }, + }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.equal(calls.length, 3); +}); + +test('runReplayScriptFile retries Maestro fuzzy tapOn without raw selector fallback', async () => { + const calls: CapturedInvocation[] = []; + let snapshotAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-fuzzy-retry', + script: ['appId: demo.app', '---', '- tapOn: Discover', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + snapshotAttempts += 1; + return { + ok: true, + data: { + nodes: + snapshotAttempts === 1 + ? [] + : [ + { + index: 1, + label: 'Discover people', + rect: { x: 10, y: 600, width: 240, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + assert.equal(response.ok, true); assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ ['snapshot', []], - ['find', ['Discover', 'click']], ['snapshot', []], - ['find', ['Discover', 'click']], + ['click', ['130', '622']], ], ); }); @@ -853,13 +1410,21 @@ test('runReplayScriptFile lets optional Maestro fuzzy tapOn click first visible flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'find' && req.flags?.findFirst === true) { - return { ok: true, data: { ref: '@e4', x: 220, y: 720 } }; + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Maybe Later', + rect: { x: 100, y: 700, width: 240, height: 44 }, + }, + ], + }, + }; } - return { - ok: false, - error: { code: 'AMBIGUOUS_MATCH', message: 'matched multiple elements' }, - }; + return { ok: true, data: {} }; }, }); @@ -868,10 +1433,9 @@ test('runReplayScriptFile lets optional Maestro fuzzy tapOn click first visible calls.map((call) => [call.command, call.positionals]), [ ['snapshot', []], - ['find', ['Later', 'click']], + ['click', ['220', '722']], ], ); - assert.equal(calls[1]?.flags?.findFirst, true); }); test('runReplayScriptFile resolves Maestro percentage point taps from snapshot size', async () => { @@ -969,11 +1533,11 @@ test('runReplayScriptFile resolves Maestro tapOn index and childOf from snapshot 'appId: demo.app', '---', '- tapOn:', - ' id: likeBtn', + ' id: childActionButton', ' childOf:', - ' id: postThreadItem-by-bob.test', + ' id: parent-row-secondary', '- tapOn:', - ' id: postDropdownBtn', + ' id: overflowButton', ' index: 1', '', ].join('\n'), @@ -985,28 +1549,28 @@ test('runReplayScriptFile resolves Maestro tapOn index and childOf from snapshot ok: true, data: { nodes: [ - { index: 1, identifier: 'postThreadItem-by-alice.test' }, + { index: 1, identifier: 'parent-row-primary' }, { index: 2, parentIndex: 1, - identifier: 'likeBtn', + identifier: 'childActionButton', rect: { x: 10, y: 10, width: 40, height: 20 }, }, - { index: 10, identifier: 'postThreadItem-by-bob.test' }, + { index: 10, identifier: 'parent-row-secondary' }, { index: 11, parentIndex: 10, - identifier: 'likeBtn', + identifier: 'childActionButton', rect: { x: 20, y: 120, width: 40, height: 20 }, }, { index: 20, - identifier: 'postDropdownBtn', + identifier: 'overflowButton', rect: { x: 100, y: 200, width: 40, height: 20 }, }, { index: 21, - identifier: 'postDropdownBtn', + identifier: 'overflowButton', rect: { x: 200, y: 300, width: 40, height: 20 }, }, ], @@ -1034,7 +1598,7 @@ test('runReplayScriptFile lets snapshot id tap handle Maestro one-point edge con const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ label: 'maestro-tap-edge-rect', - script: ['appId: demo.app', '---', '- tapOn:', ' id: e2eSignInAlice', ''].join('\n'), + script: ['appId: demo.app', '---', '- tapOn:', ' id: hiddenTestLogin', ''].join('\n'), flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); @@ -1045,7 +1609,7 @@ test('runReplayScriptFile lets snapshot id tap handle Maestro one-point edge con nodes: [ { index: 1, - identifier: 'e2eSignInAlice', + identifier: 'hiddenTestLogin', rect: { x: 0, y: 0, width: 1, height: 1 }, }, ], @@ -1074,8 +1638,8 @@ test('runReplayScriptFile keeps Maestro text-entry tapOn on the snapshot path', 'appId: demo.app', '---', '- tapOn:', - ' id: editListNameInput', - '- inputText: Muted Users', + ' id: editableNameInput', + '- inputText: Saved list', '', ].join('\n'), flags: { replayBackend: 'maestro' }, @@ -1088,7 +1652,7 @@ test('runReplayScriptFile keeps Maestro text-entry tapOn on the snapshot path', nodes: [ { index: 1, - identifier: 'editListNameInput', + identifier: 'editableNameInput', rect: { x: 20, y: 100, width: 200, height: 40 }, }, ], @@ -1105,7 +1669,7 @@ test('runReplayScriptFile keeps Maestro text-entry tapOn on the snapshot path', [ ['snapshot', []], ['click', ['120', '120']], - ['type', ['Muted Users']], + ['type', ['Saved list']], ], ); assert.equal(calls[0]?.flags?.noRecord, true); @@ -1155,6 +1719,54 @@ test('runReplayScriptFile resolves Maestro swipe.label from a labeled element re ); }); +test('runReplayScriptFile resolves Maestro screen swipes from the snapshot frame', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-screen-swipe', + script: [ + 'appId: demo.app', + '---', + '- swipe:', + ' direction: LEFT', + ' duration: 300', + '- swipe:', + ' start: 90%,50%', + ' end: 10%,50%', + ' duration: 300', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['swipe', ['320', '400', '80', '400', '300']], + ['swipe', ['360', '400', '40', '400', '300']], + ], + ); +}); + test('runReplayScriptFile maps Maestro enter to keyboard enter', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ @@ -1254,6 +1866,20 @@ test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absen flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + ], + }, + }; + } return { ok: false, error: { @@ -1268,8 +1894,71 @@ test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absen assert.equal(response.ok, true); assert.deepEqual( calls.map((call) => [call.command, call.positionals]), - [['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']]], + [['snapshot', []]], + ); +}); + +test('runReplayScriptFile retries Maestro retry commands until they pass', async () => { + const calls: CapturedInvocation[] = []; + let openAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-retry', + script: [ + 'appId: demo.app', + '---', + '- retry:', + ' maxRetries: 2', + ' commands:', + ' - openLink:', + ' link: demo://details', + ' - extendedWaitUntil:', + ' visible: Article', + ' timeout: 1', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'open') openAttempts += 1; + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + ...(openAttempts > 1 + ? [ + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'statictext', + label: 'Article', + rect: { x: 16, y: 100, width: 120, height: 24 }, + }, + ] + : []), + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.filter((call) => call.command === 'open').map((call) => [call.command, call.positionals]), + [ + ['open', ['demo://details']], + ['open', ['demo://details']], + ], ); + assert.equal(calls.filter((call) => call.command === 'snapshot').length > 1, true); }); test('runReplayScriptFile skips Maestro runFlow.when.visible commands on false predicate', async () => { @@ -1289,6 +1978,20 @@ test('runReplayScriptFile skips Maestro runFlow.when.visible commands on false p flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + ], + }, + }; + } return { ok: true, data: { pass: false } }; }, }); @@ -1296,7 +1999,7 @@ test('runReplayScriptFile skips Maestro runFlow.when.visible commands on false p assert.equal(response.ok, true); assert.deepEqual( calls.map((call) => [call.command, call.positionals]), - [['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']]], + [['snapshot', []]], ); }); @@ -1371,7 +2074,28 @@ test('runReplayScriptFile runs Maestro runFlow.when.visible commands when presen flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'is') return { ok: true, data: { pass: true } }; + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'button', + label: 'Continue', + rect: { x: 16, y: 100, width: 120, height: 44 }, + }, + ], + }, + }; + } if (req.command === 'click') { return { ok: false, @@ -1386,8 +2110,9 @@ test('runReplayScriptFile runs Maestro runFlow.when.visible commands when presen assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ - ['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']], ['snapshot', []], + ['snapshot', []], + ['click', ['76', '122']], ['find', ['Continue', 'click']], ], ); @@ -1413,7 +2138,28 @@ test('runReplayScriptFile runs nested Maestro runtime commands inside runFlow.wh flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'is') return { ok: true, data: { pass: true } }; + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'statictext', + label: 'Feed', + rect: { x: 16, y: 100, width: 120, height: 24 }, + }, + ], + }, + }; + } if (req.command === 'wait') return { ok: true, data: { found: true } }; return { ok: true, data: {} }; }, @@ -1423,7 +2169,7 @@ test('runReplayScriptFile runs nested Maestro runtime commands inside runFlow.wh assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ - ['is', ['visible', 'label="Feed" || text="Feed" || id="Feed"']], + ['snapshot', []], ['wait', ['label="Done" || text="Done" || id="Done"', '500']], ], ); diff --git a/src/daemon/handlers/__tests__/session-test-discovery.test.ts b/src/daemon/handlers/__tests__/session-test-discovery.test.ts index dfc4ffa6a..f0c572537 100644 --- a/src/daemon/handlers/__tests__/session-test-discovery.test.ts +++ b/src/daemon/handlers/__tests__/session-test-discovery.test.ts @@ -51,7 +51,29 @@ test('discoverReplayTestEntries rejects empty post-filter suites', () => { (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS' && - error.message === 'No .ad tests matched for --platform android.', + error.message === 'No replay tests matched for --platform android.', ); }); +test('discoverReplayTestEntries includes Maestro yaml flows for Maestro test suites', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-maestro-')); + fs.writeFileSync(path.join(root, '01-flow.yaml'), 'appId: demo\n---\n- launchApp\n'); + fs.writeFileSync(path.join(root, '02-flow.yml'), 'appId: demo\n---\n- launchApp\n'); + fs.writeFileSync(path.join(root, '03-flow.ad'), 'open "Demo"\n'); + + const entries = discoverReplayTestEntries({ + inputs: [root], + cwd: root, + platformFilter: 'android', + replayBackend: 'maestro', + }); + + assert.deepEqual( + entries.map((entry) => path.basename(entry.path)), + ['01-flow.yaml', '02-flow.yml', '03-flow.ad'], + ); + assert.deepEqual( + entries.map((entry) => entry.kind), + ['run', 'run', 'run'], + ); +}); diff --git a/src/daemon/handlers/__tests__/session-test-runtime.test.ts b/src/daemon/handlers/__tests__/session-test-runtime.test.ts index a9bbd969d..e9e72ea76 100644 --- a/src/daemon/handlers/__tests__/session-test-runtime.test.ts +++ b/src/daemon/handlers/__tests__/session-test-runtime.test.ts @@ -47,4 +47,3 @@ test('runReplayTestAttempt keeps cancellation active until a timed-out replay se expect(isRequestCanceled('req-timeout-open')).toBe(false); }); - diff --git a/src/daemon/handlers/__tests__/session.test.ts b/src/daemon/handlers/__tests__/session.test.ts index 9773e19ed..61d5470ac 100644 --- a/src/daemon/handlers/__tests__/session.test.ts +++ b/src/daemon/handlers/__tests__/session.test.ts @@ -4322,7 +4322,7 @@ test('test returns invalid args when no replay scripts match the platform filter invoke: noopInvoke, }); - assertInvalidArgsMessage(response, 'No .ad tests matched for --platform android.'); + assertInvalidArgsMessage(response, 'No replay tests matched for --platform android.'); }); test('test rejects duplicate replay test metadata in the context header', async () => { diff --git a/src/daemon/handlers/interaction-touch-targets.ts b/src/daemon/handlers/interaction-touch-targets.ts index d682531c0..b22beb409 100644 --- a/src/daemon/handlers/interaction-touch-targets.ts +++ b/src/daemon/handlers/interaction-touch-targets.ts @@ -110,6 +110,8 @@ export function parseFillTarget(positionals: string[]): ParsedFillTarget { ), }; } + // Preserve payload whitespace (for example Maestro/keyboard-enter newlines) + // while still rejecting selector fills that contain only whitespace. if (!parsed.text.trim()) { return { ok: false, diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index a82003772..8aa6df4ef 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -332,7 +332,10 @@ async function dispatchDirectIosSelectorInteraction(params: { fallbackX: point.x, fallbackY: point.y, referenceFrame: readReferenceFrameFromDirectSelectorTapResult(data), - extra, + extra: { + ...extra, + ...directIosSelectorFallbackDetails(selector, data), + }, }); return finalizeTouchInteraction({ session, @@ -361,6 +364,19 @@ async function dispatchDirectIosSelectorInteraction(params: { } } +function directIosSelectorFallbackDetails( + selector: DirectIosSelectorTarget, + data: Record, +): Record { + if (!selector.allowNonHittableCoordinateFallback) return {}; + const used = data.message === 'tapped via non-hittable coordinate fallback'; + return { + maestroNonHittableCoordinateFallbackAllowed: true, + maestroNonHittableCoordinateFallbackUsed: used, + ...(used ? { maestroFallbackReason: 'non-hittable-coordinate' } : {}), + }; +} + function readPointFromDirectSelectorTapResult(data: Record): { x: number; y: number; diff --git a/src/daemon/handlers/session-close.ts b/src/daemon/handlers/session-close.ts index b03c6ee64..3f1c1037a 100644 --- a/src/daemon/handlers/session-close.ts +++ b/src/daemon/handlers/session-close.ts @@ -17,6 +17,7 @@ import { IOS_SIMULATOR_POST_CLOSE_SETTLE_MS, isAndroidEmulator, isIosSimulator, + resolveCommandDevice, settleIosSimulator, } from './session-device-utils.ts'; import { errorResponse } from './response.ts'; @@ -123,7 +124,7 @@ export async function handleCloseCommand(params: { const { req, sessionName, logPath, sessionStore } = params; const session = sessionStore.get(sessionName); if (!session) { - return errorResponse('SESSION_NOT_FOUND', 'No active session'); + return await closeWithoutSession(req, logPath); } if (session.appLog) { await stopAppLog(session.appLog); @@ -188,3 +189,25 @@ export async function handleCloseCommand(params: { } return { ok: true, data: { session: sessionName, ...successText(`Closed: ${sessionName}`) } }; } + +async function closeWithoutSession(req: DaemonRequest, logPath: string): Promise { + if (!req.positionals || req.positionals.length === 0) { + return errorResponse('SESSION_NOT_FOUND', 'No active session'); + } + const device = await resolveCommandDevice({ + session: undefined, + flags: req.flags, + ensureReady: true, + }); + await dispatchCommand(device, 'close', req.positionals, req.flags?.out, { + ...contextFromFlags(logPath, req.flags), + }); + await settleIosSimulator(device, IOS_SIMULATOR_POST_CLOSE_SETTLE_MS); + return { + ok: true, + data: { + app: req.positionals[0], + ...successText(`Closed: ${req.positionals[0]}`), + }, + }; +} diff --git a/src/daemon/handlers/session-replay-action-runtime.ts b/src/daemon/handlers/session-replay-action-runtime.ts index 6ad7e658e..3ab0868df 100644 --- a/src/daemon/handlers/session-replay-action-runtime.ts +++ b/src/daemon/handlers/session-replay-action-runtime.ts @@ -7,7 +7,7 @@ import { } from '../../replay/vars.ts'; import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; import { mergeParentFlags } from './handler-utils.ts'; -import { invokeMaestroRuntimeCommand } from './session-replay-maestro-runtime.ts'; +import { invokeMaestroRuntimeCommand } from '../../compat/maestro/runtime.ts'; type ReplayBaseRequest = Omit; @@ -140,7 +140,7 @@ function appendReplayTraceEvent( fs.appendFileSync(tracePath, `${JSON.stringify(event)}\n`); } -export function buildReplayActionFlags( +function buildReplayActionFlags( parentFlags: CommandFlags | undefined, actionFlags: SessionAction['flags'] | undefined, ): CommandFlags { diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts deleted file mode 100644 index 8ec68ae15..000000000 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ /dev/null @@ -1,976 +0,0 @@ -import { type CommandFlags } from '../../core/dispatch.ts'; -import { MAESTRO_RUNTIME_COMMAND } from '../../compat/maestro/runtime-commands.ts'; -import { executeRunScriptFile } from '../../compat/maestro/run-script.ts'; -import type { Platform } from '../../utils/device.ts'; -import { type Rect, type SnapshotNode, type SnapshotState } from '../../utils/snapshot.ts'; -import { sleep } from '../../utils/timeouts.ts'; -import { asAppError } from '../../utils/errors.ts'; -import type { ReplayVarScope } from '../../replay/vars.ts'; -import { parseSelectorChain } from '../selectors.ts'; -import { matchesSelector } from '../selectors-match.ts'; -import { getSnapshotReferenceFrame, type TouchReferenceFrame } from '../touch-reference-frame.ts'; -import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; -import { errorResponse } from './response.ts'; - -// Keep Maestro timing and target-selection heuristics behind one policy so -// generic Agent Device command behavior does not inherit compatibility rules. -const MAESTRO_REPLAY_POLICY = { - animationPollMs: 250, - scrollUntilVisibleProbeMs: 500, - tapOnRetryMs: 250, - tapOnTimeoutMs: 30000, - optionalTapOnTimeoutMs: 3000, - swipe: { - screenRatio: 0.35, - minDistancePx: 120, - maxDistancePx: 360, - marginPx: 8, - }, - largeTextContainerBias: { - minWidth: 120, - minHeight: 70, - width: 168, - height: 48, - }, -} as const; - -const MAESTRO_TAP_TARGET_TYPE_RANK = new Map([ - ['button', 0], - ['link', 0], - ['textfield', 0], - ['textview', 0], - ['searchfield', 0], - ['switch', 0], - ['slider', 0], - ['cell', 1], - ['statictext', 2], -]); - -type ReplayBaseRequest = Omit; - -type MaestroReplayInvoker = (params: { - action: SessionAction; - line: number; - step: number; -}) => Promise; - -type MaestroRuntimeInvoke = (req: DaemonRequest) => Promise; -type FailedDaemonResponse = Extract; - -type MaestroScrollUntilVisibleParams = { - baseReq: ReplayBaseRequest; - positionals: string[]; - invoke: MaestroRuntimeInvoke; -}; - -type MaestroTapOnParams = { - baseReq: ReplayBaseRequest; - positionals: string[]; - invoke: MaestroRuntimeInvoke; -}; - -type MaestroTapOnOptions = { - childOf?: string; - index?: number; -}; - -type MaestroRunFlowWhenCondition = - | { ok: true; mode: string; predicate: string; selector: string } - | { ok: false; response: DaemonResponse }; - -type MaestroSnapshotTarget = { - node: SnapshotNode; - rect: Rect; - frame?: TouchReferenceFrame; -}; - -export async function invokeMaestroRuntimeCommand(params: { - command: string; - baseReq: ReplayBaseRequest; - positionals: string[]; - batchSteps: CommandFlags['batchSteps'] | undefined; - scope: ReplayVarScope; - line: number; - step: number; - invoke: (req: DaemonRequest) => Promise; - invokeReplayAction: MaestroReplayInvoker; -}): Promise { - switch (params.command) { - case MAESTRO_RUNTIME_COMMAND.assertNotVisible: - return await invokeMaestroAssertNotVisible(params); - case MAESTRO_RUNTIME_COMMAND.pressEnter: - return await invokeMaestroPressEnter(params); - case MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd: - return await invokeMaestroWaitForAnimationToEnd(params); - case MAESTRO_RUNTIME_COMMAND.scrollUntilVisible: - return await invokeMaestroScrollUntilVisible(params); - case MAESTRO_RUNTIME_COMMAND.swipeOn: - return await invokeMaestroSwipeOn(params); - case MAESTRO_RUNTIME_COMMAND.tapOn: - return await invokeMaestroTapOn(params); - case MAESTRO_RUNTIME_COMMAND.tapPointPercent: - return await invokeMaestroTapPointPercent(params); - case MAESTRO_RUNTIME_COMMAND.runFlowWhen: - return await invokeMaestroRunFlowWhen(params); - case MAESTRO_RUNTIME_COMMAND.runScript: - return invokeMaestroRunScript(params); - default: - return undefined; - } -} - -async function invokeMaestroPressEnter(params: { - baseReq: ReplayBaseRequest; - invoke: MaestroRuntimeInvoke; -}): Promise { - const keyboardResponse = await params.invoke({ - ...params.baseReq, - command: 'keyboard', - positionals: ['enter'], - }); - if (keyboardResponse.ok) return keyboardResponse; - - return await params.invoke({ - ...params.baseReq, - command: 'type', - positionals: ['\n'], - }); -} - -async function invokeMaestroAssertNotVisible(params: { - baseReq: ReplayBaseRequest; - positionals: string[]; - invoke: MaestroRuntimeInvoke; -}): Promise { - const [selector] = params.positionals; - if (!selector) { - return errorResponse('INVALID_ARGS', 'assertNotVisible requires a selector.'); - } - const response = await params.invoke({ - ...params.baseReq, - command: 'is', - positionals: ['visible', selector], - flags: { ...params.baseReq.flags, noRecord: true }, - }); - if (!response.ok) { - return { ok: true, data: { pass: true, selector, absent: true } }; - } - if (response.data?.pass === false) { - return { ok: true, data: { pass: true, selector } }; - } - return errorResponse('COMMAND_FAILED', `Expected not visible but matched: ${selector}`); -} - -async function invokeMaestroWaitForAnimationToEnd(params: { - baseReq: ReplayBaseRequest; - positionals: string[]; - invoke: MaestroRuntimeInvoke; -}): Promise { - const timeoutMs = Number(params.positionals[0] ?? 15000); - if (!Number.isFinite(timeoutMs) || timeoutMs < 0) { - return errorResponse('INVALID_ARGS', 'waitForAnimationToEnd timeout must be a number.'); - } - const startedAt = Date.now(); - let previousSignature: string | undefined; - let lastResponse: DaemonResponse | undefined; - - while (Date.now() - startedAt < timeoutMs) { - const response = await captureMaestroRawSnapshot(params); - const poll = readAnimationPollResult(response, previousSignature, timeoutMs); - if (poll.done) return poll.response; - previousSignature = poll.signature ?? previousSignature; - lastResponse = response; - await sleep(MAESTRO_REPLAY_POLICY.animationPollMs); - } - - return lastResponse?.ok === false - ? lastResponse - : { ok: true, data: { stable: false, timeoutMs } }; -} - -function readAnimationPollResult( - response: DaemonResponse, - previousSignature: string | undefined, - timeoutMs: number, -): { done: true; response: DaemonResponse } | { done: false; signature?: string } { - const signature = readSnapshotStabilitySignature(response); - if (!response.ok) return { done: false }; - if (!signature) return { done: true, response }; - if (previousSignature === signature) { - return { done: true, response: { ok: true, data: { stable: true, timeoutMs } } }; - } - return { done: false, signature }; -} - -async function captureMaestroRawSnapshot(params: { - baseReq: ReplayBaseRequest; - invoke: MaestroRuntimeInvoke; -}): Promise { - return await params.invoke({ - ...params.baseReq, - command: 'snapshot', - positionals: [], - flags: { - ...params.baseReq.flags, - noRecord: true, - snapshotRaw: true, - snapshotForceFull: true, - }, - }); -} - -function readSnapshotStabilitySignature(response: DaemonResponse): string | null { - if (!response.ok) return null; - const snapshot = readSnapshotState(response.data); - return snapshot ? snapshotStabilitySignature(snapshot) : null; -} - -async function invokeMaestroScrollUntilVisible( - params: MaestroScrollUntilVisibleParams, -): Promise { - const [selector, timeoutValue = '5000', direction = 'down'] = params.positionals; - if (!selector) { - return errorResponse('INVALID_ARGS', 'scrollUntilVisible requires a selector.'); - } - const timeoutMs = Number(timeoutValue); - if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { - return errorResponse('INVALID_ARGS', 'scrollUntilVisible timeout must be a positive number.'); - } - const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); - const attempts = Math.max( - 1, - Math.ceil(timeoutMs / MAESTRO_REPLAY_POLICY.scrollUntilVisibleProbeMs), - ); - let lastWaitResponse: FailedDaemonResponse | null = null; - - for (let index = 0; index < attempts; index += 1) { - const probeResponse = await probeMaestroScrollVisibility( - params, - selector, - fuzzyTextQuery, - scrollProbeMs(timeoutMs, index), - ); - if (probeResponse.ok) return probeResponse; - lastWaitResponse = probeResponse; - - if (index === attempts - 1) break; - - const scrollResponse = await params.invoke({ - ...params.baseReq, - command: 'scroll', - positionals: [direction], - }); - if (!scrollResponse.ok) return scrollResponse; - } - - return withMaestroScrollTimeoutContext(lastWaitResponse, selector, timeoutMs); -} - -async function probeMaestroScrollVisibility( - params: MaestroScrollUntilVisibleParams, - selector: string, - fuzzyTextQuery: string | null, - probeMs: number, -): Promise { - const waitResponse = await params.invoke({ - ...params.baseReq, - command: 'wait', - positionals: [selector, String(probeMs)], - }); - if (waitResponse.ok || !fuzzyTextQuery) return waitResponse; - - const fuzzyResponse = await params.invoke({ - ...params.baseReq, - command: 'find', - positionals: [fuzzyTextQuery, 'wait', String(probeMs)], - }); - return fuzzyResponse; -} - -function scrollProbeMs(timeoutMs: number, index: number): number { - return Math.min( - MAESTRO_REPLAY_POLICY.scrollUntilVisibleProbeMs, - Math.max(1, timeoutMs - index * MAESTRO_REPLAY_POLICY.scrollUntilVisibleProbeMs), - ); -} - -async function invokeMaestroTapPointPercent(params: { - baseReq: ReplayBaseRequest; - positionals: string[]; - invoke: (req: DaemonRequest) => Promise; -}): Promise { - const [xValue, yValue] = params.positionals; - const xPercent = Number(xValue); - const yPercent = Number(yValue); - if (!Number.isFinite(xPercent) || !Number.isFinite(yPercent)) { - return errorResponse('INVALID_ARGS', 'tapOn percentage point requires numeric x/y values.'); - } - - const snapshotResponse = await params.invoke({ - ...params.baseReq, - command: 'snapshot', - positionals: [], - flags: { - ...params.baseReq.flags, - noRecord: true, - snapshotRaw: true, - snapshotForceFull: true, - }, - }); - if (!snapshotResponse.ok) return snapshotResponse; - - const snapshot = readSnapshotState(snapshotResponse.data); - if (!snapshot) { - return errorResponse( - 'COMMAND_FAILED', - 'Unable to read snapshot data for Maestro percentage point tap.', - ); - } - - const frame = getSnapshotReferenceFrame(snapshot); - if (!frame) { - return errorResponse( - 'COMMAND_FAILED', - 'Unable to resolve screen size for Maestro percentage point tap.', - ); - } - - return await params.invoke({ - ...params.baseReq, - command: 'click', - positionals: [ - String(Math.round((frame.referenceWidth * xPercent) / 100)), - String(Math.round((frame.referenceHeight * yPercent) / 100)), - ], - }); -} - -function readSnapshotState(data: unknown): SnapshotState | undefined { - if ( - typeof data === 'object' && - data !== null && - Array.isArray((data as { nodes?: unknown }).nodes) - ) { - return data as SnapshotState; - } - return undefined; -} - -function snapshotStabilitySignature(snapshot: SnapshotState): string { - return JSON.stringify( - snapshot.nodes.map((node) => ({ - index: node.index, - parentIndex: node.parentIndex, - type: node.type, - identifier: node.identifier, - label: node.label, - value: node.value, - rect: node.rect - ? { - x: Math.round(node.rect.x), - y: Math.round(node.rect.y), - width: Math.round(node.rect.width), - height: Math.round(node.rect.height), - } - : undefined, - })), - ); -} - -async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { - const [selector, rawOptions] = params.positionals; - if (!selector) { - return errorResponse('INVALID_ARGS', 'tapOn requires a selector.'); - } - const options = readMaestroTapOnOptions(rawOptions); - if (!options.ok) return options.response; - const startedAt = Date.now(); - const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); - const timeoutMs = maestroTapOnTimeoutMs(params); - let lastResponse: DaemonResponse | undefined; - while (Date.now() - startedAt < timeoutMs) { - const attempt = await attemptMaestroTapOn( - params, - selector, - options.value ?? {}, - fuzzyTextQuery, - ); - if (!attempt.retry) return attempt.response; - lastResponse = attempt.response; - await sleep(MAESTRO_REPLAY_POLICY.tapOnRetryMs); - } - - return maestroTapOnTimeoutResponse(params, selector, lastResponse); -} - -function maestroTapOnTimeoutMs(params: MaestroTapOnParams): number { - return params.baseReq.flags?.maestro?.optional === true - ? MAESTRO_REPLAY_POLICY.optionalTapOnTimeoutMs - : MAESTRO_REPLAY_POLICY.tapOnTimeoutMs; -} - -function maestroTapOnTimeoutResponse( - params: MaestroTapOnParams, - selector: string, - lastResponse: DaemonResponse | undefined, -): DaemonResponse { - if (params.baseReq.flags?.maestro?.optional === true) { - return { ok: true, data: { skipped: true, optional: true, selector } }; - } - return ( - lastResponse ?? errorResponse('COMMAND_FAILED', `tapOn timed out for selector: ${selector}`) - ); -} - -async function attemptMaestroTapOn( - params: MaestroTapOnParams, - selector: string, - options: MaestroTapOnOptions, - fuzzyTextQuery: string | null, -): Promise< - { retry: false; response: DaemonResponse } | { retry: true; response: FailedDaemonResponse } -> { - const attempt = await invokeMaestroSnapshotTapOn(params, selector, options); - if (attempt.ok) return { retry: false, response: attempt }; - if (!fuzzyTextQuery) return { retry: true, response: attempt }; - return await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); -} - -async function invokeMaestroSnapshotTapOn( - params: MaestroTapOnParams, - selector: string, - options: MaestroTapOnOptions, -): Promise { - const target = await resolveMaestroSnapshotTarget(params, selector, options, 'tapOn'); - if (!target.ok) return target.response; - const point = pointForMaestroTapOnTarget(target.target, selector); - return await params.invoke({ - ...params.baseReq, - command: 'click', - positionals: [String(point.x), String(point.y)], - }); -} - -async function invokeMaestroSwipeOn(params: { - baseReq: ReplayBaseRequest; - positionals: string[]; - invoke: MaestroRuntimeInvoke; -}): Promise { - const [selector, direction = 'up', durationMs] = params.positionals; - if (!selector) return errorResponse('INVALID_ARGS', 'swipe.label requires a label selector.'); - const target = await resolveMaestroSnapshotTarget(params, selector, {}, 'swipe.label'); - if (!target.ok) return target.response; - const swipe = swipeCoordinatesFromTarget(target.target, direction); - if (!swipe.ok) return swipe.response; - return await params.invoke({ - ...params.baseReq, - command: 'swipe', - positionals: [ - String(swipe.start.x), - String(swipe.start.y), - String(swipe.end.x), - String(swipe.end.y), - ...(durationMs ? [durationMs] : []), - ], - }); -} - -async function invokeMaestroFuzzyTapOn( - params: MaestroTapOnParams, - query: string, -): Promise< - { retry: false; response: DaemonResponse } | { retry: true; response: FailedDaemonResponse } -> { - const findResponse = await params.invoke({ - ...params.baseReq, - command: 'find', - positionals: [query, 'click'], - flags: { - ...params.baseReq.flags, - findFirst: true, - }, - }); - if (findResponse.ok) return { retry: false, response: findResponse }; - return { retry: true, response: findResponse }; -} - -async function resolveMaestroSnapshotTarget( - params: { - baseReq: ReplayBaseRequest; - invoke: MaestroRuntimeInvoke; - }, - selector: string, - options: MaestroTapOnOptions, - commandLabel: string, -): Promise<{ ok: true; target: MaestroSnapshotTarget } | { ok: false; response: DaemonResponse }> { - const snapshotResponse = await params.invoke({ - ...params.baseReq, - command: 'snapshot', - positionals: [], - flags: { - ...params.baseReq.flags, - noRecord: true, - snapshotRaw: true, - snapshotForceFull: true, - }, - }); - if (!snapshotResponse.ok) return { ok: false, response: snapshotResponse }; - - const snapshot = readSnapshotState(snapshotResponse.data); - if (!snapshot) { - return { - ok: false, - response: errorResponse( - 'COMMAND_FAILED', - `Unable to read snapshot data for ${commandLabel}.`, - ), - }; - } - - const frame = getSnapshotReferenceFrame(snapshot); - const resolution = resolveMaestroNodeFromSnapshot( - snapshot, - selector, - options, - readMaestroSelectorPlatform(params.baseReq.flags), - frame, - ); - if (!resolution.ok) { - return { - ok: false, - response: errorResponse('ELEMENT_NOT_FOUND', resolution.message), - }; - } - return { - ok: true, - target: { - node: resolution.node, - rect: resolution.rect, - frame, - }, - }; -} - -function resolveMaestroNodeFromSnapshot( - snapshot: SnapshotState, - selector: string, - options: MaestroTapOnOptions, - platform: Platform, - frame: TouchReferenceFrame | undefined, -): { ok: true; node: SnapshotNode; rect: Rect } | { ok: false; message: string } { - let matches = findMaestroSelectorMatches(snapshot, selector, platform); - if (options.childOf) { - const parents = findMaestroSelectorMatches(snapshot, options.childOf, platform); - if (parents.length === 0) { - return { ok: false, message: `Maestro childOf parent did not match: ${options.childOf}` }; - } - matches = matches.filter((node) => - parents.some((parent) => isDescendantOfSnapshotNode(snapshot.nodes, node, parent)), - ); - } - - const target = selectMaestroSnapshotMatch( - snapshot.nodes, - matches, - options.index, - extractMaestroVisibleTextQuery(selector) !== null, - frame, - ); - if (!target) { - const index = options.index ?? 0; - return { - ok: false, - message: `Maestro selector did not match index ${index}: ${selector}`, - }; - } - return { ok: true, node: target.node, rect: target.rect }; -} - -function findMaestroSelectorMatches( - snapshot: SnapshotState, - selectorExpression: string, - platform: Platform, -): SnapshotNode[] { - const chain = parseSelectorChain(selectorExpression); - for (const selector of chain.selectors) { - const matches = snapshot.nodes.filter((node) => matchesSelector(node, selector, platform)); - if (matches.length > 0) return matches; - } - return []; -} - -function resolveNodeRect(nodes: SnapshotState['nodes'], node: SnapshotNode): Rect | null { - if (node.rect && node.rect.width > 0 && node.rect.height > 0) return node.rect; - return ( - findSnapshotAncestor(nodes, node, (ancestor) => - ancestor.rect && ancestor.rect.width > 0 && ancestor.rect.height > 0 ? ancestor : null, - )?.rect ?? null - ); -} - -function selectMaestroSnapshotMatch( - nodes: SnapshotState['nodes'], - matches: SnapshotNode[], - index: number | undefined, - preferOnScreen: boolean, - frame: TouchReferenceFrame | undefined, -): { node: SnapshotNode; rect: Rect } | null { - const resolved = matches - .map((node) => { - const rect = resolveNodeRect(nodes, node); - return rect ? { node, rect } : null; - }) - .filter((candidate): candidate is { node: SnapshotNode; rect: Rect } => Boolean(candidate)); - const candidates = - preferOnScreen && index === undefined ? preferOnScreenMatches(resolved, frame) : resolved; - if (index !== undefined) return candidates[index] ?? null; - return candidates.sort(compareMaestroSnapshotMatches)[0] ?? null; -} - -function preferOnScreenMatches( - matches: { node: SnapshotNode; rect: Rect }[], - frame: TouchReferenceFrame | undefined, -): { node: SnapshotNode; rect: Rect }[] { - const onScreen = matches.filter((match) => isRectOnScreen(match.rect, frame)); - return onScreen.length > 0 ? onScreen : matches; -} - -function isRectOnScreen(rect: Rect, frame: TouchReferenceFrame | undefined): boolean { - const maxX = frame?.referenceWidth ?? Number.POSITIVE_INFINITY; - const maxY = frame?.referenceHeight ?? Number.POSITIVE_INFINITY; - return rect.x < maxX && rect.y < maxY && rect.x + rect.width > 0 && rect.y + rect.height > 0; -} - -function compareMaestroSnapshotMatches( - left: { node: SnapshotNode; rect: Rect }, - right: { node: SnapshotNode; rect: Rect }, -): number { - const typeRank = maestroTapTargetTypeRank(left.node) - maestroTapTargetTypeRank(right.node); - if (typeRank !== 0) return typeRank; - - const areaRank = left.rect.width * left.rect.height - right.rect.width * right.rect.height; - if (areaRank !== 0) return areaRank; - - return (right.node.depth ?? 0) - (left.node.depth ?? 0); -} - -function maestroTapTargetTypeRank(node: SnapshotNode): number { - return MAESTRO_TAP_TARGET_TYPE_RANK.get(node.type?.toLowerCase() ?? '') ?? 3; -} - -function isDescendantOfSnapshotNode( - nodes: SnapshotState['nodes'], - node: SnapshotNode, - ancestor: SnapshotNode, -): boolean { - return Boolean( - findSnapshotAncestor(nodes, node, (candidate) => - candidate === ancestor || candidate.index === ancestor.index ? candidate : null, - ), - ); -} - -function findSnapshotAncestor( - nodes: SnapshotState['nodes'], - node: SnapshotNode, - resolve: (ancestor: SnapshotNode) => T | null, -): T | null { - let current: SnapshotNode | undefined = node; - const byIndex = new Map(nodes.map((candidate) => [candidate.index, candidate])); - while (typeof current.parentIndex === 'number') { - current = byIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; - if (!current) return null; - const result = resolve(current); - if (result) return result; - } - return null; -} - -function readMaestroTapOnOptions( - rawOptions: string | undefined, -): { ok: true; value: MaestroTapOnOptions | null } | { ok: false; response: DaemonResponse } { - if (!rawOptions) return { ok: true, value: null }; - try { - const value = JSON.parse(rawOptions) as MaestroTapOnOptions; - return { ok: true, value }; - } catch { - return { - ok: false, - response: errorResponse('INVALID_ARGS', 'tapOn runtime options must be valid JSON.'), - }; - } -} - -function readMaestroSelectorPlatform(flags: ReplayBaseRequest['flags']): Platform { - return flags?.platform === 'android' ? 'android' : 'ios'; -} - -function swipeCoordinatesFromTarget( - target: MaestroSnapshotTarget, - direction: string, -): - | { ok: true; start: { x: number; y: number }; end: { x: number; y: number } } - | { ok: false; response: DaemonResponse } { - const center = pointInsideRect(target.rect); - const frame = target.frame; - const horizontalDistance = swipeDistance(frame?.referenceWidth, target.rect.width); - const verticalDistance = swipeDistance(frame?.referenceHeight, target.rect.height); - const margin = MAESTRO_REPLAY_POLICY.swipe.marginPx; - const minX = margin; - const minY = margin; - const maxX = frame ? frame.referenceWidth - margin : center.x + horizontalDistance; - const maxY = frame ? frame.referenceHeight - margin : center.y + verticalDistance; - switch (direction.toLowerCase()) { - case 'up': - return { - ok: true, - start: center, - end: { x: center.x, y: clampCoordinate(center.y - verticalDistance, minY, maxY) }, - }; - case 'down': - return { - ok: true, - start: center, - end: { x: center.x, y: clampCoordinate(center.y + verticalDistance, minY, maxY) }, - }; - case 'left': - return { - ok: true, - start: center, - end: { x: clampCoordinate(center.x - horizontalDistance, minX, maxX), y: center.y }, - }; - case 'right': - return { - ok: true, - start: center, - end: { x: clampCoordinate(center.x + horizontalDistance, minX, maxX), y: center.y }, - }; - default: - return { - ok: false, - response: errorResponse( - 'INVALID_ARGS', - 'swipe.label direction must be up, down, left, or right.', - ), - }; - } -} - -function swipeDistance(frameSize: number | undefined, rectSize: number): number { - const screenRelative = - typeof frameSize === 'number' ? frameSize * MAESTRO_REPLAY_POLICY.swipe.screenRatio : 0; - return Math.round( - Math.min( - MAESTRO_REPLAY_POLICY.swipe.maxDistancePx, - Math.max(MAESTRO_REPLAY_POLICY.swipe.minDistancePx, screenRelative, rectSize * 1.5), - ), - ); -} - -function clampCoordinate(value: number, min: number, max: number): number { - return Math.round(Math.min(max, Math.max(min, value))); -} - -function pointInsideRect(rect: Rect): { x: number; y: number } { - return { - x: interiorCoordinate(rect.x, rect.width), - y: interiorCoordinate(rect.y, rect.height), - }; -} - -function pointForMaestroTapOnTarget( - target: MaestroSnapshotTarget, - selector: string, -): { x: number; y: number } { - if (!shouldBiasMaestroVisibleTextTap(target.node, selector, target.rect)) { - return pointInsideRect(target.rect); - } - return { - x: interiorCoordinate( - target.rect.x, - Math.min(target.rect.width, MAESTRO_REPLAY_POLICY.largeTextContainerBias.width), - ), - y: interiorCoordinate( - target.rect.y, - Math.min(target.rect.height, MAESTRO_REPLAY_POLICY.largeTextContainerBias.height), - ), - }; -} - -function shouldBiasMaestroVisibleTextTap( - node: SnapshotNode, - selector: string, - rect: Rect, -): boolean { - if (!extractMaestroVisibleTextQuery(selector)) return false; - if ( - rect.height < MAESTRO_REPLAY_POLICY.largeTextContainerBias.minHeight || - rect.width < MAESTRO_REPLAY_POLICY.largeTextContainerBias.minWidth - ) { - return false; - } - const type = node.type?.toLowerCase(); - return type === 'cell' || type === 'other' || type === 'scrollview'; -} - -function interiorCoordinate(origin: number, size: number): number { - if (size <= 1) return Math.floor(origin); - const min = Math.ceil(origin); - const max = Math.floor(origin + size - 1); - return clampCoordinate(origin + size / 2, min, max); -} - -function invokeMaestroRunScript(params: { - baseReq: ReplayBaseRequest; - positionals: string[]; - scope: ReplayVarScope; -}): DaemonResponse { - const [scriptPath] = params.positionals; - if (!scriptPath) { - return errorResponse('INVALID_ARGS', 'runScript requires a file path.'); - } - try { - const outputEnv = executeRunScriptFile({ - scriptPath, - env: { - ...params.scope.values, - ...(params.baseReq.flags?.maestro?.runScriptEnv ?? {}), - }, - }); - return { ok: true, data: { outputEnv } }; - } catch (error) { - const appError = asAppError(error); - return errorResponse(appError.code, appError.message, appError.details); - } -} - -async function invokeMaestroRunFlowWhen(params: { - baseReq: ReplayBaseRequest; - positionals: string[]; - batchSteps: CommandFlags['batchSteps'] | undefined; - line: number; - step: number; - invoke: (req: DaemonRequest) => Promise; - invokeReplayAction: MaestroReplayInvoker; -}): Promise { - const condition = readMaestroRunFlowWhenCondition(params.positionals); - if (!condition.ok) return condition.response; - const conditionResponse = await params.invoke({ - ...params.baseReq, - command: 'is', - positionals: [condition.predicate, condition.selector], - flags: { ...params.baseReq.flags, noRecord: true }, - }); - if (isMaestroWhenConditionMiss(conditionResponse)) { - return { - ok: true, - data: { skipped: true, condition: condition.mode, selector: condition.selector }, - }; - } - if (!conditionResponse.ok) return conditionResponse; - return await invokeMaestroRunFlowWhenSteps(params, condition); -} - -function readMaestroRunFlowWhenCondition(positionals: string[]): MaestroRunFlowWhenCondition { - const [mode, selector] = positionals; - if ((mode !== 'visible' && mode !== 'notVisible') || !selector) { - return { - ok: false, - response: errorResponse( - 'INVALID_ARGS', - 'runFlow.when requires visible/notVisible and a selector.', - ), - }; - } - return { - ok: true, - mode, - predicate: mode === 'visible' ? 'visible' : 'hidden', - selector, - }; -} - -async function invokeMaestroRunFlowWhenSteps( - params: { - batchSteps: CommandFlags['batchSteps'] | undefined; - line: number; - step: number; - invokeReplayAction: MaestroReplayInvoker; - }, - condition: Extract, -): Promise { - const steps = (params.batchSteps ?? []).map(batchStepToSessionAction); - for (const [index, action] of steps.entries()) { - // Preserve stable parent-step ordering for nested runtime commands while - // keeping the substep distinguishable in traces. - const response = await params.invokeReplayAction({ - action, - line: params.line, - step: params.step + index / 1000, - }); - if (!response.ok) return response; - } - - return { - ok: true, - data: { ran: steps.length, condition: condition.mode, selector: condition.selector }, - }; -} - -function isMaestroWhenConditionMiss(response: DaemonResponse): boolean { - if (response.ok) return response.data?.pass === false; - const details = response.error.details; - return ( - details?.command === 'is' && - (details.reason === 'selector_not_found' || details.reason === 'predicate_failed') - ); -} - -function batchStepToSessionAction( - step: NonNullable[number], -): SessionAction { - const action: SessionAction = { - ts: Date.now(), - command: step.command, - positionals: step.positionals ?? [], - flags: step.flags ?? {}, - }; - if (step.runtime && typeof step.runtime === 'object') { - action.runtime = step.runtime as SessionAction['runtime']; - } - return action; -} - -function extractMaestroVisibleTextQuery(selectorExpression: string): string | null { - const chain = parseSelectorChain(selectorExpression); - const terms = chain.selectors.flatMap((selector) => selector.terms); - if (terms.length === 0) return null; - // Mixed selectors may encode more than a visible-text lookup, so they keep - // the exact selector path instead of fuzzy text fallback. - if (!terms.some((term) => term.key === 'label' || term.key === 'text')) return null; - if (!terms.every((term) => ['label', 'text', 'id'].includes(term.key))) return null; - const values = terms.map((term) => (typeof term.value === 'string' ? term.value : '')); - const first = values[0]; - if (!first || !values.every((value) => value === first)) return null; - return first; -} - -function withMaestroScrollTimeoutContext( - response: FailedDaemonResponse | null, - selector: string, - timeoutMs: number, -): DaemonResponse { - if (!response) { - return errorResponse( - 'COMMAND_FAILED', - `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}`, - ); - } - return { - ok: false, - error: { - ...response.error, - message: `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}. Last wait: ${response.error.message}`, - }, - }; -} diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index 9b830fee0..37c59e8e4 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -198,7 +198,7 @@ function buildReplayMetadataFlags( }; } -export function withReplayFailureContext( +function withReplayFailureContext( response: DaemonResponse, action: SessionAction, index: number, diff --git a/src/daemon/handlers/session-test-discovery.ts b/src/daemon/handlers/session-test-discovery.ts index df2101840..85ecc5113 100644 --- a/src/daemon/handlers/session-test-discovery.ts +++ b/src/daemon/handlers/session-test-discovery.ts @@ -27,11 +27,13 @@ export function discoverReplayTestEntries(params: { inputs: string[]; cwd?: string; platformFilter?: PlatformSelector; + replayBackend?: string; }): ReplayTestDiscoveryEntry[] { - const { inputs, cwd, platformFilter } = params; + const { inputs, cwd, platformFilter, replayBackend } = params; + const extensions = replayTestExtensions(replayBackend); const resolvedCwd = cwd ?? process.cwd(); const filePaths = [ - ...new Set(inputs.flatMap((input) => expandReplayTestInput(input, resolvedCwd))), + ...new Set(inputs.flatMap((input) => expandReplayTestInput(input, resolvedCwd, extensions))), ] .map((entry) => path.normalize(entry)) .sort((left, right) => left.localeCompare(right)); @@ -45,12 +47,16 @@ export function discoverReplayTestEntries(params: { continue; } if (!metadata.platform) { - entries.push({ - kind: 'skip', - path: filePath, - reason: 'skipped-by-filter', - message: `missing platform metadata for --platform ${platformFilter}`, - }); + if (isMaestroReplayBackend(replayBackend)) { + entries.push({ kind: 'run', path: filePath, metadata }); + } else { + entries.push({ + kind: 'skip', + path: filePath, + reason: 'skipped-by-filter', + message: `missing platform metadata for --platform ${platformFilter}`, + }); + } continue; } if (!matchesPlatformFilter(platformFilter, metadata.platform)) { @@ -62,7 +68,7 @@ export function discoverReplayTestEntries(params: { const runnableCount = entries.filter((entry) => entry.kind === 'run').length; if (runnableCount === 0) { const suffix = platformFilter ? ` for --platform ${platformFilter}` : ''; - throw new AppError('INVALID_ARGS', `No .ad tests matched${suffix}.`); + throw new AppError('INVALID_ARGS', `No replay tests matched${suffix}.`); } return entries; @@ -123,18 +129,20 @@ export function resolveReplayTestRetries( return Math.max(0, Math.min(MAX_REPLAY_TEST_RETRIES, resolved)); } -function expandReplayTestInput(input: string, cwd: string): string[] { +function expandReplayTestInput(input: string, cwd: string, extensions: Set): string[] { const expandedInput = SessionStore.expandHome(input, cwd); if (fs.existsSync(expandedInput)) { const stat = fs.statSync(expandedInput); if (stat.isDirectory()) { - return fs - .globSync('**/*.ad', { cwd: expandedInput }) - .map((match) => path.join(expandedInput, match)); + return replayTestGlobPatterns(extensions).flatMap((pattern) => + fs + .globSync(pattern, { cwd: expandedInput }) + .map((match) => path.join(expandedInput, match)), + ); } if (stat.isFile()) { - if (path.extname(expandedInput) !== '.ad') { - throw new AppError('INVALID_ARGS', `test requires .ad files. Received: ${input}`); + if (!extensions.has(path.extname(expandedInput))) { + throw new AppError('INVALID_ARGS', `test does not support this file type: ${input}`); } return [expandedInput]; } @@ -152,7 +160,21 @@ function expandReplayTestInput(input: string, cwd: string): string[] { return matches .map((match) => (path.isAbsolute(match) ? match : path.resolve(cwd, match))) - .filter((match) => path.extname(match) === '.ad' && isExistingFile(match)); + .filter((match) => extensions.has(path.extname(match)) && isExistingFile(match)); +} + +function replayTestExtensions(replayBackend: string | undefined): Set { + return isMaestroReplayBackend(replayBackend) + ? new Set(['.ad', '.yaml', '.yml']) + : new Set(['.ad']); +} + +function replayTestGlobPatterns(extensions: Set): string[] { + return [...extensions].map((extension) => `**/*${extension}`); +} + +function isMaestroReplayBackend(replayBackend: string | undefined): boolean { + return replayBackend === 'maestro'; } function looksLikeGlob(value: string): boolean { diff --git a/src/daemon/handlers/session-test.ts b/src/daemon/handlers/session-test.ts index 14f21bf9b..35b3d1ab2 100644 --- a/src/daemon/handlers/session-test.ts +++ b/src/daemon/handlers/session-test.ts @@ -41,6 +41,7 @@ export async function runReplayTestSuite( inputs: req.positionals, cwd: req.meta?.cwd, platformFilter: req.flags?.platform, + replayBackend: req.flags?.replayBackend, }); const suiteInvocationId = buildReplayTestInvocationId(req.meta?.requestId); const suiteArtifactsDir = resolveReplayTestArtifactsDir({ diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index 47ecf8cdb..17b5917ec 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -60,7 +60,10 @@ export async function captureSnapshot(params: CaptureSnapshotParams): Promise<{ androidSnapshot?: AndroidSnapshotBackendMetadata; freshness?: AndroidFreshnessCaptureMeta; }> { - if (params.device.platform === 'ios' && params.session?.postGestureStabilization) { + if ( + (params.device.platform === 'ios' || params.device.platform === 'android') && + params.session?.postGestureStabilization + ) { return { snapshot: await capturePostGestureStabilizedSnapshot({ session: params.session, diff --git a/src/daemon/post-gesture-stabilization.ts b/src/daemon/post-gesture-stabilization.ts index dd4e17dde..a99b400f5 100644 --- a/src/daemon/post-gesture-stabilization.ts +++ b/src/daemon/post-gesture-stabilization.ts @@ -8,7 +8,7 @@ const STABILIZATION_INTERVAL_MS = 200; const RECT_TOLERANCE_PX = 1; export function markPostGestureStabilization(session: SessionState, action: string): void { - if (session.device.platform !== 'ios') return; + if (!supportsPostGestureStabilization(session.device.platform)) return; if (!isPostGestureStabilizingAction(action)) return; session.postGestureStabilization = { action, @@ -27,7 +27,7 @@ export async function capturePostGestureStabilizedSnapshot(params: { }): Promise { const { session, capture } = params; const pending = session?.postGestureStabilization; - if (!session || session.device.platform !== 'ios' || !pending) { + if (!session || !supportsPostGestureStabilization(session.device.platform) || !pending) { return await capture(); } @@ -75,6 +75,10 @@ function isPostGestureStabilizingAction(action: string): boolean { return action === 'swipe' || action === 'scroll'; } +function supportsPostGestureStabilization(platform: SessionState['device']['platform']): boolean { + return platform === 'ios' || platform === 'android'; +} + type StabilityEntry = { key: string; x: number; diff --git a/src/daemon/runtime-session.ts b/src/daemon/runtime-session.ts index 1ef016b54..5ac613d6d 100644 --- a/src/daemon/runtime-session.ts +++ b/src/daemon/runtime-session.ts @@ -7,7 +7,7 @@ export type RuntimeSessionRecordOptions = { metadata?: Record; }; -export function toRuntimeSessionRecord( +function toRuntimeSessionRecord( session: SessionState | undefined, name: string, options: RuntimeSessionRecordOptions = {}, diff --git a/src/mcp/server.ts b/src/mcp/server.ts index da6da9783..e18fc59d2 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -32,9 +32,7 @@ export async function runAgentDeviceMcpServer(): Promise { }); } -export function handleMcpPayload( - messageOrBatch: JsonRpcMessage | JsonRpcMessage[], -): unknown | null { +function handleMcpPayload(messageOrBatch: JsonRpcMessage | JsonRpcMessage[]): unknown | null { if (Array.isArray(messageOrBatch)) { const responses = messageOrBatch.flatMap((message) => responseArray(handleMcpMessage(message))); return responses.length > 0 ? responses : null; diff --git a/src/platforms/android/__tests__/adb-provider-scope.test.ts b/src/platforms/android/__tests__/adb-provider-scope.test.ts index b1ddec423..50f32347c 100644 --- a/src/platforms/android/__tests__/adb-provider-scope.test.ts +++ b/src/platforms/android/__tests__/adb-provider-scope.test.ts @@ -4,9 +4,7 @@ import os from 'node:os'; import path from 'node:path'; import { test } from 'vitest'; import { runCmd } from '../../../utils/exec.ts'; -import { - withAndroidAdbProvider, -} from '../adb-executor.ts'; +import { withAndroidAdbProvider } from '../adb-executor.ts'; const device = { platform: 'android', @@ -65,4 +63,3 @@ test('withAndroidAdbProvider ignores adb commands for another serial', async () assert.equal(result.stdout, 'local -s other-device shell echo local'); assert.deepEqual(calls, []); }); - diff --git a/src/platforms/android/__tests__/index.test.ts b/src/platforms/android/__tests__/index.test.ts index 375077663..bec00446b 100644 --- a/src/platforms/android/__tests__/index.test.ts +++ b/src/platforms/android/__tests__/index.test.ts @@ -734,6 +734,38 @@ test('openAndroidApp reports localhost reverse failures with port context', asyn ); }); +test('openAndroidApp binds deep link URLs to the requested package', async () => { + await withMockedAdb( + 'agent-device-android-open-deep-link-package-', + [ + '#!/bin/sh', + 'printf "__CMD__\\n" >> "$AGENT_DEVICE_TEST_ARGS_FILE"', + 'printf "%s\\n" "$@" >> "$AGENT_DEVICE_TEST_ARGS_FILE"', + 'if [ "$1" = "-s" ]; then', + ' shift', + ' shift', + 'fi', + 'if [ "$1" = "shell" ] && [ "$2" = "pm" ] && [ "$3" = "list" ]; then', + ' echo "package:com.example.app"', + ' exit 0', + 'fi', + 'if [ "$1" = "shell" ] && [ "$2" = "am" ] && [ "$3" = "start" ]; then', + ' echo "Status: ok"', + ' exit 0', + 'fi', + 'exit 0', + '', + ].join('\n'), + async ({ argsLogPath, device }) => { + await openAndroidApp(device, 'com.example.app', { url: 'example://bottom-tabs' }); + const logged = await fs.readFile(argsLogPath, 'utf8'); + assert.match(logged, /shell\nam\nstart\n-W\n-a\nandroid\.intent\.action\.VIEW/); + assert.match(logged, /-d\nexample:\/\/bottom-tabs/); + assert.match(logged, /-p\ncom\.example\.app/); + }, + ); +}); + test('setAndroidSetting appearance toggle flips current mode', async () => { await withMockedAdb( 'agent-device-android-appearance-toggle-', diff --git a/src/platforms/android/__tests__/perf.test.ts b/src/platforms/android/__tests__/perf.test.ts index 46067904b..274f5a726 100644 --- a/src/platforms/android/__tests__/perf.test.ts +++ b/src/platforms/android/__tests__/perf.test.ts @@ -1,9 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { - parseAndroidFramePerfSample, - parseAndroidMemInfoSample, -} from '../perf.ts'; +import { parseAndroidFramePerfSample, parseAndroidMemInfoSample } from '../perf.ts'; test('parseAndroidMemInfoSample supports legacy total row layout', () => { const sample = parseAndroidMemInfoSample( diff --git a/src/platforms/android/__tests__/snapshot-helper.test.ts b/src/platforms/android/__tests__/snapshot-helper.test.ts index 5bd98da4a..45fed04ae 100644 --- a/src/platforms/android/__tests__/snapshot-helper.test.ts +++ b/src/platforms/android/__tests__/snapshot-helper.test.ts @@ -47,6 +47,7 @@ test('parseAndroidSnapshotHelperOutput reconstructs XML chunks and metadata', () helperApiVersion: '1', outputFormat: 'uiautomator-xml', waitForIdleTimeoutMs: '25', + waitForIdleQuietMs: '10', timeoutMs: '8000', maxDepth: '128', maxNodes: '5000', @@ -66,6 +67,7 @@ test('parseAndroidSnapshotHelperOutput reconstructs XML chunks and metadata', () helperApiVersion: '1', outputFormat: 'uiautomator-xml', waitForIdleTimeoutMs: 25, + waitForIdleQuietMs: 10, timeoutMs: 8000, maxDepth: 128, maxNodes: 5000, @@ -522,6 +524,11 @@ test('ensureAndroidSnapshotHelper retry install also uses provider install capab test('captureAndroidSnapshotWithHelper uses injected adb executor', async () => { let capturedArgs: string[] | undefined; const adb: AndroidAdbExecutor = async (args, options) => { + if (args[0] === 'shell' && args[1] === 'rm') { + assert.equal(options?.allowFailure, true); + assert.equal(options?.timeoutMs, 5000); + return { exitCode: 0, stdout: '', stderr: '' }; + } capturedArgs = args; assert.equal(options?.allowFailure, true); assert.equal(options?.timeoutMs, 14000); @@ -533,6 +540,7 @@ test('captureAndroidSnapshotWithHelper uses injected adb executor', async () => ok: 'true', outputFormat: 'uiautomator-xml', waitForIdleTimeoutMs: '10', + waitForIdleQuietMs: '5', timeoutMs: '9000', maxDepth: '64', maxNodes: '100', @@ -545,9 +553,11 @@ test('captureAndroidSnapshotWithHelper uses injected adb executor', async () => const result = await captureAndroidSnapshotWithHelper({ adb, waitForIdleTimeoutMs: 10, + waitForIdleQuietMs: 5, timeoutMs: 9000, maxDepth: 64, maxNodes: 100, + outputPath: '/sdcard/Android/data/com.callstack.agentdevice.snapshothelper/files/test.xml', }); assert.deepEqual(capturedArgs, [ @@ -559,6 +569,9 @@ test('captureAndroidSnapshotWithHelper uses injected adb executor', async () => 'waitForIdleTimeoutMs', '10', '-e', + 'waitForIdleQuietMs', + '5', + '-e', 'timeoutMs', '9000', '-e', @@ -567,6 +580,9 @@ test('captureAndroidSnapshotWithHelper uses injected adb executor', async () => '-e', 'maxNodes', '100', + '-e', + 'outputPath', + '/sdcard/Android/data/com.callstack.agentdevice.snapshothelper/files/test.xml', 'com.callstack.agentdevice.snapshothelper/.SnapshotInstrumentation', ]); assert.equal(result.xml, ''); @@ -576,7 +592,10 @@ test('captureAndroidSnapshotWithHelper uses injected adb executor', async () => test('captureAndroidSnapshotWithHelper gives adb command overhead beyond helper timeout', async () => { let commandTimeoutMs: number | undefined; await captureAndroidSnapshotWithHelper({ - adb: async (_args, options) => { + adb: async (args, options) => { + if (args[0] === 'shell' && args[1] === 'rm') { + return { exitCode: 0, stdout: '', stderr: '' }; + } commandTimeoutMs = options?.timeoutMs; return { exitCode: 0, @@ -622,6 +641,42 @@ test('captureAndroidSnapshotWithHelper wraps unparseable failed output with adb ); }); +test('captureAndroidSnapshotWithHelper reads helper output file when instrumentation output is unparseable', async () => { + const calls: string[][] = []; + const result = await captureAndroidSnapshotWithHelper({ + adb: async (args) => { + calls.push(args); + if (args[0] === 'shell' && args[1] === 'am') { + return { + exitCode: 0, + stdout: 'INSTRUMENTATION_RESULT: shortMsg=Process crashed.', + stderr: '', + }; + } + if (args[0] === 'shell' && args[1] === 'cat') { + return { + exitCode: 0, + stdout: '', + stderr: '', + }; + } + if (args[0] === 'shell' && args[1] === 'rm') { + return { exitCode: 0, stdout: '', stderr: '' }; + } + throw new Error(`unexpected args: ${args.join(' ')}`); + }, + outputPath: '/sdcard/Android/data/com.callstack.agentdevice.snapshothelper/files/test.xml', + }); + + assert.equal(result.xml, ''); + assert.equal(result.metadata.outputFormat, 'uiautomator-xml'); + assert.deepEqual(calls.at(1), [ + 'shell', + 'cat', + '/sdcard/Android/data/com.callstack.agentdevice.snapshothelper/files/test.xml', + ]); +}); + test('prepareAndroidSnapshotHelperArtifactFromManifestUrl downloads and verifies APK', async () => { const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'snapshot-helper-download-')); const apk = Buffer.from('downloaded-helper'); diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index 6c42d9bee..215c01b6b 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -246,6 +246,9 @@ async function withTempScreenshot( function mockAndroidSnapshotXml(xml: string, activityDump = ''): void { mockRunCmd.mockImplementation(async (_cmd, args) => { + if (isAndroidSdkVersionCommand(args)) { + return { exitCode: 0, stdout: '35', stderr: '' }; + } if (args.includes('exec-out')) { return { exitCode: 0, stdout: xml, stderr: '' }; } @@ -256,6 +259,12 @@ function mockAndroidSnapshotXml(xml: string, activityDump = ''): void { }); } +function isAndroidSdkVersionCommand(args: string[]): boolean { + return ( + args.includes('shell') && args.includes('getprop') && args.includes('ro.build.version.sdk') + ); +} + function adbTimeout(args: string[]): AppError { return new AppError('COMMAND_FAILED', 'adb timed out after 8000ms', { cmd: 'adb', @@ -329,7 +338,7 @@ test('snapshotAndroid uses injected helper artifact before stock uiautomator', a assert.equal(result.androidSnapshot.installReason, 'current'); assert.equal(result.androidSnapshot.captureMode, 'interactive-windows'); assert.equal(result.androidSnapshot.windowCount, 1); - assert.deepEqual(timeouts, [30000, 8000]); + assert.deepEqual(timeouts, [30000, 30000]); assert.equal(mockRunCmd.mock.calls.length, 0); }); @@ -410,6 +419,9 @@ test('snapshotAndroid resolves helper adb through scoped provider', async () => stderr: '', }; } + if (isAndroidSdkVersionCommand(args)) { + return { exitCode: 0, stdout: '35', stderr: '' }; + } if (args.includes('instrument')) { return { exitCode: 0, @@ -419,6 +431,9 @@ test('snapshotAndroid resolves helper adb through scoped provider', async () => stderr: '', }; } + if (args[0] === 'shell' && args[1] === 'rm') { + return { exitCode: 0, stdout: '', stderr: '' }; + } throw new Error(`unexpected scoped helper adb args: ${args.join(' ')}`); }, }; @@ -611,6 +626,52 @@ test('snapshotAndroid skips stock fallback after structured helper timeout', asy assert.equal(stockAttempted, false); }); +test('snapshotAndroid skips stock fallback after killed helper instrumentation', async () => { + let stockAttempted = false; + const helperAdb = createHelperAdb({ + instrument: async () => ({ exitCode: 137, stdout: '', stderr: '' }), + stock: async () => { + stockAttempted = true; + throw new Error('stock fallback should not run'); + }, + }); + + await assert.rejects( + () => snapshotAndroidWithHelper(helperAdb), + (error) => { + assert.match( + (error as Error).message, + /Android snapshot helper failed before returning parseable output/, + ); + assert.match((error as Error).message, /Stock UIAutomator fallback was skipped/); + assert.equal((error as { details?: Record }).details?.exitCode, 137); + return true; + }, + ); + assert.equal(stockAttempted, false); +}); + +test('snapshotAndroid skips stock fallback after unparseable helper output', async () => { + let stockAttempted = false; + const helperAdb = createHelperAdb({ + instrument: async () => ({ exitCode: 0, stdout: '', stderr: '' }), + stock: async () => { + stockAttempted = true; + throw new Error('stock fallback should not run'); + }, + }); + + await assert.rejects( + () => snapshotAndroidWithHelper(helperAdb), + (error) => { + assert.match((error as Error).message, /Android snapshot helper output could not be parsed/); + assert.match((error as Error).message, /Stock UIAutomator fallback was skipped/); + return true; + }, + ); + assert.equal(stockAttempted, false); +}); + test('snapshotAndroid falls back to stock dump after helper adb timeout', async () => { const stockXml = ''; @@ -753,6 +814,33 @@ test('dumpUiHierarchy reads fallback XML when dump exits non-zero', async () => assert.equal(catCall?.[2], undefined); }); +test('dumpUiHierarchy does not read a stale fallback file when dump fails without a path', async () => { + mockRunCmd.mockImplementation(async (_cmd, args) => { + if (args.includes('exec-out')) { + return { exitCode: 137, stdout: 'Killed', stderr: '' }; + } + if ( + args.includes('uiautomator') && + args.includes('dump') && + args.includes('/sdcard/window_dump.xml') + ) { + return { exitCode: 137, stdout: 'Killed', stderr: '' }; + } + if (args.includes('cat') && args.includes('/sdcard/window_dump.xml')) { + throw new Error('cat should not read a stale dump file'); + } + throw new Error(`unexpected args: ${args.join(' ')}`); + }); + + await assert.rejects( + dumpUiHierarchy(device), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + error.message.includes('did not produce a fresh XML file'), + ); +}); + test('dumpUiHierarchy retries when fallback dump file is temporarily missing', async () => { const xml = ''; let catAttempts = 0; @@ -903,6 +991,9 @@ test('snapshotAndroid skips activity dump when snapshot has no scrollable nodes' `; mockRunCmd.mockImplementation(async (_cmd, args) => { + if (isAndroidSdkVersionCommand(args)) { + return { exitCode: 0, stdout: '35', stderr: '' }; + } if (args.includes('exec-out')) { return { exitCode: 0, stdout: xml, stderr: '' }; } @@ -929,6 +1020,9 @@ test('snapshotAndroid skips hidden content hints when disabled', async () => { `; mockRunCmd.mockImplementation(async (_cmd, args) => { + if (isAndroidSdkVersionCommand(args)) { + return { exitCode: 0, stdout: '35', stderr: '' }; + } if (args.includes('exec-out')) { return { exitCode: 0, stdout: xml, stderr: '' }; } diff --git a/src/platforms/android/app-lifecycle.ts b/src/platforms/android/app-lifecycle.ts index fb1ccb118..cc8fdc67f 100644 --- a/src/platforms/android/app-lifecycle.ts +++ b/src/platforms/android/app-lifecycle.ts @@ -291,72 +291,132 @@ async function ensureAndroidLocalhostReverse(device: DeviceInfo, target: string) } } +export type OpenAndroidAppOptions = { + activity?: string; + appBundleId?: string; + url?: string; +}; + export async function openAndroidApp( device: DeviceInfo, app: string, - activity?: string, + optionsOrActivity?: OpenAndroidAppOptions | string, ): Promise { if (!device.booted) { await waitForAndroidBoot(device.id); } + const options = normalizeOpenAndroidAppOptions(optionsOrActivity); + const activity = options.activity; const deepLinkTarget = app.trim(); if (isDeepLinkTarget(deepLinkTarget)) { - if (activity) { - throw new AppError( - 'INVALID_ARGS', - 'Activity override is not supported when opening a deep link URL', - ); - } - await ensureAndroidLocalhostReverse(device, deepLinkTarget); - await runAndroidAdb(device, [ - 'shell', - 'am', - 'start', - '-W', - '-a', - 'android.intent.action.VIEW', - '-d', - deepLinkTarget, - ]); + await openAndroidDeepLink(device, deepLinkTarget, options); + return; + } + if (options.url !== undefined) { + await openAndroidAppBoundDeepLink(device, app, options); return; } const resolved = await resolveAndroidApp(device, app); const launchCategory = resolveAndroidLauncherCategory(device); if (resolved.type === 'intent') { - if (activity) { - throw new AppError( - 'INVALID_ARGS', - 'Activity override requires a package name, not an intent', - ); - } - await runAndroidAdb(device, ['shell', 'am', 'start', '-W', '-a', resolved.value]); + await openAndroidIntent(device, resolved.value, activity); return; } if (activity) { - const component = activity.includes('/') - ? activity - : `${resolved.value}/${activity.startsWith('.') ? activity : `.${activity}`}`; - try { - await runAndroidAdb(device, [ - 'shell', - 'am', - 'start', - '-W', - '-a', - 'android.intent.action.MAIN', - '-c', - ANDROID_DEFAULT_CATEGORY, - '-c', - launchCategory, - '-n', - component, - ]); - } catch (error) { - await maybeRethrowAndroidMissingPackageError(device, resolved.value, error); - throw error; - } + await openAndroidPackageActivity(device, resolved.value, activity, launchCategory); return; } + await openAndroidPackage(device, resolved.value, launchCategory); +} + +async function openAndroidDeepLink( + device: DeviceInfo, + target: string, + options: OpenAndroidAppOptions, +): Promise { + if (options.activity) { + throw new AppError( + 'INVALID_ARGS', + 'Activity override is not supported when opening a deep link URL', + ); + } + await ensureAndroidLocalhostReverse(device, target); + await runAndroidAdb(device, [ + 'shell', + 'am', + 'start', + '-W', + '-a', + 'android.intent.action.VIEW', + '-d', + target, + ...androidDeepLinkPackageArgs(options.appBundleId), + ]); +} + +async function openAndroidAppBoundDeepLink( + device: DeviceInfo, + app: string, + options: OpenAndroidAppOptions, +): Promise { + if (options.activity) { + throw new AppError( + 'INVALID_ARGS', + 'Activity override is not supported when opening an app-bound deep link URL', + ); + } + const deepLinkUrl = options.url?.trim() ?? ''; + if (!isDeepLinkTarget(deepLinkUrl)) { + throw new AppError('INVALID_ARGS', 'Android app-bound open requires a valid URL target'); + } + const resolved = await resolveAndroidPackageForOpen(device, app, 'app-bound open'); + await runAndroidAdb(device, [ + 'shell', + 'am', + 'start', + '-W', + '-a', + 'android.intent.action.VIEW', + '-d', + deepLinkUrl, + '-p', + resolved, + ]); +} + +async function openAndroidIntent( + device: DeviceInfo, + intent: string, + activity: string | undefined, +): Promise { + if (activity) { + throw new AppError('INVALID_ARGS', 'Activity override requires a package name, not an intent'); + } + await runAndroidAdb(device, ['shell', 'am', 'start', '-W', '-a', intent]); +} + +async function openAndroidPackageActivity( + device: DeviceInfo, + packageName: string, + activity: string, + launchCategory: string, +): Promise { + const component = activity.includes('/') + ? activity + : `${packageName}/${activity.startsWith('.') ? activity : `.${activity}`}`; + try { + await runAndroidAdb(device, buildAndroidActivityLaunchArgs(component, launchCategory)); + } catch (error) { + await maybeRethrowAndroidMissingPackageError(device, packageName, error); + throw error; + } +} + +async function openAndroidPackage( + device: DeviceInfo, + packageName: string, + launchCategory: string, +): Promise { const primaryResult = await runAndroidAdb( device, [ @@ -371,24 +431,28 @@ export async function openAndroidApp( '-c', launchCategory, '-p', - resolved.value, + packageName, ], { allowFailure: true }, ); if (primaryResult.exitCode === 0 && !isAmStartError(primaryResult.stdout, primaryResult.stderr)) { return; } - const component = await resolveAndroidLaunchComponent(device, resolved.value); + const component = await resolveAndroidLaunchComponent(device, packageName); if (!component) { - if (!(await isAndroidPackageInstalled(device, resolved.value))) { - throw buildAndroidPackageNotInstalledError(resolved.value); + if (!(await isAndroidPackageInstalled(device, packageName))) { + throw buildAndroidPackageNotInstalledError(packageName); } - throw new AppError('COMMAND_FAILED', `Failed to launch ${resolved.value}`, { + throw new AppError('COMMAND_FAILED', `Failed to launch ${packageName}`, { stdout: primaryResult.stdout, stderr: primaryResult.stderr, }); } - await runAndroidAdb(device, [ + await runAndroidAdb(device, buildAndroidActivityLaunchArgs(component, launchCategory)); +} + +function buildAndroidActivityLaunchArgs(component: string, launchCategory: string): string[] { + return [ 'shell', 'am', 'start', @@ -401,7 +465,31 @@ export async function openAndroidApp( launchCategory, '-n', component, - ]); + ]; +} + +async function resolveAndroidPackageForOpen( + device: DeviceInfo, + app: string, + label: string, +): Promise { + const resolved = await resolveAndroidApp(device, app); + if (resolved.type === 'intent') { + throw new AppError('INVALID_ARGS', `Android ${label} requires a package name, not an intent`); + } + return resolved.value; +} + +function normalizeOpenAndroidAppOptions( + optionsOrActivity: OpenAndroidAppOptions | string | undefined, +): OpenAndroidAppOptions { + if (typeof optionsOrActivity === 'string') return { activity: optionsOrActivity }; + return optionsOrActivity ?? {}; +} + +function androidDeepLinkPackageArgs(packageName: string | undefined): string[] { + const normalized = packageName?.trim(); + return normalized ? ['-p', normalized] : []; } function buildAndroidPackageNotInstalledError(packageName: string): AppError { diff --git a/src/platforms/android/perf.ts b/src/platforms/android/perf.ts index a7c4bbcc0..8412f2ccd 100644 --- a/src/platforms/android/perf.ts +++ b/src/platforms/android/perf.ts @@ -73,7 +73,7 @@ export async function sampleAndroidMemoryPerf( } } -export function parseAndroidCpuInfoSample( +function parseAndroidCpuInfoSample( stdout: string, packageName: string, measuredAt: string, diff --git a/src/platforms/android/snapshot-helper-capture.ts b/src/platforms/android/snapshot-helper-capture.ts index 4dea0e922..a20ab39c5 100644 --- a/src/platforms/android/snapshot-helper-capture.ts +++ b/src/platforms/android/snapshot-helper-capture.ts @@ -7,6 +7,7 @@ import { ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT, ANDROID_SNAPSHOT_HELPER_PACKAGE, ANDROID_SNAPSHOT_HELPER_PROTOCOL, + ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_QUIET_MS, ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, } from './snapshot-helper-types.ts'; import type { @@ -29,70 +30,172 @@ type AndroidInstrumentationRecordState = { currentResult: Record | null; }; +type AndroidSnapshotHelperResolvedCaptureOptions = { + waitForIdleTimeoutMs: number; + waitForIdleQuietMs: number; + timeoutMs: number; + commandTimeoutMs: number; + maxDepth: number; + maxNodes: number; + packageName: string; + runner: string; + outputPath?: string; +}; + export async function captureAndroidSnapshotWithHelper( options: AndroidSnapshotHelperCaptureOptions, ): Promise { - const waitForIdleTimeoutMs = - options.waitForIdleTimeoutMs ?? ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS; - const timeoutMs = options.timeoutMs ?? 8_000; - const commandTimeoutMs = - options.commandTimeoutMs ?? timeoutMs + ANDROID_SNAPSHOT_HELPER_COMMAND_OVERHEAD_MS; - const maxDepth = options.maxDepth ?? 128; - const maxNodes = options.maxNodes ?? 5_000; - const packageName = options.packageName ?? ANDROID_SNAPSHOT_HELPER_PACKAGE; - const runner = options.instrumentationRunner ?? `${packageName}/.SnapshotInstrumentation`; - const args = [ + const resolved = resolveAndroidSnapshotHelperCaptureOptions(options); + const result = await options.adb(buildAndroidSnapshotHelperArgs(resolved), { + allowFailure: true, + timeoutMs: resolved.commandTimeoutMs, + }); + const output = await readAndroidSnapshotHelperOutput(options, resolved, result); + if (resolved.outputPath) await removeHelperOutputFile(options.adb, resolved.outputPath); + if (result.exitCode !== 0) { + throw new AppError('COMMAND_FAILED', 'Android snapshot helper failed', { + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.exitCode, + helper: output.metadata, + }); + } + return output; +} + +function resolveAndroidSnapshotHelperCaptureOptions( + options: AndroidSnapshotHelperCaptureOptions, +): AndroidSnapshotHelperResolvedCaptureOptions { + const timeoutMs = withDefault(options.timeoutMs, 8_000); + const packageName = withDefault(options.packageName, ANDROID_SNAPSHOT_HELPER_PACKAGE); + return { + waitForIdleTimeoutMs: withDefault( + options.waitForIdleTimeoutMs, + ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, + ), + waitForIdleQuietMs: withDefault( + options.waitForIdleQuietMs, + ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_QUIET_MS, + ), + timeoutMs, + commandTimeoutMs: withDefault( + options.commandTimeoutMs, + timeoutMs + ANDROID_SNAPSHOT_HELPER_COMMAND_OVERHEAD_MS, + ), + maxDepth: withDefault(options.maxDepth, 128), + maxNodes: withDefault(options.maxNodes, 5_000), + packageName, + runner: withDefault(options.instrumentationRunner, `${packageName}/.SnapshotInstrumentation`), + ...(options.outputPath ? { outputPath: options.outputPath } : {}), + }; +} + +function withDefault(value: T | undefined, fallback: T): T { + return value === undefined ? fallback : value; +} + +function buildAndroidSnapshotHelperArgs( + options: AndroidSnapshotHelperResolvedCaptureOptions, +): string[] { + return [ 'shell', 'am', 'instrument', '-w', '-e', 'waitForIdleTimeoutMs', - String(waitForIdleTimeoutMs), + String(options.waitForIdleTimeoutMs), + '-e', + 'waitForIdleQuietMs', + String(options.waitForIdleQuietMs), '-e', 'timeoutMs', - String(timeoutMs), + String(options.timeoutMs), '-e', 'maxDepth', - String(maxDepth), + String(options.maxDepth), '-e', 'maxNodes', - String(maxNodes), - runner, + String(options.maxNodes), + ...(options.outputPath ? ['-e', 'outputPath', options.outputPath] : []), + options.runner, ]; +} - const result = await options.adb(args, { - allowFailure: true, - timeoutMs: commandTimeoutMs, - }); - let output: AndroidSnapshotHelperOutput; +async function readAndroidSnapshotHelperOutput( + options: AndroidSnapshotHelperCaptureOptions, + resolved: AndroidSnapshotHelperResolvedCaptureOptions, + result: Awaited>, +): Promise { try { // The helper can report structured ok=false details even when am exits non-zero. - output = parseAndroidSnapshotHelperOutput(`${result.stdout}\n${result.stderr}`); + return parseAndroidSnapshotHelperOutput(`${result.stdout}\n${result.stderr}`); } catch (error) { - if (error instanceof AppError && result.exitCode !== 0 && error.details?.helper) throw error; - throw new AppError( - 'COMMAND_FAILED', - result.exitCode === 0 - ? 'Android snapshot helper output could not be parsed' - : 'Android snapshot helper failed before returning parseable output', - { - stdout: result.stdout, - stderr: result.stderr, - exitCode: result.exitCode, - }, - error, - ); + return await readFallbackHelperOutputOrThrow(options, resolved, result, error); } - if (result.exitCode !== 0) { - throw new AppError('COMMAND_FAILED', 'Android snapshot helper failed', { +} + +async function readFallbackHelperOutputOrThrow( + options: AndroidSnapshotHelperCaptureOptions, + resolved: AndroidSnapshotHelperResolvedCaptureOptions, + result: Awaited>, + error: unknown, +): Promise { + if (resolved.outputPath) { + const fileOutput = await readHelperOutputFile(options.adb, resolved.outputPath, { + waitForIdleTimeoutMs: resolved.waitForIdleTimeoutMs, + waitForIdleQuietMs: resolved.waitForIdleQuietMs, + timeoutMs: resolved.timeoutMs, + maxDepth: resolved.maxDepth, + maxNodes: resolved.maxNodes, + }); + if (fileOutput) return fileOutput; + } + if (error instanceof AppError && result.exitCode !== 0 && error.details?.helper) throw error; + throw new AppError( + 'COMMAND_FAILED', + result.exitCode === 0 + ? 'Android snapshot helper output could not be parsed' + : 'Android snapshot helper failed before returning parseable output', + { stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, - helper: output.metadata, - }); - } - return output; + }, + error, + ); +} + +async function readHelperOutputFile( + adb: AndroidSnapshotHelperCaptureOptions['adb'], + outputPath: string, + metadata: Omit, +): Promise { + const result = await adb(['shell', 'cat', outputPath], { + allowFailure: true, + timeoutMs: 5_000, + }); + await removeHelperOutputFile(adb, outputPath); + if (result.exitCode !== 0) return undefined; + const xml = result.stdout.trim(); + if (!xml.includes('')) return undefined; + return { + xml, + metadata: { + ...metadata, + outputFormat: ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT, + }, + }; +} + +async function removeHelperOutputFile( + adb: AndroidSnapshotHelperCaptureOptions['adb'], + outputPath: string, +): Promise { + await adb(['shell', 'rm', '-f', outputPath], { + allowFailure: true, + timeoutMs: 5_000, + }); } export function parseAndroidSnapshotHelperOutput(output: string): AndroidSnapshotHelperOutput { @@ -241,6 +344,7 @@ function readHelperMetadata(finalResult: Record): AndroidSnapsho helperApiVersion: finalResult.helperApiVersion, outputFormat: ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT, waitForIdleTimeoutMs: readOptionalNumber(finalResult.waitForIdleTimeoutMs), + waitForIdleQuietMs: readOptionalNumber(finalResult.waitForIdleQuietMs), timeoutMs: readOptionalNumber(finalResult.timeoutMs), maxDepth: readOptionalNumber(finalResult.maxDepth), maxNodes: readOptionalNumber(finalResult.maxNodes), diff --git a/src/platforms/android/snapshot-helper-types.ts b/src/platforms/android/snapshot-helper-types.ts index fede2b5ae..c06430cd6 100644 --- a/src/platforms/android/snapshot-helper-types.ts +++ b/src/platforms/android/snapshot-helper-types.ts @@ -10,6 +10,7 @@ export const ANDROID_SNAPSHOT_HELPER_RUNNER = export const ANDROID_SNAPSHOT_HELPER_PROTOCOL = 'android-snapshot-helper-v1'; export const ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT = 'uiautomator-xml'; export const ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS = 500; +export const ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_QUIET_MS = 100; export const ANDROID_SNAPSHOT_HELPER_COMMAND_OVERHEAD_MS = 5_000; export type { AndroidAdbExecutor } from './adb-executor.ts'; @@ -56,16 +57,19 @@ export type AndroidSnapshotHelperCaptureOptions = { packageName?: string; instrumentationRunner?: string; waitForIdleTimeoutMs?: number; + waitForIdleQuietMs?: number; timeoutMs?: number; commandTimeoutMs?: number; maxDepth?: number; maxNodes?: number; + outputPath?: string; }; export type AndroidSnapshotHelperMetadata = { helperApiVersion?: string; outputFormat: 'uiautomator-xml'; waitForIdleTimeoutMs?: number; + waitForIdleQuietMs?: number; timeoutMs?: number; maxDepth?: number; maxNodes?: number; diff --git a/src/platforms/android/snapshot-types.ts b/src/platforms/android/snapshot-types.ts index 66f072422..809c5f936 100644 --- a/src/platforms/android/snapshot-types.ts +++ b/src/platforms/android/snapshot-types.ts @@ -7,6 +7,7 @@ export type AndroidSnapshotBackendMetadata = { fallbackReason?: string; installReason?: 'missing' | 'outdated' | 'forced' | 'current' | 'skipped'; waitForIdleTimeoutMs?: number; + waitForIdleQuietMs?: number; timeoutMs?: number; maxDepth?: number; maxNodes?: number; diff --git a/src/platforms/android/snapshot.ts b/src/platforms/android/snapshot.ts index d344a71d4..1ee69180d 100644 --- a/src/platforms/android/snapshot.ts +++ b/src/platforms/android/snapshot.ts @@ -32,6 +32,8 @@ import { type AndroidAdbExecutor, type AndroidSnapshotHelperArtifact, type AndroidSnapshotHelperInstallPolicy, + type AndroidSnapshotHelperInstallResult, + type AndroidSnapshotHelperOutput, } from './snapshot-helper.ts'; import { ANDROID_SNAPSHOT_MAX_NODES, @@ -41,7 +43,17 @@ import { const UI_HIERARCHY_DUMP_TIMEOUT_MS = 8_000; const HELPER_INSTALL_TIMEOUT_MS = 30_000; const HELPER_CAPTURE_TIMEOUT_MS = 5_000; -const HELPER_COMMAND_TIMEOUT_MS = 8_000; +const HELPER_COMMAND_TIMEOUT_MS = 30_000; +const RETRYABLE_ADB_STDERR_PATTERNS = [ + 'device offline', + 'device not found', + 'transport error', + 'connection reset', + 'broken pipe', + 'timed out', + 'no such file or directory', +] as const; +const disabledSnapshotHelpers = new Map(); type AndroidSnapshotOptions = SnapshotOptions & { helperArtifact?: AndroidSnapshotHelperArtifact; @@ -107,91 +119,7 @@ async function captureAndroidUiHierarchy( async () => await resolveAndroidSnapshotHelperArtifact(options.helperArtifact), ); if (helper.artifact) { - const helperDeviceKey = getAndroidSnapshotHelperDeviceKey(device); - try { - const adbProvider = resolveAndroidAdbProvider(device, options.helperAdb); - const install = await withDiagnosticTimer( - 'android_snapshot_helper_install', - async () => - await ensureAndroidSnapshotHelper({ - adb, - adbProvider, - artifact: helper.artifact!, - deviceKey: helperDeviceKey, - installPolicy: options.helperInstallPolicy, - timeoutMs: HELPER_INSTALL_TIMEOUT_MS, - }), - { - packageName: helper.artifact.manifest.packageName, - versionCode: helper.artifact.manifest.versionCode, - installPolicy: options.helperInstallPolicy ?? 'missing-or-outdated', - }, - ); - emitDiagnostic({ - phase: 'android_snapshot_helper_install_decision', - data: { - packageName: install.packageName, - versionCode: install.versionCode, - installedVersionCode: install.installedVersionCode, - installed: install.installed, - reason: install.reason, - }, - }); - const capture = await withDiagnosticTimer( - 'android_snapshot_helper_capture', - async () => - await captureAndroidSnapshotWithHelper({ - adb, - packageName: helper.artifact!.manifest.packageName, - instrumentationRunner: helper.artifact!.manifest.instrumentationRunner, - waitForIdleTimeoutMs: - options.helperWaitForIdleTimeoutMs ?? - ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, - timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, - commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, - }), - { - packageName: helper.artifact.manifest.packageName, - version: helper.artifact.manifest.version, - timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, - commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, - }, - ); - return { - xml: capture.xml, - metadata: { - backend: 'android-helper', - helperVersion: helper.artifact.manifest.version, - helperApiVersion: capture.metadata.helperApiVersion, - installReason: install.reason, - waitForIdleTimeoutMs: capture.metadata.waitForIdleTimeoutMs, - timeoutMs: capture.metadata.timeoutMs, - maxDepth: capture.metadata.maxDepth, - maxNodes: capture.metadata.maxNodes, - rootPresent: capture.metadata.rootPresent, - captureMode: capture.metadata.captureMode, - windowCount: capture.metadata.windowCount, - nodeCount: capture.metadata.nodeCount, - helperTruncated: capture.metadata.truncated, - elapsedMs: capture.metadata.elapsedMs, - }, - }; - } catch (error) { - const busyError = formatAndroidSnapshotHelperBusyError(error); - if (busyError) throw busyError; - const fallbackReason = formatAndroidSnapshotHelperFallbackReason(error); - emitDiagnostic({ - level: 'warn', - phase: 'android_snapshot_helper_fallback', - data: { reason: fallbackReason }, - }); - forgetAndroidSnapshotHelperInstall({ - deviceKey: helperDeviceKey, - packageName: helper.artifact.manifest.packageName, - versionCode: helper.artifact.manifest.versionCode, - }); - return await captureStockUiHierarchy(device, fallbackReason, adb); - } + return await captureAndroidUiHierarchyWithHelper(device, options, adb, helper.artifact); } emitDiagnostic({ @@ -202,6 +130,188 @@ async function captureAndroidUiHierarchy( return await captureStockUiHierarchy(device, helper.fallbackReason, adb); } +async function captureAndroidUiHierarchyWithHelper( + device: DeviceInfo, + options: AndroidSnapshotOptions, + adb: AndroidAdbExecutor, + artifact: AndroidSnapshotHelperArtifact, +): Promise<{ xml: string; metadata: AndroidSnapshotBackendMetadata }> { + const helperDeviceKey = getAndroidSnapshotHelperDeviceKey(device); + const helperRuntimeKey = getAndroidSnapshotHelperRuntimeKey(helperDeviceKey, artifact); + const disabledReason = disabledSnapshotHelpers.get(helperRuntimeKey); + if (disabledReason && !options.helperAdb) { + return await captureUiHierarchyWithDisabledHelper(device, disabledReason, adb); + } + + let captureAttempted = false; + try { + const install = await installAndroidSnapshotHelper( + device, + options, + adb, + artifact, + helperDeviceKey, + ); + const capture = await captureAndroidUiHierarchyFromHelper(options, adb, artifact, () => { + captureAttempted = true; + }); + return formatAndroidHelperCaptureResult(capture, artifact, install.reason); + } catch (error) { + return await recoverAndroidHelperCaptureFailure({ + error, + captureAttempted, + options, + helperRuntimeKey, + helperDeviceKey, + artifact, + device, + adb, + }); + } +} + +async function captureUiHierarchyWithDisabledHelper( + device: DeviceInfo, + disabledReason: string, + adb: AndroidAdbExecutor, +): Promise<{ xml: string; metadata: AndroidSnapshotBackendMetadata }> { + emitDiagnostic({ + level: 'warn', + phase: 'android_snapshot_helper_disabled', + data: { reason: disabledReason }, + }); + const disabledBusyError = formatAndroidSnapshotHelperDisabledBusyError(disabledReason); + if (disabledBusyError) throw disabledBusyError; + return await captureStockUiHierarchy(device, disabledReason, adb); +} + +async function installAndroidSnapshotHelper( + device: DeviceInfo, + options: AndroidSnapshotOptions, + adb: AndroidAdbExecutor, + artifact: AndroidSnapshotHelperArtifact, + deviceKey: string, +): Promise { + const install = await withDiagnosticTimer( + 'android_snapshot_helper_install', + async () => + await ensureAndroidSnapshotHelper({ + adb, + adbProvider: resolveAndroidAdbProvider(device, options.helperAdb), + artifact, + deviceKey, + installPolicy: options.helperInstallPolicy, + timeoutMs: HELPER_INSTALL_TIMEOUT_MS, + }), + { + packageName: artifact.manifest.packageName, + versionCode: artifact.manifest.versionCode, + installPolicy: options.helperInstallPolicy ?? 'missing-or-outdated', + }, + ); + emitDiagnostic({ + phase: 'android_snapshot_helper_install_decision', + data: { + packageName: install.packageName, + versionCode: install.versionCode, + installedVersionCode: install.installedVersionCode, + installed: install.installed, + reason: install.reason, + }, + }); + return install; +} + +async function captureAndroidUiHierarchyFromHelper( + options: AndroidSnapshotOptions, + adb: AndroidAdbExecutor, + artifact: AndroidSnapshotHelperArtifact, + onAttempt: () => void, +): Promise { + return await withDiagnosticTimer( + 'android_snapshot_helper_capture', + async () => { + onAttempt(); + return await captureAndroidSnapshotWithHelper({ + adb, + packageName: artifact.manifest.packageName, + instrumentationRunner: artifact.manifest.instrumentationRunner, + waitForIdleTimeoutMs: + options.helperWaitForIdleTimeoutMs ?? ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, + timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, + commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, + }); + }, + { + packageName: artifact.manifest.packageName, + version: artifact.manifest.version, + timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, + commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, + }, + ); +} + +function formatAndroidHelperCaptureResult( + capture: AndroidSnapshotHelperOutput, + artifact: AndroidSnapshotHelperArtifact, + installReason: AndroidSnapshotHelperInstallResult['reason'], +): { xml: string; metadata: AndroidSnapshotBackendMetadata } { + return { + xml: capture.xml, + metadata: { + backend: 'android-helper', + helperVersion: artifact.manifest.version, + helperApiVersion: capture.metadata.helperApiVersion, + installReason, + waitForIdleTimeoutMs: capture.metadata.waitForIdleTimeoutMs, + waitForIdleQuietMs: capture.metadata.waitForIdleQuietMs, + timeoutMs: capture.metadata.timeoutMs, + maxDepth: capture.metadata.maxDepth, + maxNodes: capture.metadata.maxNodes, + rootPresent: capture.metadata.rootPresent, + captureMode: capture.metadata.captureMode, + windowCount: capture.metadata.windowCount, + nodeCount: capture.metadata.nodeCount, + helperTruncated: capture.metadata.truncated, + elapsedMs: capture.metadata.elapsedMs, + }, + }; +} + +async function recoverAndroidHelperCaptureFailure(params: { + error: unknown; + captureAttempted: boolean; + options: AndroidSnapshotOptions; + helperRuntimeKey: string; + helperDeviceKey: string; + artifact: AndroidSnapshotHelperArtifact; + device: DeviceInfo; + adb: AndroidAdbExecutor; +}): Promise<{ xml: string; metadata: AndroidSnapshotBackendMetadata }> { + const busyError = formatAndroidSnapshotHelperBusyError(params.error); + if (busyError) throw busyError; + const fallbackReason = formatAndroidSnapshotHelperFallbackReason(params.error); + emitDiagnostic({ + level: 'warn', + phase: 'android_snapshot_helper_fallback', + data: { reason: fallbackReason }, + }); + if (params.captureAttempted && !params.options.helperAdb) { + disabledSnapshotHelpers.set(params.helperRuntimeKey, fallbackReason); + emitDiagnostic({ + level: 'warn', + phase: 'android_snapshot_helper_disabled', + data: { reason: fallbackReason }, + }); + } + forgetAndroidSnapshotHelperInstall({ + deviceKey: params.helperDeviceKey, + packageName: params.artifact.manifest.packageName, + versionCode: params.artifact.manifest.versionCode, + }); + return await captureStockUiHierarchy(params.device, fallbackReason, params.adb); +} + function formatAndroidSnapshotHelperFallbackReason(error: unknown): string { const normalized = normalizeError(error); const helperMessage = readHelperMessage(normalized.details?.helper); @@ -217,7 +327,13 @@ function formatAndroidSnapshotHelperFallbackReason(error: unknown): string { function formatAndroidSnapshotHelperBusyError(error: unknown): AppError | undefined { const normalized = normalizeError(error); - if (!isStructuredHelperTimeout(normalized.details?.helper, normalized.message)) return undefined; + if ( + !isStructuredHelperTimeout(normalized.details?.helper, normalized.message) && + !isKilledHelperInstrumentationFailure(normalized) && + !isUnsafeStockFallbackHelperReason(normalized.message) + ) { + return undefined; + } const reason = formatAndroidSnapshotHelperFallbackReason(error); const hint = 'Android accessibility snapshots can be blocked by busy or continuously changing app UI. Use screenshot as visual truth after this timeout and report the busy UI if it persists.'; @@ -232,6 +348,31 @@ function formatAndroidSnapshotHelperBusyError(error: unknown): AppError | undefi ); } +function formatAndroidSnapshotHelperDisabledBusyError(reason: string): AppError | undefined { + if (!isUnsafeStockFallbackHelperReason(reason)) return undefined; + const hint = + 'Android accessibility snapshots can be blocked by busy or continuously changing app UI. Use screenshot as visual truth after this timeout and report the busy UI if it persists.'; + return new AppError( + 'COMMAND_FAILED', + `${reason}. Stock UIAutomator fallback was skipped because this usually means the Android accessibility tree is busy or stalled.`, + { hint }, + ); +} + +function isKilledHelperInstrumentationFailure(error: { + message: string; + details?: Record; +}): boolean { + if (error.details?.exitCode !== 137) return false; + return /Android snapshot helper (failed before returning parseable output|output could not be parsed)/.test( + error.message, + ); +} + +function isUnsafeStockFallbackHelperReason(reason: string): boolean { + return /Android snapshot helper output could not be parsed/.test(reason); +} + function readHelperMessage(helper: unknown): string | undefined { if (!helper || typeof helper !== 'object' || !('message' in helper)) return undefined; const message = String(helper.message).trim(); @@ -333,6 +474,18 @@ function getAndroidSnapshotHelperDeviceKey(device: DeviceInfo): string { return `${device.platform}:${device.id}`; } +function getAndroidSnapshotHelperRuntimeKey( + deviceKey: string, + artifact: AndroidSnapshotHelperArtifact, +): string { + return [ + deviceKey, + artifact.manifest.packageName, + artifact.manifest.versionCode, + artifact.manifest.sha256, + ].join(':'); +} + async function deriveScrollableContentHintsIfNeeded( device: DeviceInfo, nodes: RawSnapshotNode[], @@ -390,7 +543,15 @@ async function dumpUiHierarchyOnce(adb: AndroidAdbExecutor): Promise { allowFailure: true, timeoutMs: UI_HIERARCHY_DUMP_TIMEOUT_MS, }); - const actualPath = resolveDumpPath(dumpPath, dumpResult.stdout, dumpResult.stderr); + const reportedPath = readDumpPath(dumpResult.stdout, dumpResult.stderr); + if (dumpResult.exitCode !== 0 && !reportedPath) { + throw new AppError('COMMAND_FAILED', 'uiautomator dump did not produce a fresh XML file', { + stdout: dumpResult.stdout, + stderr: dumpResult.stderr, + exitCode: dumpResult.exitCode, + }); + } + const actualPath = reportedPath ?? dumpPath; const result = await adb(['shell', 'cat', actualPath]); const xml = extractUiDumpXml(result.stdout, result.stderr); @@ -403,10 +564,10 @@ async function dumpUiHierarchyOnce(adb: AndroidAdbExecutor): Promise { return xml; } -function resolveDumpPath(defaultPath: string, stdout: string, stderr: string): string { +function readDumpPath(stdout: string, stderr: string): string | undefined { const text = `${stdout}\n${stderr}`; const match = /dumped to:\s*(\S+)/i.exec(text); - return match?.[1] ?? defaultPath; + return match?.[1]; } function extractUiDumpXml(stdout: string, stderr: string): string | null { @@ -425,14 +586,7 @@ function isRetryableAdbError(err: unknown): boolean { if (err.code !== 'COMMAND_FAILED') return false; const rawStderr = err.details?.stderr; const stderr = (typeof rawStderr === 'string' ? rawStderr : '').toLowerCase(); - if (stderr.includes('device offline')) return true; - if (stderr.includes('device not found')) return true; - if (stderr.includes('transport error')) return true; - if (stderr.includes('connection reset')) return true; - if (stderr.includes('broken pipe')) return true; - if (stderr.includes('timed out')) return true; - if (stderr.includes('no such file or directory')) return true; - return false; + return RETRYABLE_ADB_STDERR_PATTERNS.some((pattern) => stderr.includes(pattern)); } function isUiHierarchyDumpTimeout(err: unknown): err is AppError { @@ -440,14 +594,16 @@ function isUiHierarchyDumpTimeout(err: unknown): err is AppError { if (err.code !== 'COMMAND_FAILED') return false; const timeoutMs = err.details?.timeoutMs; if (typeof timeoutMs !== 'number') return false; - const cmd = err.details?.cmd; - const rawArgs = err.details?.args; + return err.details?.cmd === 'adb' && isUiAutomatorDumpArgs(err.details?.args); +} + +function isUiAutomatorDumpArgs(rawArgs: unknown): boolean { const args = Array.isArray(rawArgs) ? rawArgs.map(String) : typeof rawArgs === 'string' ? rawArgs.split(/\s+/) : []; - return cmd === 'adb' && args.includes('uiautomator') && args.includes('dump'); + return args.includes('uiautomator') && args.includes('dump'); } async function dumpActivityTop( diff --git a/src/platforms/ios/runner-client.ts b/src/platforms/ios/runner-client.ts index 71506ec6f..3ddb7c6b6 100644 --- a/src/platforms/ios/runner-client.ts +++ b/src/platforms/ios/runner-client.ts @@ -28,7 +28,6 @@ import { } from './runner-provider.ts'; import { ensureXctestrun } from './runner-xctestrun.ts'; export { - isReadOnlyRunnerCommand, isRetryableRunnerError, resolveRunnerEarlyExitHint, resolveRunnerBuildFailureHint, diff --git a/src/replay/__tests__/script.test.ts b/src/replay/__tests__/script.test.ts index dba907b43..ac72b639b 100644 --- a/src/replay/__tests__/script.test.ts +++ b/src/replay/__tests__/script.test.ts @@ -290,4 +290,3 @@ test('readReplayScriptMetadata rejects conflicting metadata keys in context head /Conflicting replay test metadata "timeoutMs"/.test(error.message), ); }); - diff --git a/src/replay/vars.ts b/src/replay/vars.ts index 969bca5d0..7f1ce516c 100644 --- a/src/replay/vars.ts +++ b/src/replay/vars.ts @@ -2,7 +2,7 @@ import { AppError } from '../utils/errors.ts'; import type { SessionAction } from '../daemon/types.ts'; export type ReplayVarScope = { - readonly values: Record; + values: Readonly>; }; export type ReplayVarSources = { @@ -57,7 +57,7 @@ export function mergeReplayVarScopeValues( scope: ReplayVarScope, values: Record, ): void { - Object.assign(scope.values, values); + Object.assign(scope.values as Record, values); } export function collectReplayShellEnv(processEnv: NodeJS.ProcessEnv): Record { diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index d0c2ee7c2..4792ae804 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -130,6 +130,26 @@ test('parseArgs recognizes command-specific flag combinations', async () => { assert.equal(parsed.flags.timeoutMs, 240000); }, }, + { + label: 'test maestro suite', + argv: [ + 'test', + './e2e/maestro', + '--maestro', + '--env', + 'APP_ID=com.example', + '--platform', + 'android', + ], + strictFlags: true, + assertParsed: (parsed) => { + assert.equal(parsed.command, 'test'); + assert.deepEqual(parsed.positionals, ['./e2e/maestro']); + assert.equal(parsed.flags.replayMaestro, true); + assert.deepEqual(parsed.flags.replayEnv, ['APP_ID=com.example']); + assert.equal(parsed.flags.platform, 'android'); + }, + }, ]; for (const scenario of scenarios) { @@ -930,6 +950,14 @@ test('usageForCommand includes Maestro replay flag', () => { assert.match(help, /issues\/558/); }); +test('usageForCommand includes Maestro test suite flag', () => { + const help = usageForCommand('test'); + if (help === null) throw new Error('Expected test help text'); + assert.match(help, /Run one or more replay scripts as a serial test suite/); + assert.match(help, /--maestro/); + assert.match(help, /Replay\/Test: inject or override/); +}); + test('usageForCommand resolves workflow help topic', () => { const help = usageForCommand('workflow'); if (help === null) throw new Error('Expected workflow help text'); @@ -1323,7 +1351,7 @@ test('usage renders concise commands inline with descriptions', () => { / metro prepare --public-base-url \| --proxy-base-url ; metro reload\s{2,}Prepare Metro or reload apps/, ); assert.match(help, / batch --steps \| --steps-file \s{2,}Run multiple commands/); - assert.match(help, / test \.\.\.\s{2,}Run \.ad test suites/); + assert.match(help, / test \.\.\.\s{2,}Run replay test suites/); assert.match(help, / session list\s{2,}List active sessions/); assert.doesNotMatch(help, / metro prepare[^\n]*--project-root/); assert.doesNotMatch(help, /\n batch\s{2,}Run multiple commands/); @@ -1334,7 +1362,8 @@ test('command usage describes test suite flags', () => { const help = usageForCommand('test'); if (help === null) throw new Error('Expected command help text'); assert.match(help, /Usage:\s+agent-device test \.\.\./); - assert.match(help, /Run one or more \.ad scripts as a serial test suite/); + assert.match(help, /Run one or more replay scripts as a serial test suite/); + assert.match(help, /--maestro/); assert.match(help, /--fail-fast/); assert.match(help, /--timeout /); assert.match(help, /--retries /); diff --git a/src/utils/__tests__/daemon-client.test.ts b/src/utils/__tests__/daemon-client.test.ts index 9f65f1b74..991544433 100644 --- a/src/utils/__tests__/daemon-client.test.ts +++ b/src/utils/__tests__/daemon-client.test.ts @@ -13,6 +13,7 @@ import { } from '../../__tests__/test-utils/index.ts'; import { runCmdBackground } from '../exec.ts'; import { + canConnectSocket, cleanupFailedDaemonStartupMetadata, computeDaemonCodeSignature, downloadRemoteArtifact, @@ -285,6 +286,40 @@ test('cleanupFailedDaemonStartupMetadata removes stale daemon metadata on timeou } }); +test('canConnectSocket times out stalled local daemon probes', async () => { + const originalCreateConnection = net.createConnection; + let timeoutMs: number | undefined; + let destroyed = false; + const socket = new EventEmitter() as EventEmitter & { + destroy: () => void; + setTimeout: (ms: number) => typeof socket; + }; + socket.destroy = () => { + destroyed = true; + }; + socket.setTimeout = (ms: number) => { + timeoutMs = ms; + setImmediate(() => socket.emit('timeout')); + return socket; + }; + + (net as unknown as { createConnection: typeof net.createConnection }).createConnection = (( + _options: unknown, + _listener?: () => void, + ) => socket) as typeof net.createConnection; + + try { + const reachable = await canConnectSocket(65_530); + + assert.equal(reachable, false); + assert.equal(timeoutMs, 500); + assert.equal(destroyed, true); + } finally { + (net as unknown as { createConnection: typeof net.createConnection }).createConnection = + originalCreateConnection; + } +}); + test('sendToDaemon reuses reachable local socket daemon metadata', async (t) => { if (!(await supportsLoopbackBind())) { t.skip('loopback listeners are not permitted in this environment'); diff --git a/src/utils/__tests__/device.test.ts b/src/utils/__tests__/device.test.ts index b66eb5ad1..0426e52c8 100644 --- a/src/utils/__tests__/device.test.ts +++ b/src/utils/__tests__/device.test.ts @@ -158,4 +158,3 @@ test('resolveDevice returns physical device when explicitly selected by deviceNa }); assert.equal(result.id, 'phys-1'); }); - diff --git a/src/utils/__tests__/selector-is-predicates.test.ts b/src/utils/__tests__/selector-is-predicates.test.ts new file mode 100644 index 000000000..4e3cb9730 --- /dev/null +++ b/src/utils/__tests__/selector-is-predicates.test.ts @@ -0,0 +1,98 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { evaluateIsPredicate } from '../selector-is-predicates.ts'; +import type { SnapshotNode } from '../snapshot.ts'; + +test('visible predicate treats zero-height hittable Android nodes as hidden', () => { + const nodes: SnapshotNode[] = [ + { + index: 0, + ref: 'e0', + type: 'android.widget.FrameLayout', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + ref: 'e1', + parentIndex: 0, + type: 'android.widget.Button', + identifier: 'tab-4', + label: 'Tab 4', + rect: { x: 0, y: 800, width: 100, height: 0 }, + hittable: true, + }, + ]; + + const result = evaluateIsPredicate({ + predicate: 'visible', + node: nodes[1]!, + nodes, + platform: 'android', + }); + + assert.equal(result.pass, false); +}); + +test('visible predicate treats rectless hittable Android nodes as hidden', () => { + const nodes: SnapshotNode[] = [ + { + index: 0, + ref: 'e0', + type: 'android.widget.FrameLayout', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + ref: 'e1', + type: 'android.widget.Button', + label: 'Library', + hittable: true, + }, + ]; + + const result = evaluateIsPredicate({ + predicate: 'visible', + node: nodes[1]!, + nodes, + platform: 'android', + }); + + assert.equal(result.pass, false); +}); + +test('visible predicate uses visible Android ancestor geometry for rectless text', () => { + const nodes: SnapshotNode[] = [ + { + index: 0, + ref: 'e0', + type: 'android.widget.FrameLayout', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + ref: 'e1', + parentIndex: 0, + type: 'android.widget.Button', + label: 'Library', + rect: { x: 20, y: 100, width: 160, height: 80 }, + hittable: true, + }, + { + index: 2, + ref: 'e2', + parentIndex: 1, + type: 'android.widget.TextView', + label: 'Library', + hittable: false, + }, + ]; + + const result = evaluateIsPredicate({ + predicate: 'visible', + node: nodes[2]!, + nodes, + platform: 'android', + }); + + assert.equal(result.pass, true); +}); diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index d4decd992..f94fbd3c1 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -358,9 +358,11 @@ React Native dev loop: There is no open-url command; use open with the URL target or host + URL form. Direct iOS URL open remains valid when no host shell is known, but verify that the app UI loaded: agent-device open exp://127.0.0.1:8081 --platform ios - Android uses the URL target directly; do not write open there: + Android Expo URLs can be opened directly when no specific app package must be forced: agent-device open exp://127.0.0.1:8081 --platform android - Android URL/deep-link opens infer the foreground package after launch when possible, so logs/perf can remain package-bound. If perf still says no package is associated, open the host package/app id first, then open the URL in the same session. + Android app-bound deep links can use open when a known package must handle the link: + agent-device open com.example.app example://screen --platform android + Android URL/deep-link opens infer the foreground package after launch when possible, so logs/perf can remain package-bound. If perf still says no package is associated, use the app-bound form when the package id is known. If apps lookup misses the project but shows Expo Go/dev-client and a project URL is available, open the URL/host shell; if no URL is available, ask instead of inventing an app id. Expo Dev Client/development builds: open the installed dev-client app id/name; if a dev-client URL is provided, open that URL next. For Metro setup use metro prepare --kind expo. @@ -1263,7 +1265,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--maestro', usageDescription: - 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with Apple-platform launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible, ordered trusted runScript file/env steps with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional, index, childOf, label, and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, eraseText for focused fields, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe plus swipe.label, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + + 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with Apple-platform launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible/true, ordered trusted runScript file/env steps with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional, index, childOf, label, and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, eraseText for focused fields, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe plus swipe.label, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + 'Unsupported syntax fails loudly with a link to https://github.com/callstackincubator/agent-device/issues/558', }, { @@ -1683,12 +1685,13 @@ const COMMAND_SCHEMAS: Record = { test: { usageOverride: 'test ...', listUsageOverride: 'test ...', - helpDescription: 'Run one or more .ad scripts as a serial test suite', - summary: 'Run .ad test suites', + helpDescription: 'Run one or more replay scripts as a serial test suite', + summary: 'Run replay test suites', positionalArgs: ['pathOrGlob'], allowsExtraPositionals: true, allowedFlags: [ 'replayUpdate', + 'replayMaestro', 'replayEnv', 'failFast', 'timeoutMs', diff --git a/src/utils/keyboard-actions.ts b/src/utils/keyboard-actions.ts new file mode 100644 index 000000000..92fa204e6 --- /dev/null +++ b/src/utils/keyboard-actions.ts @@ -0,0 +1,7 @@ +const KEYBOARD_ACTIONS = ['status', 'get', 'dismiss', 'enter', 'return'] as const; + +export type KeyboardAction = (typeof KEYBOARD_ACTIONS)[number]; + +export function isKeyboardAction(action: string): action is KeyboardAction { + return KEYBOARD_ACTIONS.includes(action as KeyboardAction); +} diff --git a/src/utils/selector-is-predicates.ts b/src/utils/selector-is-predicates.ts index 9c369f548..0d94a6ad9 100644 --- a/src/utils/selector-is-predicates.ts +++ b/src/utils/selector-is-predicates.ts @@ -21,7 +21,8 @@ export function evaluateIsPredicate(params: { const actualText = extractNodeText(node); const editable = isNodeEditable(node, platform); const selected = node.selected === true; - const visible = predicate === 'text' ? isNodeVisible(node) : isAssertionVisible(node, nodes); + const visible = + predicate === 'text' ? isNodeVisible(node) : isAssertionVisible(node, nodes, platform); let pass = false; switch (predicate) { case 'visible': @@ -54,14 +55,14 @@ export function evaluateIsPredicate(params: { function isAssertionVisible( node: SnapshotState['nodes'][number], nodes: SnapshotState['nodes'], + platform: Platform, ): boolean { - if (node.hittable === true) return true; if (hasPositiveRect(node.rect)) return isRectVisibleInViewport(node, nodes); if (node.rect) return false; + if (platform !== 'android' && node.hittable === true) return true; const anchor = resolveVisibilityAnchor(node, nodes); if (!anchor) return false; - if (anchor.hittable === true) return true; - if (!hasPositiveRect(anchor.rect)) return false; + if (!hasPositiveRect(anchor.rect)) return platform !== 'android' && anchor.hittable === true; return isRectVisibleInViewport(anchor, nodes); } diff --git a/src/utils/snapshot-label-signals.ts b/src/utils/snapshot-label-signals.ts index 19f9d8b0c..2f4727078 100644 --- a/src/utils/snapshot-label-signals.ts +++ b/src/utils/snapshot-label-signals.ts @@ -4,6 +4,6 @@ export function normalizeRepeatedNodeLabel(label: string): string | null { return normalized; } -export function isEmailLikeLabel(label: string): boolean { +function isEmailLikeLabel(label: string): boolean { return /\S+@\S+\.\S+/.test(label); } diff --git a/test/integration/provider-scenarios/android-test-suite.test.ts b/test/integration/provider-scenarios/android-test-suite.test.ts index 70a8ea9c5..45be5e6e0 100644 --- a/test/integration/provider-scenarios/android-test-suite.test.ts +++ b/test/integration/provider-scenarios/android-test-suite.test.ts @@ -68,3 +68,72 @@ test('Provider-backed integration Android replay test suite covers retries and f assert.equal(failFastSuite.notRun, 1, JSON.stringify(failFastSuite)); }); }); + +test('Provider-backed integration Android Maestro replay uses fresh selector snapshots and content-lane swipes', async () => { + let snapshots = 0; + await withProviderScenarioResource( + async () => + await createAndroidSettingsWorld({ + snapshotXml: () => { + snapshots += 1; + return androidMaestroReplayXml( + snapshots === 1 ? '[16,24][374,80]' : '[100,300][260,360]', + ); + }, + }), + async (world) => { + const client = world.daemon.client(); + const suiteRoot = path.join(world.tempRoot, 'suite-maestro'); + fs.mkdirSync(suiteRoot, { recursive: true }); + const flowPath = path.join(suiteRoot, 'maestro-flow.yaml'); + fs.writeFileSync( + flowPath, + [ + 'appId: com.android.settings', + '---', + '- launchApp', + '- assertVisible: Apps', + '- tapOn: Search', + '- swipe:', + ' start: 90%, 50%', + ' end: 10%, 50%', + ' duration: 300', + '', + ].join('\n'), + ); + + const suite = await client.replay.test({ + paths: [flowPath], + backend: 'maestro', + artifactsDir: path.join(suiteRoot, 'artifacts'), + timeoutMs: 30000, + ...world.selection, + }); + + assert.equal(suite.total, 1, JSON.stringify(suite)); + assert.equal(suite.passed, 1, JSON.stringify(suite)); + assert.equal(suite.failed, 0, JSON.stringify(suite)); + assert.deepEqual( + world.adbCalls.find((call) => call.slice(0, 3).join(' ') === 'shell input tap'), + ['shell', 'input', 'tap', '180', '330'], + ); + assert.deepEqual( + world.adbCalls.find((call) => call.slice(0, 3).join(' ') === 'shell input swipe'), + ['shell', 'input', 'swipe', '351', '390', '39', '390', '300'], + ); + assert.equal(snapshots >= 2, true); + }, + ); +}); + +function androidMaestroReplayXml(searchBounds: string): string { + return [ + '', + '', + ' ', + ' ', + ` `, + ' ', + '', + ].join('\n'); +} diff --git a/test/integration/provider-scenarios/android-world.ts b/test/integration/provider-scenarios/android-world.ts index 67aa25f6b..d151745b5 100644 --- a/test/integration/provider-scenarios/android-world.ts +++ b/test/integration/provider-scenarios/android-world.ts @@ -320,6 +320,13 @@ function androidCaptureAdbResult( searchText: string, snapshotXml?: () => string, ): AndroidAdbResult | undefined { + if (key.startsWith('shell am instrument ')) { + return { + stdout: androidSnapshotHelperOutput(snapshotXml?.() ?? androidSettingsXml(searchText)), + stderr: '', + exitCode: 0, + }; + } if (key === 'exec-out uiautomator dump /dev/tty') { return { stdout: snapshotXml?.() ?? androidSettingsXml(searchText), @@ -333,6 +340,22 @@ function androidCaptureAdbResult( return undefined; } +function androidSnapshotHelperOutput(xml: string): string { + return [ + 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', + 'INSTRUMENTATION_STATUS: helperApiVersion=1', + 'INSTRUMENTATION_STATUS: outputFormat=uiautomator-xml', + 'INSTRUMENTATION_STATUS: chunkIndex=0', + 'INSTRUMENTATION_STATUS: chunkCount=1', + `INSTRUMENTATION_STATUS: payloadBase64=${Buffer.from(xml, 'utf8').toString('base64')}`, + 'INSTRUMENTATION_STATUS_CODE: 1', + 'INSTRUMENTATION_RESULT: agentDeviceProtocol=android-snapshot-helper-v1', + 'INSTRUMENTATION_RESULT: helperApiVersion=1', + 'INSTRUMENTATION_RESULT: ok=true', + 'INSTRUMENTATION_CODE: 0', + ].join('\n'); +} + export function androidSettingsXml( searchText: string, options: { duplicateAppsRow?: boolean } = {}, diff --git a/test/skillgym/suites/agent-device-smoke-suite.ts b/test/skillgym/suites/agent-device-smoke-suite.ts index 41d7244de..a599d244d 100644 --- a/test/skillgym/suites/agent-device-smoke-suite.ts +++ b/test/skillgym/suites/agent-device-smoke-suite.ts @@ -1159,7 +1159,7 @@ const SKILL_GUIDANCE_CASES: Case[] = [ 'Platform: Android', 'Launch context: Expo Go', 'Project URL: exp://10.0.2.2:8081', - 'Android does not support open ; use a URL target for deep links', + 'Android Expo URLs can be opened directly when no specific app package must be forced', ], task: 'Plan the command to launch the Expo project on Android using the project URL.', outputs: [plannedCommand('open'), /exp:\/\/10\.0\.2\.2:8081/i, /--platform android/i], diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index a42384dcd..fd44fc32c 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -61,11 +61,11 @@ Maestro compatibility translates supported YAML commands into Agent Device repla - Supported and unsupported capabilities: https://github.com/callstackincubator/agent-device/issues/558 - New focused compatibility request: https://github.com/callstackincubator/agent-device/issues/new -Currently supported areas include app launch with Apple-platform launch arguments and iOS simulator `clearState`, file and inline `runFlow` with `when.platform`, `when.visible`, and `when.notVisible`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional`, `index`, `childOf`, `label`, and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, focused-field `eraseText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, `swipe.label`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as an ordered Maestro compatibility step for trusted file/env scripts that use `http.post`, `json`, and `output` variables; it can make network requests, and is not a native `.ad` command or security sandbox. Script execution uses Node `vm` only for compatibility isolation, not for security; the script timeout bounds synchronous execution, while `http.post` requests are bounded by the helper process timeout. Output keys cannot contain `.` because exported variables are addressed as `output.`. +Currently supported areas include app launch with Apple-platform launch arguments and iOS simulator `clearState`, file and inline `runFlow` with `when.platform`, `when.visible`, `when.notVisible`, and limited `when.true` boolean/platform expressions, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional`, `index`, `childOf`, `label`, and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, focused-field `eraseText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, `swipe.label`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as an ordered Maestro compatibility step for trusted file/env scripts that use `http.post`, `json`, and `output` variables; it can make network requests, and is not a native `.ad` command or security sandbox. Script execution uses Node `vm` only for compatibility isolation, not for security; the script timeout bounds synchronous execution, while `http.post` requests are bounded by the helper process timeout. Output keys cannot contain `.` because exported variables are addressed as `output.`. Maestro `env` values use the same replay precedence as `.ad` files: flow `env` is the default, shell `AD_VAR_*` values override it, and CLI `-e KEY=VALUE` wins over both. -Unsupported Maestro features such as `repeat.while`, `runFlow.when.true`, full expression predicates, `evalScript`, device utility commands, Android app launch arguments, and Android app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. +Unsupported Maestro features such as `repeat.while`, full expression predicates beyond boolean literals and `maestro.platform` comparisons, `evalScript`, device utility commands, Android app launch arguments, and Android app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. ## Run a lightweight `.ad` suite From 8d5a1167d64d1d9b54884812a7c4fd019c491004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 27 May 2026 20:46:39 +0200 Subject: [PATCH 8/8] fix: stabilize Android provider snapshot tests --- .../__tests__/session-replay-vars.test.ts | 555 ------------------ .../android/__tests__/snapshot.test.ts | 3 +- src/platforms/android/snapshot.ts | 78 +-- .../provider-scenarios/android-find.test.ts | 12 +- .../android-lifecycle.test.ts | 2 + .../provider-scenarios/android-world.ts | 2 +- 6 files changed, 21 insertions(+), 631 deletions(-) diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index 3872a22d7..d5fbc0d69 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -693,34 +693,6 @@ test('runReplayScriptFile retries Maestro scrollUntilVisible with scroll probes' ); }); -test('runReplayScriptFile lets Maestro scrollUntilVisible use fuzzy visible text matching', async () => { - const calls: CapturedInvocation[] = []; - const { response } = await runReplayFixture({ - label: 'maestro-scroll-until-visible-fuzzy-text', - script: ['appId: demo.app', '---', '- scrollUntilVisible:', ' element: Discover', ''].join( - '\n', - ), - flags: { replayBackend: 'maestro' }, - invoke: async (req) => { - calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'find') return { ok: true, data: { found: true } }; - return { - ok: false, - error: { code: 'COMMAND_FAILED', message: 'wait timed out' }, - }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual( - calls.map((call) => [call.command, call.positionals]), - [ - ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], - ['find', ['Discover', 'wait', '500']], - ], - ); -}); - test('runReplayScriptFile lets Maestro tapOn use fuzzy visible text matching', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ @@ -810,413 +782,6 @@ test('runReplayScriptFile reuses successful Maestro visibility snapshot for foll ); }); -test('runReplayScriptFile lets exact Maestro text tapOn win before fuzzy matching', async () => { - const calls: CapturedInvocation[] = []; - const { response } = await runReplayFixture({ - label: 'maestro-tap-visible-text-exact-before-fuzzy', - script: ['appId: demo.app', '---', '- tapOn: Mute accounts', ''].join('\n'), - flags: { replayBackend: 'maestro' }, - invoke: async (req) => { - calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'snapshot') { - return { - ok: true, - data: { - nodes: [ - { - index: 1, - label: 'Block accounts', - rect: { x: 10, y: 600, width: 240, height: 44 }, - }, - { - index: 2, - label: 'Mute accounts', - rect: { x: 10, y: 540, width: 240, height: 44 }, - }, - ], - }, - }; - } - return { ok: true, data: {} }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual( - calls.map((call) => [call.command, call.positionals]), - [ - ['snapshot', []], - ['click', ['130', '562']], - ], - ); -}); - -test('runReplayScriptFile prefers exact Maestro text before actionable target type', async () => { - const calls: CapturedInvocation[] = []; - const { response } = await runReplayFixture({ - label: 'maestro-tap-visible-text-exact-before-type-rank', - script: ['appId: demo.app', '---', '- tapOn: Search', ''].join('\n'), - flags: { platform: 'android', replayBackend: 'maestro' }, - invoke: async (req) => { - calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'snapshot') { - return { - ok: true, - data: { - nodes: [ - { - index: 1, - type: 'android.widget.Button', - label: 'search', - rect: { x: 673, y: 165, width: 132, height: 132 }, - }, - { - index: 2, - type: 'android.widget.FrameLayout', - label: 'Search', - rect: { x: 810, y: 2054, width: 270, height: 220 }, - }, - ], - metadata: { referenceWidth: 1080, referenceHeight: 2340 }, - }, - }; - } - return { ok: true, data: {} }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual( - calls.map((call) => [call.command, call.positionals]), - [ - ['snapshot', []], - ['click', ['945', '2164']], - ], - ); -}); - -test('runReplayScriptFile prefers fuller same-type Maestro text matches', async () => { - const calls: CapturedInvocation[] = []; - const { response } = await runReplayFixture({ - label: 'maestro-tap-visible-text-prefers-fuller-same-type', - script: ['appId: demo.app', '---', '- tapOn: Open details', ''].join('\n'), - flags: { replayBackend: 'maestro' }, - invoke: async (req) => { - calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'snapshot') { - return { - ok: true, - data: { - nodes: [ - { - index: 1, - type: 'android.widget.Button', - label: 'Open details', - rect: { x: 0, y: 352, width: 109, height: 110 }, - }, - { - index: 2, - type: 'android.widget.Button', - label: 'Open details', - rect: { x: 33, y: 433, width: 340, height: 110 }, - }, - ], - metadata: { referenceWidth: 1080, referenceHeight: 2340 }, - }, - }; - } - return { ok: true, data: {} }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual( - calls.map((call) => [call.command, call.positionals]), - [ - ['snapshot', []], - ['click', ['203', '488']], - ], - ); -}); - -test('runReplayScriptFile ignores zero-size Maestro tapOn matches', async () => { - const calls: CapturedInvocation[] = []; - const { response } = await runReplayFixture({ - label: 'maestro-tap-visible-text-ignores-zero-size', - script: ['appId: demo.app', '---', '- tapOn: Retain', ''].join('\n'), - flags: { replayBackend: 'maestro' }, - invoke: async (req) => { - calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'snapshot') { - return { - ok: true, - data: { - nodes: [ - { - index: 1, - type: 'android.widget.ScrollView', - rect: { x: 0, y: 319, width: 816, height: 2021 }, - }, - { - index: 2, - parentIndex: 1, - type: 'android.widget.Button', - label: 'Retain', - rect: { x: 0, y: 2340, width: 771, height: 0 }, - }, - { - index: 3, - parentIndex: 2, - type: 'android.widget.TextView', - label: 'Retain', - }, - { - index: 4, - type: 'android.widget.Button', - label: 'Retain', - rect: { x: 418, y: 1385, width: 244, height: 110 }, - }, - ], - metadata: { referenceWidth: 1080, referenceHeight: 2340 }, - }, - }; - } - return { ok: true, data: {} }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual( - calls.map((call) => [call.command, call.positionals]), - [ - ['snapshot', []], - ['click', ['540', '1440']], - ], - ); -}); - -test('runReplayScriptFile prefers own-rect Maestro tapOn matches over inherited overlay rects', async () => { - const calls: CapturedInvocation[] = []; - const { response } = await runReplayFixture({ - label: 'maestro-tap-visible-text-prefers-own-rect', - script: ['appId: demo.app', '---', '- tapOn: Library', ''].join('\n'), - flags: { replayBackend: 'maestro' }, - invoke: async (req) => { - calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'snapshot') { - return { - ok: true, - data: { - nodes: [ - { - index: 1, - type: 'android.view.View', - label: 'Library', - rect: { x: 585, y: 319, width: 222, height: 132 }, - }, - { - index: 2, - type: 'android.view.ViewGroup', - rect: { x: 0, y: 0, width: 816, height: 2340 }, - }, - { - index: 3, - parentIndex: 2, - type: 'android.view.ViewGroup', - }, - { - index: 4, - parentIndex: 3, - type: 'android.widget.TextView', - label: 'Library', - }, - ], - metadata: { referenceWidth: 1080, referenceHeight: 2340 }, - }, - }; - } - return { ok: true, data: {} }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual( - calls.map((call) => [call.command, call.positionals]), - [ - ['snapshot', []], - ['click', ['696', '385']], - ], - ); -}); - -test('runReplayScriptFile lets Maestro text tapOn match regex labels', async () => { - const calls: CapturedInvocation[] = []; - const { response } = await runReplayFixture({ - label: 'maestro-tap-visible-text-regex', - script: ['appId: demo.app', '---', "- tapOn: '.*Featured.*'", ''].join('\n'), - flags: { replayBackend: 'maestro' }, - invoke: async (req) => { - calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'snapshot') { - return { - ok: true, - data: { - nodes: [ - { - index: 1, - type: 'button', - label: 'Featured, selected tab', - rect: { x: 20, y: 720, width: 110, height: 44 }, - }, - ], - }, - }; - } - return { ok: true, data: {} }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual( - calls.map((call) => [call.command, call.positionals]), - [ - ['snapshot', []], - ['click', ['75', '742']], - ], - ); -}); - -test('runReplayScriptFile prefers on-screen Maestro text tapOn matches', async () => { - const calls: CapturedInvocation[] = []; - const { response } = await runReplayFixture({ - label: 'maestro-tap-visible-text-onscreen', - script: ['appId: demo.app', '---', '- tapOn: Sign in', ''].join('\n'), - flags: { replayBackend: 'maestro' }, - invoke: async (req) => { - calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'snapshot') { - return { - ok: true, - data: { - nodes: [ - { - index: 1, - type: 'Button', - label: 'Sign in', - rect: { x: -328, y: 182, width: 328, height: 42 }, - }, - { - index: 2, - type: 'Button', - label: 'Sign in', - rect: { x: 56, y: 842, width: 328, height: 56 }, - }, - ], - metadata: { referenceWidth: 440, referenceHeight: 956 }, - }, - }; - } - return { ok: true, data: {} }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual( - calls.map((call) => [call.command, call.positionals]), - [ - ['snapshot', []], - ['click', ['220', '870']], - ], - ); -}); - -test('runReplayScriptFile taps Maestro text near the label in large action containers', async () => { - const calls: CapturedInvocation[] = []; - const { response } = await runReplayFixture({ - label: 'maestro-tap-visible-text-action-container', - script: ['appId: demo.app', '---', '- tapOn: Mute accounts', ''].join('\n'), - flags: { replayBackend: 'maestro' }, - invoke: async (req) => { - calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'snapshot') { - return { - ok: true, - data: { - nodes: [ - { - index: 1, - type: 'ScrollView', - label: 'Mute accounts', - rect: { x: 8, y: 805, width: 424, height: 93 }, - }, - { - index: 2, - parentIndex: 1, - type: 'Other', - label: 'Block accounts', - rect: { x: 31, y: 835, width: 377, height: 42 }, - }, - ], - }, - }; - } - return { ok: true, data: {} }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual( - calls.map((call) => [call.command, call.positionals]), - [ - ['snapshot', []], - ['click', ['92', '829']], - ], - ); -}); - -test('runReplayScriptFile prefers actionable Maestro tapOn matches over broad ancestors', async () => { - const calls: CapturedInvocation[] = []; - const { response } = await runReplayFixture({ - label: 'maestro-tap-prefers-actionable-match', - script: ['appId: demo.app', '---', '- tapOn: New list', ''].join('\n'), - flags: { replayBackend: 'maestro' }, - invoke: async (req) => { - calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'snapshot') { - return { - ok: true, - data: { - nodes: [ - { - index: 1, - type: 'Other', - label: 'New list', - rect: { x: 0, y: 0, width: 440, height: 956 }, - }, - { - index: 2, - type: 'Button', - label: 'New list', - rect: { x: 349, y: 67, width: 75, height: 33 }, - }, - ], - }, - }; - } - return { ok: true, data: {} }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual( - calls.map((call) => [call.command, call.positionals]), - [ - ['snapshot', []], - ['click', ['387', '84']], - ], - ); -}); - test('runReplayScriptFile treats absent Maestro assertNotVisible targets as passing', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ @@ -1276,57 +841,6 @@ test('runReplayScriptFile propagates Maestro assertNotVisible infrastructure fai assert.equal(calls.length, 1); }); -test('runReplayScriptFile treats duplicate visible Maestro assertVisible matches as passing', async () => { - const { response, calls } = await runReplayFixture({ - label: 'maestro-assert-visible-duplicate-visible', - script: ['appId: demo.app', '---', '- assertVisible: Article', ''].join('\n'), - flags: { platform: 'android', replayBackend: 'maestro' }, - invoke: async (req) => { - if (req.command === 'snapshot') { - return { - ok: true, - data: { - nodes: [ - { - index: 0, - type: 'application', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'statictext', - label: 'Article', - rect: { x: 16, y: 100, width: 120, height: 24 }, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'statictext', - label: 'Article', - rect: { x: 16, y: 140, width: 120, height: 24 }, - }, - ], - }, - }; - } - return { - ok: false, - error: { code: 'COMMAND_FAILED', message: 'unexpected command' }, - }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual( - calls.map((call) => [call.command, call.positionals]), - [['snapshot', []]], - ); - assert.equal(calls[0]?.flags?.snapshotRaw, true); -}); - test('runReplayScriptFile waits briefly for Maestro assertNotVisible to stabilize', async () => { const calls: CapturedInvocation[] = []; let visibleChecks = 0; @@ -1961,48 +1475,6 @@ test('runReplayScriptFile retries Maestro retry commands until they pass', async assert.equal(calls.filter((call) => call.command === 'snapshot').length > 1, true); }); -test('runReplayScriptFile skips Maestro runFlow.when.visible commands on false predicate', async () => { - const calls: CapturedInvocation[] = []; - const { response } = await runReplayFixture({ - label: 'maestro-run-flow-when-visible-false-skip', - script: [ - 'appId: demo.app', - '---', - '- runFlow:', - ' when:', - ' visible: Continue', - ' commands:', - ' - tapOn: Continue', - '', - ].join('\n'), - flags: { replayBackend: 'maestro' }, - invoke: async (req) => { - calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'snapshot') { - return { - ok: true, - data: { - nodes: [ - { - index: 0, - type: 'application', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - ], - }, - }; - } - return { ok: true, data: { pass: false } }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual( - calls.map((call) => [call.command, call.positionals]), - [['snapshot', []]], - ); -}); - test('runReplayScriptFile propagates Maestro runFlow.when runtime errors', async () => { const { response } = await runReplayFixture({ label: 'maestro-run-flow-when-visible-runtime-error', @@ -2030,33 +1502,6 @@ test('runReplayScriptFile propagates Maestro runFlow.when runtime errors', async } }); -test('runReplayScriptFile propagates Maestro runFlow.when COMMAND_FAILED errors without condition-miss details', async () => { - const { response } = await runReplayFixture({ - label: 'maestro-run-flow-when-visible-command-error', - script: [ - 'appId: demo.app', - '---', - '- runFlow:', - ' when:', - ' visible: Continue', - ' commands:', - ' - tapOn: Continue', - '', - ].join('\n'), - flags: { replayBackend: 'maestro' }, - invoke: async () => ({ - ok: false, - error: { code: 'COMMAND_FAILED', message: 'snapshot failed' }, - }), - }); - - assert.equal(response.ok, false); - if (!response.ok) { - assert.equal(response.error.code, 'COMMAND_FAILED'); - assert.match(response.error.message, /snapshot failed/); - } -}); - test('runReplayScriptFile runs Maestro runFlow.when.visible commands when present', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index 215c01b6b..a592add21 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -837,7 +837,8 @@ test('dumpUiHierarchy does not read a stale fallback file when dump fails withou (error: unknown) => error instanceof AppError && error.code === 'COMMAND_FAILED' && - error.message.includes('did not produce a fresh XML file'), + error.message.includes('did not return XML') && + error.details?.reason === 'missing_fresh_dump', ); }); diff --git a/src/platforms/android/snapshot.ts b/src/platforms/android/snapshot.ts index 1ee69180d..8663444d7 100644 --- a/src/platforms/android/snapshot.ts +++ b/src/platforms/android/snapshot.ts @@ -53,7 +53,6 @@ const RETRYABLE_ADB_STDERR_PATTERNS = [ 'timed out', 'no such file or directory', ] as const; -const disabledSnapshotHelpers = new Map(); type AndroidSnapshotOptions = SnapshotOptions & { helperArtifact?: AndroidSnapshotHelperArtifact; @@ -137,13 +136,6 @@ async function captureAndroidUiHierarchyWithHelper( artifact: AndroidSnapshotHelperArtifact, ): Promise<{ xml: string; metadata: AndroidSnapshotBackendMetadata }> { const helperDeviceKey = getAndroidSnapshotHelperDeviceKey(device); - const helperRuntimeKey = getAndroidSnapshotHelperRuntimeKey(helperDeviceKey, artifact); - const disabledReason = disabledSnapshotHelpers.get(helperRuntimeKey); - if (disabledReason && !options.helperAdb) { - return await captureUiHierarchyWithDisabledHelper(device, disabledReason, adb); - } - - let captureAttempted = false; try { const install = await installAndroidSnapshotHelper( device, @@ -152,16 +144,11 @@ async function captureAndroidUiHierarchyWithHelper( artifact, helperDeviceKey, ); - const capture = await captureAndroidUiHierarchyFromHelper(options, adb, artifact, () => { - captureAttempted = true; - }); + const capture = await captureAndroidUiHierarchyFromHelper(options, adb, artifact); return formatAndroidHelperCaptureResult(capture, artifact, install.reason); } catch (error) { return await recoverAndroidHelperCaptureFailure({ error, - captureAttempted, - options, - helperRuntimeKey, helperDeviceKey, artifact, device, @@ -170,21 +157,6 @@ async function captureAndroidUiHierarchyWithHelper( } } -async function captureUiHierarchyWithDisabledHelper( - device: DeviceInfo, - disabledReason: string, - adb: AndroidAdbExecutor, -): Promise<{ xml: string; metadata: AndroidSnapshotBackendMetadata }> { - emitDiagnostic({ - level: 'warn', - phase: 'android_snapshot_helper_disabled', - data: { reason: disabledReason }, - }); - const disabledBusyError = formatAndroidSnapshotHelperDisabledBusyError(disabledReason); - if (disabledBusyError) throw disabledBusyError; - return await captureStockUiHierarchy(device, disabledReason, adb); -} - async function installAndroidSnapshotHelper( device: DeviceInfo, options: AndroidSnapshotOptions, @@ -226,13 +198,11 @@ async function captureAndroidUiHierarchyFromHelper( options: AndroidSnapshotOptions, adb: AndroidAdbExecutor, artifact: AndroidSnapshotHelperArtifact, - onAttempt: () => void, ): Promise { return await withDiagnosticTimer( 'android_snapshot_helper_capture', - async () => { - onAttempt(); - return await captureAndroidSnapshotWithHelper({ + async () => + await captureAndroidSnapshotWithHelper({ adb, packageName: artifact.manifest.packageName, instrumentationRunner: artifact.manifest.instrumentationRunner, @@ -240,8 +210,7 @@ async function captureAndroidUiHierarchyFromHelper( options.helperWaitForIdleTimeoutMs ?? ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, - }); - }, + }), { packageName: artifact.manifest.packageName, version: artifact.manifest.version, @@ -280,9 +249,6 @@ function formatAndroidHelperCaptureResult( async function recoverAndroidHelperCaptureFailure(params: { error: unknown; - captureAttempted: boolean; - options: AndroidSnapshotOptions; - helperRuntimeKey: string; helperDeviceKey: string; artifact: AndroidSnapshotHelperArtifact; device: DeviceInfo; @@ -296,14 +262,6 @@ async function recoverAndroidHelperCaptureFailure(params: { phase: 'android_snapshot_helper_fallback', data: { reason: fallbackReason }, }); - if (params.captureAttempted && !params.options.helperAdb) { - disabledSnapshotHelpers.set(params.helperRuntimeKey, fallbackReason); - emitDiagnostic({ - level: 'warn', - phase: 'android_snapshot_helper_disabled', - data: { reason: fallbackReason }, - }); - } forgetAndroidSnapshotHelperInstall({ deviceKey: params.helperDeviceKey, packageName: params.artifact.manifest.packageName, @@ -348,17 +306,6 @@ function formatAndroidSnapshotHelperBusyError(error: unknown): AppError | undefi ); } -function formatAndroidSnapshotHelperDisabledBusyError(reason: string): AppError | undefined { - if (!isUnsafeStockFallbackHelperReason(reason)) return undefined; - const hint = - 'Android accessibility snapshots can be blocked by busy or continuously changing app UI. Use screenshot as visual truth after this timeout and report the busy UI if it persists.'; - return new AppError( - 'COMMAND_FAILED', - `${reason}. Stock UIAutomator fallback was skipped because this usually means the Android accessibility tree is busy or stalled.`, - { hint }, - ); -} - function isKilledHelperInstrumentationFailure(error: { message: string; details?: Record; @@ -469,23 +416,9 @@ function enrichStockSnapshotFailureWithHelperReason( } function getAndroidSnapshotHelperDeviceKey(device: DeviceInfo): string { - // Emulator serials are port-based and can be reused after restart; capture failure invalidates - // this key before falling back so stale process-local entries self-heal on the next snapshot. return `${device.platform}:${device.id}`; } -function getAndroidSnapshotHelperRuntimeKey( - deviceKey: string, - artifact: AndroidSnapshotHelperArtifact, -): string { - return [ - deviceKey, - artifact.manifest.packageName, - artifact.manifest.versionCode, - artifact.manifest.sha256, - ].join(':'); -} - async function deriveScrollableContentHintsIfNeeded( device: DeviceInfo, nodes: RawSnapshotNode[], @@ -545,10 +478,11 @@ async function dumpUiHierarchyOnce(adb: AndroidAdbExecutor): Promise { }); const reportedPath = readDumpPath(dumpResult.stdout, dumpResult.stderr); if (dumpResult.exitCode !== 0 && !reportedPath) { - throw new AppError('COMMAND_FAILED', 'uiautomator dump did not produce a fresh XML file', { + throw new AppError('COMMAND_FAILED', 'uiautomator dump did not return XML', { stdout: dumpResult.stdout, stderr: dumpResult.stderr, exitCode: dumpResult.exitCode, + reason: 'missing_fresh_dump', }); } const actualPath = reportedPath ?? dumpPath; diff --git a/test/integration/provider-scenarios/android-find.test.ts b/test/integration/provider-scenarios/android-find.test.ts index 19ca5bb91..67fb1160e 100644 --- a/test/integration/provider-scenarios/android-find.test.ts +++ b/test/integration/provider-scenarios/android-find.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { AndroidAdbProvider } from '../../../src/platforms/android/adb-executor.ts'; import { arrayEqual, assertCommandCall } from './assertions.ts'; -import { androidSettingsXml } from './android-world.ts'; +import { androidSettingsXml, androidSnapshotHelperOutput } from './android-world.ts'; import { PROVIDER_SCENARIO_ANDROID } from './fixtures.ts'; import { createProviderScenarioHarness } from './harness.ts'; @@ -160,7 +160,6 @@ test('Provider-backed integration Android find flow covers refs, wait, ambiguity assert.equal(lastAppMatch.x, 122); assert.equal(lastAppMatch.y, 217); - assertCommandCall(adbCalls, ['exec-out', 'uiautomator', 'dump', '/dev/tty']); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '88', '151']); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '122', '217']); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '195', '52']); @@ -197,6 +196,15 @@ function androidFindAdbResult( exitCode: 0, }; } + if (args.join(' ').startsWith('shell am instrument ')) { + return { + stdout: androidSnapshotHelperOutput( + androidSettingsXml(searchText, { duplicateAppsRow: includeDuplicateAppsRow }), + ), + stderr: '', + exitCode: 0, + }; + } return { stdout: '', stderr: '', exitCode: 0 }; } diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index a97c8957b..b4dc93d42 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -1226,6 +1226,8 @@ function assertAndroidPushAndEventContract(world: AndroidSettingsWorld): void { 'android.intent.action.VIEW', '-d', 'demo://agent-device/event?name=screenshot_taken&payload=%7B%22source%22%3A%22provider-scenario%22%2C%22foreground%22%3Atrue%7D&platform=android', + '-p', + 'com.example.demo', ]); assertCommandCall(adbCalls, ['shell', 'cmd', 'clipboard', 'get', 'text']); assertCommandCall(adbCalls, ['shell', 'cmd', 'clipboard', 'set', 'text', 'android otp']); diff --git a/test/integration/provider-scenarios/android-world.ts b/test/integration/provider-scenarios/android-world.ts index d151745b5..f5ecb7b2a 100644 --- a/test/integration/provider-scenarios/android-world.ts +++ b/test/integration/provider-scenarios/android-world.ts @@ -340,7 +340,7 @@ function androidCaptureAdbResult( return undefined; } -function androidSnapshotHelperOutput(xml: string): string { +export function androidSnapshotHelperOutput(xml: string): string { return [ 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', 'INSTRUMENTATION_STATUS: helperApiVersion=1',