From 0820282d4613aad0283a0301c46ee0f61913fcc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 22 Aug 2026 14:10:53 +0200 Subject: [PATCH 1/4] perf(android): keep the automation helper warm across fill and scroll MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Android permits one UiAutomation owner, so a command-scoped helper call stops the instrumentation session when it finishes and the next call pays a fresh `am instrument` start plus the UiAutomation connect wait. `fill` reads the live hierarchy four times per attempt (the pre-action target read plus the 0/150/350 ms settling samples) and every one of those reads was command-scoped, so a single fill ran four instrumentation lifecycles — seven on retry — and left `scroll` without a session for its viewport read. Thread the existing `daemon-session` scope, which app-backed sessions already select for snapshots and whose release session teardown already owns, through fill's captures and the gesture viewport read. The viewport read may now warm the session so it and the gesture that follows share one instrumentation instead of starting two. Snapshot capture and the viewport read build their capture options through one path, because the session identity is derived from them: two builders would restart each other's session instead of sharing it. Session teardown also stops force-stopping the runtime once the helper both acknowledged `quit` and its process exit was observed — that pair is the release evidence. Forced, timed-out, and aborted teardowns still force-stop, so a daemon that dies without sending `quit` cannot leave the helper squatting UiAutomation. The settling samples still capture separately; only the session is shared. --- src/core/interactors/android.test.ts | 47 +++ src/core/interactors/android.ts | 31 +- .../__tests__/fill-verification.test.ts | 166 +++++++++++ .../__tests__/input-actions-fill.test.ts | 50 ++++ .../snapshot-helper-retirement.test.ts | 60 ++++ .../snapshot-helper-session.fixtures.ts | 253 ++++++++++++++++ .../__tests__/snapshot-helper-session.test.ts | 17 +- .../android/__tests__/snapshot.test.ts | 274 +++--------------- .../__tests__/touch-helper-session.test.ts | 101 +++++++ src/platforms/android/fill-verification.ts | 22 +- src/platforms/android/input-actions.ts | 21 +- .../android/snapshot-helper-capture.ts | 35 +++ .../android/snapshot-helper-retirement.ts | 21 +- .../android/snapshot-helper-session.ts | 49 +++- .../android/snapshot-helper-types.ts | 17 ++ src/platforms/android/snapshot-helper.ts | 1 - src/platforms/android/snapshot.ts | 30 +- src/platforms/android/touch-executor.ts | 8 +- src/platforms/android/touch-helper.ts | 30 +- 19 files changed, 937 insertions(+), 296 deletions(-) create mode 100644 src/platforms/android/__tests__/fill-verification.test.ts create mode 100644 src/platforms/android/__tests__/snapshot-helper-session.fixtures.ts diff --git a/src/core/interactors/android.test.ts b/src/core/interactors/android.test.ts index a23b5aff69..70ff109863 100644 --- a/src/core/interactors/android.test.ts +++ b/src/core/interactors/android.test.ts @@ -6,12 +6,28 @@ import { import type { DeviceInfo } from '@agent-device/kernel/device'; import { createAndroidInteractor } from './android.ts'; import { snapshotAndroid } from '../../platforms/android/snapshot.ts'; +import { fillAndroid, scrollAndroid } from '../../platforms/android/input-actions.ts'; vi.mock('../../platforms/android/snapshot.ts', () => ({ snapshotAndroid: vi.fn(), })); +vi.mock('../../platforms/android/input-actions.ts', () => ({ + fillAndroid: vi.fn(), + scrollAndroid: vi.fn(), + appSwitcherAndroid: vi.fn(), + backAndroid: vi.fn(), + focusAndroid: vi.fn(), + homeAndroid: vi.fn(), + longPressAndroid: vi.fn(), + pressAndroid: vi.fn(), + pressAndroidTvRemote: vi.fn(), + setAndroidOrientation: vi.fn(), + typeAndroid: vi.fn(), +})); const snapshotAndroidMock = vi.mocked(snapshotAndroid); +const fillAndroidMock = vi.mocked(fillAndroid); +const scrollAndroidMock = vi.mocked(scrollAndroid); const device: DeviceInfo = { platform: 'android', id: 'emulator-5554', @@ -43,3 +59,34 @@ test('preserves Android clickability evidence through the interactor snapshot ad expect(readSnapshotClickabilityEvidence(result)).toEqual(evidence); expect(JSON.stringify(result)).not.toContain('clickable'); }); + +test('an app-backed session keeps the helper warm across fill and scroll', async () => { + const interactor = createAndroidInteractor(device, undefined, { + appBundleId: 'com.example.app', + }); + + await interactor.fill(10, 20, 'chips'); + await interactor.scroll('down', { amount: 1 }); + + expect(fillAndroidMock).toHaveBeenCalledWith(device, 10, 20, 'chips', undefined, { + helperSessionScope: 'daemon-session', + }); + expect(scrollAndroidMock).toHaveBeenCalledWith(device, 'down', { + amount: 1, + helperSessionScope: 'daemon-session', + }); +}); + +test('a device-only session releases the helper after fill and scroll', async () => { + const interactor = createAndroidInteractor(device); + + await interactor.fill(10, 20, 'chips'); + await interactor.scroll('down'); + + expect(fillAndroidMock).toHaveBeenCalledWith(device, 10, 20, 'chips', undefined, { + helperSessionScope: 'command', + }); + expect(scrollAndroidMock).toHaveBeenCalledWith(device, 'down', { + helperSessionScope: 'command', + }); +}); diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index 391e17e003..77f021d66f 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -30,6 +30,7 @@ import { } from '../../platforms/android/device-input-state.ts'; import { setAndroidSetting } from '../../platforms/android/settings.ts'; import { snapshotAndroid } from '../../platforms/android/snapshot.ts'; +import type { AndroidHelperSessionScope } from '../../platforms/android/snapshot-helper-types.ts'; import { screenshotAndroid } from '../../platforms/android/screenshot.ts'; import { withDiagnosticTimer } from '../../utils/diagnostics.ts'; import { withMethodScope } from '../../utils/method-scope.ts'; @@ -40,11 +41,21 @@ import { snapshotCaptureAnnotationsFrom, } from '@agent-device/contracts/capture'; +/** + * `appBundleId` is present exactly for app-backed daemon sessions, whose teardown releases the + * helper. Standalone device work has no such owner, so it releases the helper per command and + * leaves nothing squatting UiAutomation. + */ +function androidHelperSessionScope(appBundleId: string | undefined): AndroidHelperSessionScope { + return appBundleId ? 'daemon-session' : 'command'; +} + export function createAndroidInteractor( device: DeviceInfo, provider?: AndroidAdbProvider, - runnerContext?: Pick, + runnerContext?: Pick, ): Interactor { + const helperSessionScope = androidHelperSessionScope(runnerContext?.appBundleId); const interactor: Interactor = { open: (app, options) => openAndroidApp(device, app, { @@ -63,15 +74,19 @@ export function createAndroidInteractor( longPress: (x, y, durationMs) => longPressAndroid(device, x, y, durationMs), focus: (x, y) => focusAndroid(device, x, y), type: (text, delayMs) => typeAndroid(device, text, delayMs), - fill: (x, y, text, delayMs) => fillAndroid(device, x, y, text, delayMs), - scroll: (direction, options) => scrollAndroid(device, direction, options), + fill: (x, y, text, delayMs) => fillAndroid(device, x, y, text, delayMs, { helperSessionScope }), + scroll: (direction, options) => + scrollAndroid(device, direction, { ...options, helperSessionScope }), performGesture: (plan) => executeAndroidTouchPlan(device, plan), - gestureViewport: () => readAndroidGestureViewport(device), + gestureViewport: () => readAndroidGestureViewport(device, { helperSessionScope }), screenshot: (outPath, options) => screenshotAndroid(device, outPath, options), // uiautomator reads the node covering a point; `undefined` means nothing covers it. - readTextAtPoint: async (point) => { + readTextAtPoint: async (point, options) => { const { readAndroidTextAtPoint } = await import('../../platforms/android/input-actions.ts'); - return (await readAndroidTextAtPoint(device, point.x, point.y)) ?? undefined; + const read = await readAndroidTextAtPoint(device, point.x, point.y, { + helperSessionScope: androidHelperSessionScope(options?.appBundleId), + }); + return read ?? undefined; }, snapshot: async (options) => { const snapshotOptions = options ?? {}; @@ -86,9 +101,7 @@ export function createAndroidInteractor( scope: snapshotOptions.scope, raw: snapshotOptions.raw, includeHiddenContentHints: snapshotOptions.includeHiddenContentHints, - // appBundleId is present for app-backed daemon sessions; keep the helper warm there, - // but release it after standalone device snapshots so UiAutomation is not squatted. - helperSessionScope: snapshotOptions.appBundleId ? 'daemon-session' : 'command', + helperSessionScope: androidHelperSessionScope(snapshotOptions.appBundleId), }), { backend: 'android' }, ); diff --git a/src/platforms/android/__tests__/fill-verification.test.ts b/src/platforms/android/__tests__/fill-verification.test.ts new file mode 100644 index 0000000000..003d1e3ff4 --- /dev/null +++ b/src/platforms/android/__tests__/fill-verification.test.ts @@ -0,0 +1,166 @@ +// Fill reads the live hierarchy four times per attempt (one pre-action target read plus the +// 0/150/350 ms settling samples). Android permits ONE UiAutomation owner, so a command-scoped +// capture stops the automation-helper session after every one of those reads and the next read +// pays a fresh `am instrument` start. These tests pin who owns the helper session across the +// samples — not what the samples conclude, which fill-diagnostics/input-actions-fill own. + +import { afterEach, beforeEach, test } from 'vitest'; +import assert from 'node:assert/strict'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { withAndroidAdbProvider, type AndroidAdbProvider } from '../adb-executor.ts'; +import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from '../../../__tests__/test-utils/android-snapshot-helper.ts'; +import { + readAndroidFillTargetBeforeMutation, + verifyAndroidFilledText, +} from '../fill-verification.ts'; +import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session.ts'; +import { + createPersistentSnapshotHelperProvider, + isAndroidHelperForwardRemoval, + type FakeAndroidProcess, +} from './snapshot-helper-session.fixtures.ts'; + +const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, +}; + +beforeEach(async () => { + await resetAndroidSnapshotHelperSessions(); +}); + +afterEach(async () => { + await resetAndroidSnapshotHelperSessions(); +}); + +test('daemon-session verification samples reuse one warm helper session', async () => { + const session = createFillHelperSession(); + + const verification = await withFillHelperProvider( + session.provider, + async () => + await verifyAndroidFilledText(device, 10, 10, 'chips', { + helperSessionScope: 'daemon-session', + }), + ); + + assert.equal(verification.ok, true); + assert.equal(session.captureCount(), 3); + assert.equal(session.spawnArgs.length, 1, 'one instrumentation start for every sample'); + assert.equal(session.processes[0]?.exitCode, null, 'the session must outlive the command'); + assert.equal(session.forwardRemovals(), 0); +}); + +test('command-scoped verification samples release the helper after every sample', async () => { + const session = createFillHelperSession(); + + await withFillHelperProvider( + session.provider, + async () => await verifyAndroidFilledText(device, 10, 10, 'chips'), + ); + + assert.equal(session.captureCount(), 3); + assert.equal(session.spawnArgs.length, 3, 'each sample pays its own instrumentation start'); + assert.equal(session.forwardRemovals(), 3); +}); + +test('verification samples re-read the hierarchy instead of sharing one capture', async () => { + // The 0/150/350 ms samples exist to observe settling. Sharing the session must not turn into + // sharing its capture: replaying the first sample's still-settling text would report a mismatch + // for a field that did take the value. + const settlingText = ['chi', 'chip', 'chips']; + const session = createFillHelperSession({ + textForCapture: (captureIndex) => settlingText[captureIndex - 1] ?? 'chips', + }); + + const verification = await withFillHelperProvider( + session.provider, + async () => + await verifyAndroidFilledText(device, 10, 10, 'chips', { + helperSessionScope: 'daemon-session', + }), + ); + + assert.equal(session.captureCount(), 3); + assert.equal(verification.actual, 'chips', 'the last sample is read from its own capture'); + assert.equal(verification.ok, true); +}); + +test('the pre-action target read shares the daemon-session helper with the samples', async () => { + const session = createFillHelperSession(); + + const target = await withFillHelperProvider(session.provider, async () => { + const before = await readAndroidFillTargetBeforeMutation(device, 10, 10, { + helperSessionScope: 'daemon-session', + }); + await verifyAndroidFilledText(device, 10, 10, 'chips', { + helperSessionScope: 'daemon-session', + }); + return before; + }); + + assert.equal(target?.resourceId, 'com.example:id/field'); + assert.equal(session.captureCount(), 4); + assert.equal(session.spawnArgs.length, 1); + assert.equal(session.forwardRemovals(), 0); +}); + +type FillHelperSession = { + provider: AndroidAdbProvider; + spawnArgs: string[][]; + processes: FakeAndroidProcess[]; + captureCount: () => number; + forwardRemovals: () => number; +}; + +function createFillHelperSession( + options: { textForCapture?: (captureIndex: number) => string } = {}, +): FillHelperSession { + const calls: string[][] = []; + const spawnArgs: string[][] = []; + const processes: FakeAndroidProcess[] = []; + let captureCount = 0; + const provider = createPersistentSnapshotHelperProvider({ + calls, + spawnArgs, + processes, + sessionXml: () => { + captureCount += 1; + return filledFieldXml(options.textForCapture?.(captureCount) ?? 'chips'); + }, + }); + return { + provider: { ...provider, exec: withKeyboardStateProbe(provider.exec) }, + spawnArgs, + processes, + captureCount: () => captureCount, + forwardRemovals: () => calls.filter(isAndroidHelperForwardRemoval).length, + }; +} + +async function withFillHelperProvider( + provider: AndroidAdbProvider, + fn: () => Promise, +): Promise { + return await withAndroidAdbProvider( + { ...provider, snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT }, + { serial: device.id }, + fn, + ); +} + +function withKeyboardStateProbe(exec: AndroidAdbProvider['exec']): AndroidAdbProvider['exec'] { + return async (args, execOptions) => { + if (args.join(' ') === 'shell dumpsys input_method') { + return { exitCode: 0, stdout: 'mCurMethodId=com.example.ime/.Ime', stderr: '' }; + } + return await exec(args, execOptions); + }; +} + +function filledFieldXml(text: string): string { + return ``; +} diff --git a/src/platforms/android/__tests__/input-actions-fill.test.ts b/src/platforms/android/__tests__/input-actions-fill.test.ts index dd870df184..34b2979627 100644 --- a/src/platforms/android/__tests__/input-actions-fill.test.ts +++ b/src/platforms/android/__tests__/input-actions-fill.test.ts @@ -14,6 +14,12 @@ import { readAndroidTextAtPointInHierarchy, verifyAndroidFilledTextInHierarchy, } from '../fill-verification.ts'; +import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session.ts'; +import { + createPersistentSnapshotHelperProvider, + isAndroidHelperForwardRemoval, + type FakeAndroidProcess, +} from './snapshot-helper-session.fixtures.ts'; test('fillAndroid reports when the IME captures input instead of the app field', async () => { const calls: string[][] = []; @@ -403,6 +409,50 @@ test('readAndroidTextAtPointInHierarchy reads the EditText under the requested p assert.equal(readAndroidTextAtPointInHierarchy(hierarchy, 100, 250), 'fallback@example.com'); }); +test('fillAndroid runs the whole attempt on one warm daemon-session helper', async () => { + const calls: string[][] = []; + const spawnArgs: string[][] = []; + const processes: FakeAndroidProcess[] = []; + let typed = ''; + let captureCount = 0; + const provider = createPersistentSnapshotHelperProvider({ + calls, + spawnArgs, + processes, + sessionXml: () => { + captureCount += 1; + return androidInputXml({ text: typed }); + }, + }); + + await resetAndroidSnapshotHelperSessions(); + await withAndroidAdbProvider( + { + ...provider, + exec: async (args, options) => { + if (isDeleteKey(args)) typed = ''; + if (isTextInput(args)) typed = args[3] ?? ''; + if (args[0] === 'shell' && args[1] !== 'am') return adbResult(''); + return await provider.exec(args, options); + }, + snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT, + }, + { serial: ANDROID_EMULATOR.id }, + async () => { + await fillAndroid(ANDROID_EMULATOR, 10, 10, 'chips', 0, { + helperSessionScope: 'daemon-session', + }); + }, + ); + + // One pre-action target read plus the three settling samples. + assert.equal(captureCount, 4); + assert.equal(spawnArgs.length, 1); + assert.equal(calls.filter(isAndroidHelperForwardRemoval).length, 0); + assert.equal(processes[0]?.exitCode, null); + await resetAndroidSnapshotHelperSessions(); +}); + const IME_RESOURCE_ID = 'com.google.android.inputmethod.latin:id/0_resource_name_obfuscated'; async function withFillAdb(exec: AndroidAdbExecutor, fn: () => Promise): Promise; diff --git a/src/platforms/android/__tests__/snapshot-helper-retirement.test.ts b/src/platforms/android/__tests__/snapshot-helper-retirement.test.ts index c68542baa3..3d5b6b8de1 100644 --- a/src/platforms/android/__tests__/snapshot-helper-retirement.test.ts +++ b/src/platforms/android/__tests__/snapshot-helper-retirement.test.ts @@ -1,10 +1,14 @@ import assert from 'node:assert/strict'; import { beforeEach, test } from 'vitest'; +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; import { recoverAndroidSnapshotHelperRetirement, resetAndroidSnapshotHelperRetirements, retireCanceledAndroidSnapshotHelperCapture, + settleAndroidSnapshotHelperSessionCleanup, } from '../snapshot-helper-retirement.ts'; +import type { AndroidAdbProcess } from '../adb-executor.ts'; import type { AndroidAdbExecutor } from '../snapshot-helper-types.ts'; beforeEach(() => { @@ -55,3 +59,59 @@ test('requires positive recovery evidence after uncertain runtime retirement', a ]), ); }); + +test('session cleanup force-stops the runtime when release was not confirmed', async () => { + const calls: string[][] = []; + const cleanup = await settleAndroidSnapshotHelperSessionCleanup({ + adb: recordingAdb(calls), + process: new StubAndroidProcess(), + port: 41234, + packageName: 'com.callstack.agentdevice.snapshothelper', + timeoutMs: 2_000, + forceStopRuntime: true, + }); + + assert.equal(cleanup.runtimeForceStopped, true); + assert.deepEqual(calls, [ + ['shell', 'am', 'force-stop', 'com.callstack.agentdevice.snapshothelper'], + ['forward', '--remove', 'tcp:41234'], + ]); +}); + +test('session cleanup skips the force-stop round trip once release is confirmed', async () => { + const calls: string[][] = []; + const cleanup = await settleAndroidSnapshotHelperSessionCleanup({ + adb: recordingAdb(calls), + process: new StubAndroidProcess(), + port: 41234, + packageName: 'com.callstack.agentdevice.snapshothelper', + timeoutMs: 2_000, + forceStopRuntime: false, + }); + + // The helper already released UiAutomation, so nothing was force-stopped and nothing may claim + // it was: the caller reads this flag to decide whether the retirement needs quarantining. + assert.equal(cleanup.runtimeForceStopped, false); + assert.deepEqual(calls, [['forward', '--remove', 'tcp:41234']]); +}); + +function recordingAdb(calls: string[][]): AndroidAdbExecutor { + return async (args) => { + calls.push(args); + return { exitCode: 0, stdout: '', stderr: '' }; + }; +} + +class StubAndroidProcess extends EventEmitter implements AndroidAdbProcess { + stdin = new PassThrough(); + stdout = new PassThrough(); + stderr = new PassThrough(); + exitCode: number | null = 0; + signalCode: NodeJS.Signals | null = null; + killed = false; + + kill(): boolean { + this.killed = true; + return true; + } +} diff --git a/src/platforms/android/__tests__/snapshot-helper-session.fixtures.ts b/src/platforms/android/__tests__/snapshot-helper-session.fixtures.ts new file mode 100644 index 0000000000..513bd58414 --- /dev/null +++ b/src/platforms/android/__tests__/snapshot-helper-session.fixtures.ts @@ -0,0 +1,253 @@ +// Fakes for the persistent Android automation-helper session: a spawned instrumentation process +// plus a local TCP server standing in for the helper's session socket. Every test that observes +// session lifetime (snapshot capture, fill verification samples, gesture/viewport transport) reads +// the same fake, so "one warm session" means the same thing in all of them. + +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import net from 'node:net'; +import { PassThrough } from 'node:stream'; +import type { AndroidAdbProcess, AndroidAdbProvider } from '../adb-executor.ts'; +import type { AndroidAdbExecutor } from '../snapshot-helper-types.ts'; + +export const ANDROID_HELPER_INSTALLED_VERSION_PROBE = { + exitCode: 0, + stdout: 'package:com.callstack.agentdevice.snapshothelper versionCode:13004', + stderr: '', +}; + +export class FakeAndroidProcess extends EventEmitter implements AndroidAdbProcess { + stdin = new PassThrough(); + stdout = new PassThrough(); + stderr = new PassThrough(); + exitCode: number | null = null; + signalCode: NodeJS.Signals | null = null; + killed = false; + onKill: (() => void) | undefined; + + kill(): boolean { + if (this.killed) return true; + this.killed = true; + this.onKill?.(); + return true; + } + + emitExit(code: number | null, signal: NodeJS.Signals | null): void { + this.exitCode = code; + this.signalCode = signal; + this.emit('exit', code, signal); + this.emit('close', code, signal); + } +} + +export type PersistentSnapshotHelperProviderOptions = { + calls: string[][]; + spawnArgs: string[][]; + processes: FakeAndroidProcess[]; + sessionResponseMode?: 'ok' | 'malformed'; + sessionXml?: (sessionIndex: number, snapshotCount: number) => string; + stalledSessionCleanup?: boolean; + oneShotAttempts?: string[][]; + oneShotXml?: string; +}; + +export function createPersistentSnapshotHelperProvider( + options: PersistentSnapshotHelperProviderOptions, +): AndroidAdbProvider { + return { + exec: createPersistentSnapshotExec(options), + spawn: (args) => { + options.spawnArgs.push(args); + const sessionIndex = options.spawnArgs.length; + const process = new FakeAndroidProcess(); + options.processes.push(process); + const port = readSessionPort(args); + let snapshotCount = 0; + const server = net.createServer((socket) => { + socket.once('data', (chunk) => { + const command = chunk.toString('utf8').trim(); + const [, requestId = ''] = command.split(/\s+/, 2); + if (command.startsWith('quit')) { + socket.end(sessionResponse({ requestId, body: '' })); + server.close(() => process.emitExit(0, null)); + return; + } + if (options.sessionResponseMode === 'malformed') { + socket.end('malformed session response'); + return; + } + if (command.startsWith('viewport')) { + socket.end( + sessionResponse({ + requestId, + body: '', + metadata: { x: '0', y: '0', width: '400', height: '800' }, + }), + ); + return; + } + snapshotCount += 1; + const body = options.sessionXml + ? options.sessionXml(sessionIndex, snapshotCount) + : ``; + socket.end( + sessionResponse({ + requestId, + body, + metadata: { + waitForIdleTimeoutMs: '500', + waitForIdleQuietMs: '100', + timeoutMs: '5000', + maxDepth: '128', + maxNodes: '5000', + rootPresent: 'true', + captureMode: 'interactive-windows', + windowCount: '1', + nodeCount: '1', + truncated: 'false', + elapsedMs: '8', + }, + }), + ); + }); + }); + server.listen(port, '127.0.0.1', () => { + process.stdout.write( + [ + 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', + 'INSTRUMENTATION_STATUS: sessionReady=true', + 'INSTRUMENTATION_STATUS_CODE: 2', + '', + ].join('\n'), + ); + }); + process.onKill = () => { + server.close(() => process.emitExit(0, null)); + }; + return process; + }, + }; +} + +export function isAndroidHelperRuntimeForceStop(args: readonly string[]): boolean { + return args[0] === 'shell' && args[1] === 'am' && args[2] === 'force-stop'; +} + +export function isAndroidHelperForwardRemoval(args: readonly string[]): boolean { + return args[0] === 'forward' && args[1] === '--remove'; +} + +export function androidHelperInstrumentationOutput( + xml: string, + options: { truncated?: boolean; nodeCount?: number; windowCount?: number } = {}, +): string { + const truncated = options.truncated ?? false; + const nodeCount = options.nodeCount ?? 1; + const windowCount = options.windowCount ?? 1; + return [ + 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', + 'INSTRUMENTATION_STATUS: helperApiVersion=1', + 'INSTRUMENTATION_STATUS: outputFormat=uiautomator-xml', + 'INSTRUMENTATION_STATUS: chunkIndex=0', + 'INSTRUMENTATION_STATUS: chunkCount=1', + `INSTRUMENTATION_STATUS: payloadBase64=${Buffer.from(xml, 'utf8').toString('base64')}`, + 'INSTRUMENTATION_STATUS_CODE: 1', + 'INSTRUMENTATION_RESULT: agentDeviceProtocol=android-snapshot-helper-v1', + 'INSTRUMENTATION_RESULT: helperApiVersion=1', + 'INSTRUMENTATION_RESULT: ok=true', + 'INSTRUMENTATION_RESULT: outputFormat=uiautomator-xml', + 'INSTRUMENTATION_RESULT: waitForIdleTimeoutMs=0', + 'INSTRUMENTATION_RESULT: timeoutMs=8000', + 'INSTRUMENTATION_RESULT: maxDepth=128', + 'INSTRUMENTATION_RESULT: maxNodes=5000', + 'INSTRUMENTATION_RESULT: rootPresent=true', + 'INSTRUMENTATION_RESULT: captureMode=interactive-windows', + `INSTRUMENTATION_RESULT: windowCount=${windowCount}`, + `INSTRUMENTATION_RESULT: nodeCount=${nodeCount}`, + `INSTRUMENTATION_RESULT: truncated=${truncated}`, + 'INSTRUMENTATION_RESULT: elapsedMs=12', + 'INSTRUMENTATION_CODE: 0', + ].join('\n'); +} + +function createPersistentSnapshotExec( + options: PersistentSnapshotHelperProviderOptions, +): AndroidAdbExecutor { + return async (args, execOptions) => { + options.calls.push(args); + const stalledCleanup = stalledPersistentCleanup(options, args, execOptions?.signal); + if (stalledCleanup) return await stalledCleanup; + return persistentSnapshotExecResult(options, args); + }; +} + +function stalledPersistentCleanup( + options: PersistentSnapshotHelperProviderOptions, + args: string[], + signal: AbortSignal | undefined, +): ReturnType | undefined { + if (!options.stalledSessionCleanup || !signal) return undefined; + return isAndroidHelperForwardRemoval(args) || isAndroidHelperRuntimeForceStop(args) + ? rejectWhenAborted(signal) + : undefined; +} + +function persistentSnapshotExecResult( + options: PersistentSnapshotHelperProviderOptions, + args: string[], +): ReturnType { + if (args.includes('--show-versioncode')) { + return Promise.resolve(ANDROID_HELPER_INSTALLED_VERSION_PROBE); + } + if (args[0] === 'forward' || isAndroidHelperRuntimeForceStop(args)) { + return Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }); + } + if (args.includes('instrument')) { + options.oneShotAttempts?.push(args); + if (options.oneShotXml) { + return Promise.resolve({ + exitCode: 0, + stdout: androidHelperInstrumentationOutput(options.oneShotXml), + stderr: '', + }); + } + } + return Promise.reject(new Error(`unexpected persistent helper adb args: ${args.join(' ')}`)); +} + +function rejectWhenAborted(signal: AbortSignal): Promise<{ + exitCode: number; + stdout: string; + stderr: string; +}> { + return new Promise((_resolve, reject) => { + const onAbort = () => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + if (signal.aborted) onAbort(); + }); +} + +function sessionResponse(params: { + requestId: string; + body: string; + metadata?: Record; +}): string { + const headers = { + agentDeviceProtocol: 'android-snapshot-helper-v1', + helperApiVersion: '1', + outputFormat: 'uiautomator-xml', + requestId: params.requestId, + ok: 'true', + byteLength: String(Buffer.byteLength(params.body, 'utf8')), + ...params.metadata, + }; + return `${Object.entries(headers) + .map(([key, value]) => `${key}=${value}`) + .join('\n')}\n\n${params.body}`; +} + +function readSessionPort(args: string[]): number { + const index = args.indexOf('sessionPort'); + assert.notEqual(index, -1); + return Number(args[index + 1]); +} diff --git a/src/platforms/android/__tests__/snapshot-helper-session.test.ts b/src/platforms/android/__tests__/snapshot-helper-session.test.ts index 9c8675be86..e37b82ac3f 100644 --- a/src/platforms/android/__tests__/snapshot-helper-session.test.ts +++ b/src/platforms/android/__tests__/snapshot-helper-session.test.ts @@ -281,7 +281,7 @@ test('restarts the helper session when capture options change', async () => { ); }); -test('allows an acknowledged helper quit to release UiAutomation before forcing termination', async () => { +test('a quit acknowledged and followed by process exit skips the force-stop round trip', async () => { const calls: string[][] = []; const processes: FakeAndroidProcess[] = []; const provider = createSessionProvider({ calls, processes, quitExitDelayMs: 25 }); @@ -295,12 +295,9 @@ test('allows an acknowledged helper quit to release UiAutomation before forcing assert.equal(processes.length, 1); assert.equal(processes[0]?.killed, false); - assert.equal( - calls.some( - (args) => args.join(' ') === 'shell am force-stop com.callstack.agentdevice.snapshothelper', - ), - true, - ); + // Acknowledged quit plus an observed exit IS the release evidence, so the extra adb round trip + // buys nothing. + assert.equal(calls.some(isHelperRuntimeForceStop), false); }); test('force terminates the helper when quit is not acknowledged', async () => { @@ -317,8 +314,14 @@ test('force terminates the helper when quit is not acknowledged', async () => { assert.equal(processes.length, 1); assert.equal(processes[0]?.killed, true); + // Nothing proved the helper released UiAutomation, so the device-side stop must still run. + assert.equal(calls.some(isHelperRuntimeForceStop), true); }); +function isHelperRuntimeForceStop(args: string[]): boolean { + return args.join(' ') === 'shell am force-stop com.callstack.agentdevice.snapshothelper'; +} + test('failed whole-module reset preserves quarantine until recovery is confirmed', async () => { const options: SessionProviderOptions = { calls: [], diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index cd10975837..63fb5e7502 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -1,11 +1,8 @@ import { afterEach, beforeEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; -import { EventEmitter } from 'node:events'; import { promises as fs } from 'node:fs'; -import net from 'node:net'; import os from 'node:os'; import path from 'node:path'; -import { PassThrough } from 'node:stream'; import { mkdtempForTest } from '../../../__tests__/test-utils/tmp-dir.ts'; vi.mock('../../../utils/exec.ts', async (importOriginal) => { @@ -19,6 +16,7 @@ vi.mock('../adb.ts', async (importOriginal) => { import { screenshotAndroid } from '../screenshot.ts'; import { snapshotAndroid } from '../snapshot.ts'; +import { readAndroidGestureViewport } from '../touch-executor.ts'; import { buildUiHierarchySnapshot, parseUiHierarchyTree } from '../ui-hierarchy.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { flushDiagnosticsToSessionFile, withDiagnosticsScope } from '../../../utils/diagnostics.ts'; @@ -30,10 +28,13 @@ import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session.t import { type AndroidAdbExecutor } from '../snapshot-helper.ts'; import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from '../../../__tests__/test-utils/android-snapshot-helper.ts'; import { - withAndroidAdbProvider, - type AndroidAdbProcess, - type AndroidAdbProvider, -} from '../adb-executor.ts'; + androidHelperInstrumentationOutput as helperOutput, + createPersistentSnapshotHelperProvider, + isAndroidHelperRuntimeForceStop as isHelperRuntimeReset, + ANDROID_HELPER_INSTALLED_VERSION_PROBE as installedHelperProbe, + type FakeAndroidProcess, +} from './snapshot-helper-session.fixtures.ts'; +import { withAndroidAdbProvider, type AndroidAdbProvider } from '../adb-executor.ts'; const VALID_PNG = Buffer.from( 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+b9xkAAAAASUVORK5CYII=', @@ -51,11 +52,6 @@ const device: DeviceInfo = { }; const helperArtifact = ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT; -const installedHelperProbe = { - exitCode: 0, - stdout: 'package:com.callstack.agentdevice.snapshothelper versionCode:13004', - stderr: '', -}; function snapshotAndroidWithHelper( helperAdb: AndroidAdbExecutor, @@ -90,198 +86,11 @@ function isHelperVersionProbe(args: string[]): boolean { return args.includes('--show-versioncode'); } -function isHelperRuntimeReset(args: string[]): boolean { - return args[0] === 'shell' && args[1] === 'am' && args[2] === 'force-stop'; -} - function helperAdbOperation(args: string[]): 'instrument' | 'activity' | undefined { if (args.includes('instrument')) return 'instrument'; return args.includes('dumpsys') && args.includes('activity') ? 'activity' : undefined; } -type PersistentSnapshotHelperProviderOptions = { - calls: string[][]; - spawnArgs: string[][]; - processes: FakeAndroidProcess[]; - sessionResponseMode?: 'ok' | 'malformed'; - sessionXml?: (sessionIndex: number, snapshotCount: number) => string; - stalledSessionCleanup?: boolean; - oneShotAttempts?: string[][]; - oneShotXml?: string; -}; - -function createPersistentSnapshotHelperProvider( - options: PersistentSnapshotHelperProviderOptions, -): AndroidAdbProvider { - return { - exec: createPersistentSnapshotExec(options), - spawn: (args) => { - options.spawnArgs.push(args); - const sessionIndex = options.spawnArgs.length; - const process = new FakeAndroidProcess(); - options.processes.push(process); - const port = readSessionPort(args); - let snapshotCount = 0; - const server = net.createServer((socket) => { - socket.once('data', (chunk) => { - const command = chunk.toString('utf8').trim(); - const [, requestId = ''] = command.split(/\s+/, 2); - if (command.startsWith('quit')) { - socket.end(sessionResponse({ requestId, body: '' })); - server.close(() => process.emitExit(0, null)); - return; - } - if (options.sessionResponseMode === 'malformed') { - socket.end('malformed session response'); - return; - } - snapshotCount += 1; - const body = options.sessionXml - ? options.sessionXml(sessionIndex, snapshotCount) - : ``; - socket.end( - sessionResponse({ - requestId, - body, - metadata: { - waitForIdleTimeoutMs: '500', - waitForIdleQuietMs: '100', - timeoutMs: '5000', - maxDepth: '128', - maxNodes: '5000', - rootPresent: 'true', - captureMode: 'interactive-windows', - windowCount: '1', - nodeCount: '1', - truncated: 'false', - elapsedMs: '8', - }, - }), - ); - }); - }); - server.listen(port, '127.0.0.1', () => { - process.stdout.write( - [ - 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', - 'INSTRUMENTATION_STATUS: sessionReady=true', - 'INSTRUMENTATION_STATUS_CODE: 2', - '', - ].join('\n'), - ); - }); - process.onKill = () => { - server.close(() => process.emitExit(0, null)); - }; - return process; - }, - }; -} - -function createPersistentSnapshotExec( - options: PersistentSnapshotHelperProviderOptions, -): AndroidAdbExecutor { - return async (args, execOptions) => { - options.calls.push(args); - const stalledCleanup = stalledPersistentCleanup(options, args, execOptions?.signal); - if (stalledCleanup) return await stalledCleanup; - return persistentSnapshotExecResult(options, args); - }; -} - -function stalledPersistentCleanup( - options: PersistentSnapshotHelperProviderOptions, - args: string[], - signal: AbortSignal | undefined, -): ReturnType | undefined { - if (!options.stalledSessionCleanup || !signal) return undefined; - const removesForward = args[0] === 'forward' && args[1] === '--remove'; - const forceStopsRuntime = args[0] === 'shell' && args[1] === 'am' && args[2] === 'force-stop'; - return removesForward || forceStopsRuntime ? rejectWhenAborted(signal) : undefined; -} - -function persistentSnapshotExecResult( - options: PersistentSnapshotHelperProviderOptions, - args: string[], -): ReturnType { - if (args.includes('--show-versioncode')) return Promise.resolve(installedHelperProbe); - if (args[0] === 'forward' || isHelperRuntimeReset(args)) { - return Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }); - } - if (args.includes('instrument')) { - options.oneShotAttempts?.push(args); - if (options.oneShotXml) { - return Promise.resolve({ - exitCode: 0, - stdout: helperOutput(options.oneShotXml), - stderr: '', - }); - } - } - return Promise.reject(new Error(`unexpected persistent helper adb args: ${args.join(' ')}`)); -} - -function rejectWhenAborted(signal: AbortSignal): Promise<{ - exitCode: number; - stdout: string; - stderr: string; -}> { - return new Promise((_resolve, reject) => { - const onAbort = () => reject(signal.reason); - signal.addEventListener('abort', onAbort, { once: true }); - if (signal.aborted) onAbort(); - }); -} - -function sessionResponse(params: { - requestId: string; - body: string; - metadata?: Record; -}): string { - const headers = { - agentDeviceProtocol: 'android-snapshot-helper-v1', - helperApiVersion: '1', - outputFormat: 'uiautomator-xml', - requestId: params.requestId, - ok: 'true', - byteLength: String(Buffer.byteLength(params.body, 'utf8')), - ...params.metadata, - }; - return `${Object.entries(headers) - .map(([key, value]) => `${key}=${value}`) - .join('\n')}\n\n${params.body}`; -} - -function readSessionPort(args: string[]): number { - const index = args.indexOf('sessionPort'); - assert.notEqual(index, -1); - return Number(args[index + 1]); -} - -class FakeAndroidProcess extends EventEmitter implements AndroidAdbProcess { - stdin = new PassThrough(); - stdout = new PassThrough(); - stderr = new PassThrough(); - exitCode: number | null = null; - signalCode: NodeJS.Signals | null = null; - killed = false; - onKill: (() => void) | undefined; - - kill(): boolean { - if (this.killed) return true; - this.killed = true; - this.onKill?.(); - return true; - } - - emitExit(code: number | null, signal: NodeJS.Signals | null): void { - this.exitCode = code; - this.signalCode = signal; - this.emit('exit', code, signal); - this.emit('close', code, signal); - } -} - beforeEach(async () => { await resetAndroidSnapshotHelperSessions(); resetAndroidSnapshotHelperInstallCache(); @@ -379,39 +188,6 @@ test('screenshotAndroid throws when PNG payload is truncated', async () => { }); }); -function helperOutput( - xml: string, - options: { truncated?: boolean; nodeCount?: number; windowCount?: number } = {}, -): string { - const truncated = options.truncated ?? false; - const nodeCount = options.nodeCount ?? 1; - const windowCount = options.windowCount ?? 1; - return [ - 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', - 'INSTRUMENTATION_STATUS: helperApiVersion=1', - 'INSTRUMENTATION_STATUS: outputFormat=uiautomator-xml', - 'INSTRUMENTATION_STATUS: chunkIndex=0', - 'INSTRUMENTATION_STATUS: chunkCount=1', - `INSTRUMENTATION_STATUS: payloadBase64=${Buffer.from(xml, 'utf8').toString('base64')}`, - 'INSTRUMENTATION_STATUS_CODE: 1', - 'INSTRUMENTATION_RESULT: agentDeviceProtocol=android-snapshot-helper-v1', - 'INSTRUMENTATION_RESULT: helperApiVersion=1', - 'INSTRUMENTATION_RESULT: ok=true', - 'INSTRUMENTATION_RESULT: outputFormat=uiautomator-xml', - 'INSTRUMENTATION_RESULT: waitForIdleTimeoutMs=0', - 'INSTRUMENTATION_RESULT: timeoutMs=8000', - 'INSTRUMENTATION_RESULT: maxDepth=128', - 'INSTRUMENTATION_RESULT: maxNodes=5000', - 'INSTRUMENTATION_RESULT: rootPresent=true', - 'INSTRUMENTATION_RESULT: captureMode=interactive-windows', - `INSTRUMENTATION_RESULT: windowCount=${windowCount}`, - `INSTRUMENTATION_RESULT: nodeCount=${nodeCount}`, - `INSTRUMENTATION_RESULT: truncated=${truncated}`, - 'INSTRUMENTATION_RESULT: elapsedMs=12', - 'INSTRUMENTATION_CODE: 0', - ].join('\n'); -} - function androidSystemWindowOnlyXml(): string { return [ '', @@ -761,6 +537,40 @@ test('snapshotAndroid keeps daemon-session helper alive for reuse until session ); }); +test('a daemon-session viewport read warms the session the next snapshot reuses', async () => { + // The gesture viewport and snapshot capture are different helper commands on the same device. + // They may only share the live session if both derive the same session identity, which is why + // their capture options have one construction path. + const adbCalls: string[][] = []; + const spawnArgs: string[][] = []; + const processes: FakeAndroidProcess[] = []; + const provider = createPersistentSnapshotHelperProvider({ + calls: adbCalls, + spawnArgs, + processes, + }); + + const snapshot = await withAndroidAdbProvider( + { ...provider, snapshotHelperArtifact: helperArtifact }, + { serial: device.id }, + async () => { + await readAndroidGestureViewport(device, { helperSessionScope: 'daemon-session' }); + return await snapshotAndroid(device, { helperSessionScope: 'daemon-session' }); + }, + ); + + assert.equal(snapshot.androidSnapshot.helperTransport, 'persistent-session'); + // `helperSessionReused` reports repeat CAPTURES, and this is the session's first one: the + // instrumentation count below is what proves the viewport read left the session warm. + assert.equal(snapshot.androidSnapshot.helperSessionReused, false); + assert.equal(spawnArgs.length, 1, 'the viewport read started the only instrumentation'); + assert.equal( + adbCalls.some((args) => args.includes('instrument')), + false, + 'neither command fell back to one-shot instrumentation', + ); +}); + test('snapshotAndroid retires content-invalid daemon helper before the next request', async () => { const adbCalls: string[][] = []; const spawnArgs: string[][] = []; diff --git a/src/platforms/android/__tests__/touch-helper-session.test.ts b/src/platforms/android/__tests__/touch-helper-session.test.ts index 12943a731e..81f32b0094 100644 --- a/src/platforms/android/__tests__/touch-helper-session.test.ts +++ b/src/platforms/android/__tests__/touch-helper-session.test.ts @@ -205,6 +205,107 @@ test('touch helper does not run one-shot while snapshot retirement is unconfirme assert.equal(oneShotCalled, false); }); +test('a daemon-session viewport read starts the session so the gesture reuses it', async () => { + const device = makeIsolatedDevice(); + let viewportCommands = 0; + let gestureCommands = 0; + const provider = createFakeTouchHelperSessionProvider((command, requestId) => { + if (command.startsWith('viewport')) { + viewportCommands += 1; + return sessionHeaderResponse({ + agentDeviceProtocol: 'android-snapshot-helper-v1', + requestId, + ok: 'true', + x: '0', + y: '0', + width: '400', + height: '800', + }); + } + if (command.startsWith('gesture')) { + gestureCommands += 1; + return sessionHeaderResponse({ + agentDeviceProtocol: 'android-snapshot-helper-v1', + requestId, + ok: 'true', + kind: 'swipe', + injectedEvents: '6', + elapsedMs: '9', + }); + } + return sessionHeaderResponse({ + agentDeviceProtocol: 'android-snapshot-helper-v1', + requestId, + ok: 'true', + }); + }); + + const oneShotArgs: string[][] = []; + const result = await withAndroidAdbProvider( + { + ...provider, + exec: currentVersionAdb(async (args) => { + if (args.includes('instrument')) oneShotArgs.push(args); + return { exitCode: 0, stdout: '', stderr: '' }; + }), + }, + { serial: device.id }, + async () => { + const viewport = await readAndroidTouchHelperViewport(device, { + helperSessionScope: 'daemon-session', + }); + const gesture = await executeAndroidTouchHelperPlan( + device, + lowerAndroidTouchPlan(flingPlan()), + ); + return { viewport, gesture }; + }, + ); + + assert.deepEqual(result.viewport, { x: 0, y: 0, width: 400, height: 800 }); + assert.equal(result.gesture.helperTransport, 'persistent-session'); + assert.equal(viewportCommands, 1); + assert.equal(gestureCommands, 1); + assert.deepEqual(oneShotArgs, [], 'viewport and gesture share the session instrumentation'); +}); + +test('a command-scoped viewport read stays one-shot and starts no session', async () => { + const device = makeIsolatedDevice(); + let sessionCommands = 0; + const provider = createFakeTouchHelperSessionProvider((_command, requestId) => { + sessionCommands += 1; + return sessionHeaderResponse({ + agentDeviceProtocol: 'android-snapshot-helper-v1', + requestId, + ok: 'true', + }); + }); + + let oneShotArgs: string[] | undefined; + const viewport = await withAndroidAdbProvider( + { + ...provider, + exec: currentVersionAdb(async (args) => { + oneShotArgs = args; + return { + exitCode: 0, + stdout: [ + resultRecord({ ok: 'true', x: '5', y: '6', width: '300', height: '400' }), + 'INSTRUMENTATION_CODE: 0', + ].join('\n'), + stderr: '', + }; + }), + }, + { serial: device.id }, + async () => await readAndroidTouchHelperViewport(device), + ); + + assert.deepEqual(viewport, { x: 5, y: 6, width: 300, height: 400 }); + assert.ok(oneShotArgs?.includes('viewport')); + assert.equal(sessionCommands, 0); +}); + async function startFakeTouchHelperSession( device: DeviceInfo, handleCommand: TouchSessionCommandHandler, diff --git a/src/platforms/android/fill-verification.ts b/src/platforms/android/fill-verification.ts index f4a6fcd662..1f93a432f2 100644 --- a/src/platforms/android/fill-verification.ts +++ b/src/platforms/android/fill-verification.ts @@ -13,6 +13,7 @@ import { sleep } from './adb.ts'; import { getAndroidKeyboardState } from './device-input-state.ts'; import { isAndroidInputMethodOwnedNode } from '@agent-device/contracts/android-input-ownership'; import { captureAndroidUiHierarchyXml } from './snapshot.ts'; +import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; import type { FillUnconfirmedVerification, @@ -45,6 +46,7 @@ export async function verifyAndroidFilledText( x: number, y: number, expected: string, + helper: AndroidHelperSessionOptions = {}, ): Promise { const verificationDelaysMs = [0, 150, 350]; let lastVerification: AndroidFillVerification | null = null; @@ -55,7 +57,7 @@ export async function verifyAndroidFilledText( if (delayMs > 0) { await sleep(delayMs); } - const verification = await inspectAndroidFilledText(device, x, y, expected, context); + const verification = await inspectAndroidFilledText(device, x, y, expected, context, helper); lastVerification = verification; if (verification.reason === 'ime_capture') { return verification; @@ -83,18 +85,24 @@ export async function readAndroidTextAtPoint( device: DeviceInfo, x: number, y: number, + helper: AndroidHelperSessionOptions = {}, ): Promise { - return readAndroidTextAtPointInHierarchy(await captureAndroidUiHierarchyXml(device), x, y); + return readAndroidTextAtPointInHierarchy( + await captureAndroidUiHierarchyXml(device, helper), + x, + y, + ); } async function readAndroidFillTargetAtPoint( device: DeviceInfo, x: number, y: number, + helper: AndroidHelperSessionOptions, ): Promise { const context = await readAndroidFillVerificationContext(device); return inspectAndroidTextAtPointInHierarchy( - await captureAndroidUiHierarchyXml(device), + await captureAndroidUiHierarchyXml(device, helper), x, y, context, @@ -105,9 +113,10 @@ export async function readAndroidFillTargetBeforeMutation( device: DeviceInfo, x: number, y: number, + helper: AndroidHelperSessionOptions = {}, ): Promise { try { - return await readAndroidFillTargetAtPoint(device, x, y); + return await readAndroidFillTargetAtPoint(device, x, y, helper); } catch (error) { emitDiagnostic({ level: 'warn', @@ -233,9 +242,12 @@ async function inspectAndroidFilledText( y: number, expected: string, context: AndroidFillVerificationContext, + helper: AndroidHelperSessionOptions, ): Promise { + // Each delay samples the live hierarchy again — settling is what the samples observe, so they + // share the helper session but never a capture. return verifyAndroidFilledTextInHierarchy( - await captureAndroidUiHierarchyXml(device), + await captureAndroidUiHierarchyXml(device, helper), x, y, expected, diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index d9eb7a8872..86601f7236 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -34,6 +34,7 @@ import { } from './ime-helper.ts'; import { isAndroidTestImeActive } from './ime-lifecycle.ts'; import { executeAndroidTouchPlan, readAndroidGestureViewport } from './touch-executor.ts'; +import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; export { readAndroidTextAtPoint } from './fill-verification.ts'; @@ -148,17 +149,18 @@ export async function fillAndroid( y: number, text: string, delayMs = 0, + helper: AndroidHelperSessionOptions = {}, ): Promise { - const beforeTarget = await readAndroidFillTargetBeforeMutation(device, x, y); + const beforeTarget = await readAndroidFillTargetBeforeMutation(device, x, y, helper); const providerText = resolveAndroidTextInjector(device); if (providerText) { await providerText({ action: 'fill', target: { x, y }, text, delayMs }); emitAndroidTextDiagnostic('fill', 'provider-native', text); - const verification = await verifyAndroidFilledText(device, x, y, text); + const verification = await verifyAndroidFilledText(device, x, y, text, helper); return completeAndroidFillVerification(text, beforeTarget, verification); } if (isAndroidTestImeActive(device)) { - const verification = await fillAndroidTestIme(device, x, y, text, beforeTarget); + const verification = await fillAndroidTestIme(device, x, y, text, beforeTarget, helper); return completeAndroidFillVerification(text, beforeTarget, verification); } assertAndroidShellTextSupported(text); @@ -204,7 +206,7 @@ export async function fillAndroid( chunkSize: attempt.chunkSize, delayMs: attempt.inputDelayMs, }); - const verification = await verifyAndroidFilledText(device, x, y, text); + const verification = await verifyAndroidFilledText(device, x, y, text, helper); lastVerification = verification; if (verification.ok) return; if (verification.reason === 'ime_capture') { @@ -247,6 +249,7 @@ async function fillAndroidTestIme( y: number, text: string, beforeTarget: AndroidFillVerification['targetInput'], + helper: AndroidHelperSessionOptions, ): Promise { const adb = resolveAndroidAdbExecutor(device); const artifact = await selectAndroidImeHelperArtifact(resolveAndroidAdbProvider(device)); @@ -257,7 +260,7 @@ async function fillAndroidTestIme( await focusAndroid(device, x, y); await clearAndroidImeHelperText(adb, packageName); if (text) await sendAndroidImeHelperText(adb, packageName, text); - const verification = await verifyAndroidFilledText(device, x, y, text); + const verification = await verifyAndroidFilledText(device, x, y, text, helper); lastVerification = verification; if (verification.ok) break; if (buildAndroidFillUnconfirmedVerification(text, beforeTarget, verification)) break; @@ -269,9 +272,13 @@ async function fillAndroidTestIme( export async function scrollAndroid( device: DeviceInfo, direction: ScrollDirection, - options?: { amount?: number; pixels?: number; durationMs?: number }, + options?: { amount?: number; pixels?: number; durationMs?: number } & AndroidHelperSessionOptions, ): Promise> { - const viewport = await readAndroidGestureViewport(device); + // The viewport read and the gesture are two helper calls one command apart: giving the read the + // command's session scope keeps both on the same instrumentation. + const viewport = await readAndroidGestureViewport(device, { + helperSessionScope: options?.helperSessionScope, + }); const relativePlan = buildScrollGesturePlan({ direction, amount: options?.amount, diff --git a/src/platforms/android/snapshot-helper-capture.ts b/src/platforms/android/snapshot-helper-capture.ts index c9f4036635..f5151a11f2 100644 --- a/src/platforms/android/snapshot-helper-capture.ts +++ b/src/platforms/android/snapshot-helper-capture.ts @@ -6,7 +6,9 @@ import { readInstrumentationResultNumber, } from './instrumentation-helper.ts'; import { + ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS, ANDROID_SNAPSHOT_HELPER_COMMAND_OVERHEAD_MS, + ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS, ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT, ANDROID_SNAPSHOT_HELPER_PACKAGE, ANDROID_SNAPSHOT_HELPER_PROTOCOL, @@ -14,10 +16,13 @@ import { ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, } from './snapshot-helper-types.ts'; import type { + AndroidAdbExecutor, + AndroidSnapshotHelperArtifact, AndroidSnapshotHelperCaptureOptions, AndroidSnapshotHelperMetadata, AndroidSnapshotHelperOutput, } from './snapshot-helper-types.ts'; +import type { AndroidAdbProvider } from './adb-executor.ts'; import { recoverAndroidSnapshotHelperRetirement, retireCanceledAndroidSnapshotHelperCapture, @@ -98,6 +103,36 @@ export async function captureAndroidSnapshotWithHelper( return output; } +/** + * The single construction path for a helper call's capture options. + * + * A persistent session is keyed by the resolved options (see `createSessionIdentity`), so two + * callers that build these fields apart would each restart the other's session instead of sharing + * it. Snapshot capture and the gesture viewport read therefore build them here. + */ +export function buildAndroidSnapshotHelperCaptureOptions(params: { + adb: AndroidAdbExecutor; + adbProvider: AndroidAdbProvider; + artifact: AndroidSnapshotHelperArtifact; + deviceKey: string; + signal?: AbortSignal; +}): AndroidSnapshotHelperCaptureOptions { + return { + adb: params.adb, + adbProvider: params.adbProvider, + deviceKey: params.deviceKey, + helperVersion: params.artifact.manifest.version, + helperVersionCode: params.artifact.manifest.versionCode, + helperSha256: params.artifact.manifest.sha256, + packageName: params.artifact.manifest.packageName, + instrumentationRunner: params.artifact.manifest.instrumentationRunner, + waitForIdleTimeoutMs: ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, + timeoutMs: ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS, + commandTimeoutMs: ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS, + ...(params.signal ? { signal: params.signal } : {}), + }; +} + export function resolveAndroidSnapshotHelperCaptureOptions( options: AndroidSnapshotHelperCaptureOptions, ): AndroidSnapshotHelperResolvedCaptureOptions { diff --git a/src/platforms/android/snapshot-helper-retirement.ts b/src/platforms/android/snapshot-helper-retirement.ts index 76d9511649..e62775ddb5 100644 --- a/src/platforms/android/snapshot-helper-retirement.ts +++ b/src/platforms/android/snapshot-helper-retirement.ts @@ -100,15 +100,24 @@ export async function settleAndroidSnapshotHelperSessionCleanup(params: { port: number; packageName: string; timeoutMs: number; + /** + * Whether the device runtime still needs `am force-stop`. Only a quit the helper acknowledged + * AND whose process exit was observed proves UiAutomation was released; every forced, timed-out, + * or aborted teardown must still stop the runtime, because a daemon that dies without sending + * `quit` would otherwise leave the helper squatting UiAutomation for the next command. + */ + forceStopRuntime: boolean; }): Promise<{ timedOut: boolean; runtimeForceStopped: boolean }> { const signal = AbortSignal.timeout(params.timeoutMs); const results = await Promise.allSettled([ - forceStopAndroidSnapshotHelperRuntime({ - adb: params.adb, - packageName: params.packageName, - timeoutMs: params.timeoutMs, - signal, - }), + params.forceStopRuntime + ? forceStopAndroidSnapshotHelperRuntime({ + adb: params.adb, + packageName: params.packageName, + timeoutMs: params.timeoutMs, + signal, + }) + : Promise.resolve(false), removeAndroidSnapshotHelperSessionForward({ ...params, signal }), ] as const); const [runtimeStopResult] = results; diff --git a/src/platforms/android/snapshot-helper-session.ts b/src/platforms/android/snapshot-helper-session.ts index b75ffb10d4..14731b11dc 100644 --- a/src/platforms/android/snapshot-helper-session.ts +++ b/src/platforms/android/snapshot-helper-session.ts @@ -76,6 +76,38 @@ const disabledSessionIdentities = new Map(); export async function captureAndroidSnapshotWithHelperSession( options: AndroidSnapshotHelperCaptureOptions, ): Promise { + const acquired = await acquireAndroidSnapshotHelperSession(options); + if (!acquired) return undefined; + return await captureFromAndroidSnapshotHelperSession({ + session: acquired.session, + deviceKey: acquired.deviceKey, + options, + resolved: acquired.resolved, + }); +} + +/** + * Starts (or reuses) the session without capturing, so a helper-backed read that is not a snapshot + * — the gesture viewport — can leave a warm session behind for the gesture that follows instead of + * paying its own one-shot instrumentation. Answers whether a session is live; `false` means the + * caller must use the one-shot transport, exactly as when a capture cannot use the session. + */ +export async function ensureAndroidSnapshotHelperSession( + options: AndroidSnapshotHelperCaptureOptions, +): Promise { + return (await acquireAndroidSnapshotHelperSession(options)) !== undefined; +} + +async function acquireAndroidSnapshotHelperSession( + options: AndroidSnapshotHelperCaptureOptions, +): Promise< + | { + session: AndroidSnapshotHelperSession; + resolved: AndroidSnapshotHelperResolvedCaptureOptions; + deviceKey: string; + } + | undefined +> { const deviceKey = options.deviceKey ?? 'android:default'; await recoverAndroidSnapshotHelperRetirement({ deviceKey, @@ -95,13 +127,7 @@ export async function captureAndroidSnapshotWithHelperSession( options, resolved, }); - if (!session) return undefined; - return await captureFromAndroidSnapshotHelperSession({ - session, - deviceKey, - options, - resolved, - }); + return session ? { session, resolved, deviceKey } : undefined; } async function resolveAndroidSnapshotHelperSession(params: { @@ -294,6 +320,10 @@ export async function stopAndroidSnapshotHelperSession( const processExit = observeAndroidSnapshotHelperProcessExit(session.process); const force = options.force === true || options.signal?.aborted === true; const graceful = await requestGracefulSessionExit(session, processExit, force, options.signal); + // The helper releases UiAutomation inside its own quit handling, so an acknowledged quit whose + // process exit was then observed IS the release evidence. Anything less — including an ack whose + // exit never arrived — still needs the device-side stop. + const runtimeReleaseConfirmed = graceful.acknowledged && graceful.exited; const cleanupTimeoutMs = !force ? FORWARD_TIMEOUT_MS : options.signal?.aborted === true @@ -315,6 +345,7 @@ export async function stopAndroidSnapshotHelperSession( port: session.port, packageName: session.helper.packageName, timeoutMs: cleanupTimeoutMs, + forceStopRuntime: !runtimeReleaseConfirmed, }), ]); emitDiagnostic({ @@ -325,6 +356,7 @@ export async function stopAndroidSnapshotHelperSession( capturedCount: session.capturedCount, lifetimeMs: Date.now() - session.startedAtMs, quitAcknowledged: graceful.acknowledged, + runtimeReleaseConfirmed, forceKilled: !graceful.exited && processStopped, forced: force || options.signal?.aborted === true, runtimeForceStopped: cleanup.runtimeForceStopped, @@ -467,6 +499,9 @@ async function startAndroidSnapshotHelperSession(params: { port: session.port, packageName: session.helper.packageName, timeoutMs: ANDROID_SNAPSHOT_HELPER_DEVICE_RETIREMENT_TIMEOUT_MS, + // Startup failed before the helper could acknowledge anything, so nothing proves it + // released UiAutomation. + forceStopRuntime: true, }), ]); if (!cleanup.runtimeForceStopped) { diff --git a/src/platforms/android/snapshot-helper-types.ts b/src/platforms/android/snapshot-helper-types.ts index 353a2d698c..be09881f4c 100644 --- a/src/platforms/android/snapshot-helper-types.ts +++ b/src/platforms/android/snapshot-helper-types.ts @@ -20,6 +20,23 @@ export const ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT = 'uiautomator-xml'; export const ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS = 500; export const ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_QUIET_MS = 100; export const ANDROID_SNAPSHOT_HELPER_COMMAND_OVERHEAD_MS = 5_000; +export const ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS = 5_000; +export const ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS = 30_000; + +/** + * Who releases the helper's persistent instrumentation session. + * + * Android permits ONE UiAutomation owner, so a `command`-scoped call stops the session when it + * finishes and the next helper call pays a fresh `am instrument` start plus the UiAutomation + * connect wait. `daemon-session` hands that release to session teardown + * (`stopSessionAndroidSnapshotHelper`), which every Android session runs, so consecutive commands + * in one session share one warm helper. Device-scoped work stays `command` so nothing squats + * UiAutomation once the command returns. + */ +export type AndroidHelperSessionScope = 'command' | 'daemon-session'; + +/** Threaded by every helper-backed read a session command performs (capture, viewport). */ +export type AndroidHelperSessionOptions = { helperSessionScope?: AndroidHelperSessionScope }; export type { AndroidAdbExecutor } from './adb-executor.ts'; diff --git a/src/platforms/android/snapshot-helper.ts b/src/platforms/android/snapshot-helper.ts index 49979cc7fc..016a85e9cd 100644 --- a/src/platforms/android/snapshot-helper.ts +++ b/src/platforms/android/snapshot-helper.ts @@ -12,7 +12,6 @@ export { ensureAndroidSnapshotHelper, forgetAndroidSnapshotHelperInstall, } from './snapshot-helper-install.ts'; -export { ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS } from './snapshot-helper-types.ts'; export type { AndroidAdbExecutor, diff --git a/src/platforms/android/snapshot.ts b/src/platforms/android/snapshot.ts index e1b170ff66..21c6db0eec 100644 --- a/src/platforms/android/snapshot.ts +++ b/src/platforms/android/snapshot.ts @@ -27,10 +27,15 @@ import { import { buildAndroidSnapshotClickabilityEvidence } from './snapshot-clickability.ts'; import { resolveAndroidAdbProvider, type AndroidAdbProvider } from './adb-executor.ts'; import { sleep } from './adb.ts'; +import { buildAndroidSnapshotHelperCaptureOptions } from './snapshot-helper-capture.ts'; +import { + ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS, + ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS, + type AndroidHelperSessionScope, +} from './snapshot-helper-types.ts'; import { captureAndroidSnapshotWithHelper, captureAndroidSnapshotWithHelperSession, - ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, ensureAndroidSnapshotHelper, forgetAndroidSnapshotHelperInstall, getAndroidSnapshotHelperSessionDeviceKey, @@ -55,8 +60,6 @@ import { } from './snapshot-helper-runtime.ts'; const HELPER_INSTALL_TIMEOUT_MS = 30_000; -const HELPER_CAPTURE_TIMEOUT_MS = 5_000; -const HELPER_COMMAND_TIMEOUT_MS = 30_000; /** * A content verdict means the capture mechanism worked but sampled a screen * mid-transition, which resolves on its own within a frame or two. Sampling @@ -70,7 +73,7 @@ export type AndroidSnapshotOptions = SnapshotOptions & { signal?: AbortSignal; helperArtifact?: AndroidSnapshotHelperArtifact; helperInstallPolicy?: AndroidSnapshotHelperInstallPolicy; - helperSessionScope?: 'command' | 'daemon-session'; + helperSessionScope?: AndroidHelperSessionScope; helperAdb?: AndroidAdbExecutor | AndroidAdbProvider; includeHiddenContentHints?: boolean; }; @@ -302,20 +305,13 @@ async function captureAndroidUiHierarchyFromHelper(params: { helperDeviceKey: string; }): Promise { const { signal, adb, adbProvider, artifact, helperDeviceKey } = params; - const captureOptions = { + const captureOptions = buildAndroidSnapshotHelperCaptureOptions({ adb, adbProvider, + artifact, deviceKey: helperDeviceKey, - helperVersion: artifact.manifest.version, - helperVersionCode: artifact.manifest.versionCode, - helperSha256: artifact.manifest.sha256, - packageName: artifact.manifest.packageName, - instrumentationRunner: artifact.manifest.instrumentationRunner, - waitForIdleTimeoutMs: ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, - timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, - commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, signal, - }; + }); try { const sessionCapture = await withDiagnosticTimer( 'android_snapshot_helper_session_capture', @@ -323,7 +319,7 @@ async function captureAndroidUiHierarchyFromHelper(params: { { packageName: artifact.manifest.packageName, version: artifact.manifest.version, - timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, + timeoutMs: ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS, }, ); if (sessionCapture) return sessionCapture; @@ -345,8 +341,8 @@ async function captureAndroidUiHierarchyFromHelper(params: { { packageName: artifact.manifest.packageName, version: artifact.manifest.version, - timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, - commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, + timeoutMs: ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS, + commandTimeoutMs: ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS, }, ); } diff --git a/src/platforms/android/touch-executor.ts b/src/platforms/android/touch-executor.ts index eaf99fddfe..409ac6b899 100644 --- a/src/platforms/android/touch-executor.ts +++ b/src/platforms/android/touch-executor.ts @@ -4,6 +4,7 @@ import { resolveAndroidTouchProvider } from './adb-executor.ts'; import { executeAndroidTouchHelperPlan, readAndroidTouchHelperViewport } from './touch-helper.ts'; import { validateAndroidGestureViewport } from './gesture-viewport.ts'; import { lowerAndroidTouchPlan, type AndroidTouchPlan } from './touch-plan.ts'; +import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; export async function executeAndroidTouchPlan( device: DeviceInfo, @@ -25,8 +26,11 @@ export async function executeAndroidTouchPlan( return await executeAndroidTouchHelperPlan(device, loweredPlan); } -export async function readAndroidGestureViewport(device: DeviceInfo): Promise { +export async function readAndroidGestureViewport( + device: DeviceInfo, + helper: AndroidHelperSessionOptions = {}, +): Promise { const provider = resolveAndroidTouchProvider(device); if (provider) return validateAndroidGestureViewport(await provider.gestureViewport()); - return await readAndroidTouchHelperViewport(device); + return await readAndroidTouchHelperViewport(device, helper); } diff --git a/src/platforms/android/touch-helper.ts b/src/platforms/android/touch-helper.ts index 8a6f5f4f80..c810d30c53 100644 --- a/src/platforms/android/touch-helper.ts +++ b/src/platforms/android/touch-helper.ts @@ -4,7 +4,11 @@ import type { Rect } from '@agent-device/kernel/snapshot'; import { AppError } from '@agent-device/kernel/errors'; import { execFailureDetails } from '../../utils/exec.ts'; import { emitDiagnostic, withDiagnosticTimer } from '../../utils/diagnostics.ts'; -import { resolveAndroidAdbProvider, type AndroidAdbExecutor } from './adb-executor.ts'; +import { + resolveAndroidAdbProvider, + type AndroidAdbExecutor, + type AndroidAdbProvider, +} from './adb-executor.ts'; import { parseInstrumentationRecords, readInstrumentationResultNumber, @@ -15,13 +19,16 @@ import { resolveAndroidHelperArtifact } from './helper-package-install.ts'; import { parseAndroidSnapshotHelperManifest } from './snapshot-helper-artifact.ts'; import { ensureAndroidSnapshotHelper } from './snapshot-helper-install.ts'; import { + ensureAndroidSnapshotHelperSession, getAndroidSnapshotHelperSessionDeviceKey, recoverAndroidSnapshotHelperRetirement, runAndroidSnapshotHelperSessionTouchCommand, stopAndroidSnapshotHelperSession, } from './snapshot-helper-session.ts'; +import { buildAndroidSnapshotHelperCaptureOptions } from './snapshot-helper-capture.ts'; import { ANDROID_SNAPSHOT_HELPER_PROTOCOL, + type AndroidHelperSessionOptions, type AndroidSnapshotHelperArtifact, type AndroidSnapshotHelperInstallResult, } from './snapshot-helper-types.ts'; @@ -47,6 +54,7 @@ type AndroidTouchHelperGestureRequest = { type PreparedAndroidTouchHelper = { adb: AndroidAdbExecutor; + adbProvider: AndroidAdbProvider; artifact: AndroidSnapshotHelperArtifact; install: AndroidSnapshotHelperInstallResult; deviceKey: string; @@ -105,8 +113,24 @@ export async function executeAndroidTouchHelperPlan( }; } -export async function readAndroidTouchHelperViewport(device: DeviceInfo): Promise { +export async function readAndroidTouchHelperViewport( + device: DeviceInfo, + helper: AndroidHelperSessionOptions = {}, +): Promise { const prepared = await prepareAndroidTouchHelper(device); + if (helper.helperSessionScope === 'daemon-session') { + // Without a live session both this read and the gesture that follows would each start their + // own `am instrument`. The session outlives the command under this scope, so warming it here + // makes the pair share one instrumentation; session teardown still owns the release. + await ensureAndroidSnapshotHelperSession( + buildAndroidSnapshotHelperCaptureOptions({ + adb: prepared.adb, + adbProvider: prepared.adbProvider, + artifact: prepared.artifact, + deviceKey: prepared.deviceKey, + }), + ); + } try { const sessionHeaders = await runAndroidSnapshotHelperSessionTouchCommand({ deviceKey: prepared.deviceKey, @@ -188,7 +212,7 @@ async function prepareAndroidTouchHelper(device: DeviceInfo): Promise { From 8ceac1bfe7a66bca61b5494f9fae1a84a5629d69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 22 Aug 2026 15:00:32 +0200 Subject: [PATCH 2/4] fix(android): reset the helper runtime on content-failure retirement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit REVIEW.md F1. Gating the teardown force-stop on confirmed release also changed what `stopAndroidSnapshotHelperSession` returning `true` means to its callers. `retireAndroidSnapshotHelperAfterContentFailure` reads that return as "the session stop was the runtime reset" and skips `resetAndroidSnapshotHelperRuntime` when it is true, so a helper whose output failed content validation three times over a daemon session was retired with no `am force-stop` at all, while the one-shot arm of the same recovery still reset the runtime. The session stop now takes `resetRuntime`, and the retirement passes it. Content failure is a recovery path, not a clean release: the helper answered with output we could not trust, so the next capture must meet a runtime that was reset, and "it quit politely" is not a reason to leave a suspect process owning it. Genuine session close still skips the round trip, which is the optimization this branch exists for. The pre-existing test "content failure retirement does not layer a second reset over a persistent session stop" rested on a premise this branch had already made false — stopping a live session was no longer the runtime reset. It is renamed to "makes the session stop reset the runtime instead of layering a second one" and asserts the requirement it now depends on, so its title states what it enforces. The reviewer's probe returns as a regression test in `snapshot.test.ts`. REVIEW.md F2. `observeAndroidSnapshotHelperProcessExit` returns an observation that knows whether the end it saw is release evidence: the process must have been alive when the teardown started watching and must then exit with code 0 and no terminating signal. A signal, a non-zero code, and a host child that was already gone before `quit` was sent all mean the transport died — and in the pre-died case the acknowledgement is positive evidence that the device-side helper OUTLIVED its host through the open forward. None of them confirm release now. What is left is declared at the gate: host-side exit codes are only as strong as adb's exit forwarding, and a device without shell protocol v2 can report 0 for an instrumentation that did not finish. REVIEW.md F3. `&& graceful.exited` survived deletion against all 423 tests. Two named tests now fail without it, both cheap because the stricter evidence above makes acknowledged-but-not-released reachable without waiting out the 11 s graceful-exit timeout. Gates: `pnpm vitest run src/platforms/android` (44 files, 423 tests) and with `src/core/interactors` (426 tests), `pnpm typecheck`, `pnpm lint`, `pnpm format`, `pnpm check:fallow --base origin/main` (24 changed files, clean). No emulator is attached in this worktree, so the teardown-path force-stop counts were not re-observed live. --- .../__tests__/snapshot-helper-runtime.test.ts | 6 +- .../__tests__/snapshot-helper-session.test.ts | 47 ++++++++++++++- .../android/__tests__/snapshot.test.ts | 27 +++++++++ .../android/snapshot-helper-retirement.ts | 60 ++++++++++++++----- .../android/snapshot-helper-runtime.ts | 5 ++ .../android/snapshot-helper-session.ts | 45 ++++++++++---- 6 files changed, 161 insertions(+), 29 deletions(-) diff --git a/src/platforms/android/__tests__/snapshot-helper-runtime.test.ts b/src/platforms/android/__tests__/snapshot-helper-runtime.test.ts index 0cc533102e..285f6043ba 100644 --- a/src/platforms/android/__tests__/snapshot-helper-runtime.test.ts +++ b/src/platforms/android/__tests__/snapshot-helper-runtime.test.ts @@ -16,7 +16,7 @@ beforeEach(() => { stopSession.mockReset(); }); -test('content failure retirement does not layer a second reset over a persistent session stop', async () => { +test('content failure retirement makes the session stop reset the runtime instead of layering a second one', async () => { stopSession.mockResolvedValueOnce(true); const calls: string[][] = []; const adb: AndroidAdbExecutor = async (args) => { @@ -32,6 +32,10 @@ test('content failure retirement does not layer a second reset over a persistent }); assert.equal(stopSession.mock.calls.length, 1); + // The session stop only force-stops the runtime when it has to. Recovery cannot read a clean quit + // as a reason to leave a suspect helper running, so it states that requirement instead of issuing + // a second `am force-stop` of its own. + assert.equal(stopSession.mock.calls[0]?.[1]?.resetRuntime, true); assert.equal(calls.length, 0); }); diff --git a/src/platforms/android/__tests__/snapshot-helper-session.test.ts b/src/platforms/android/__tests__/snapshot-helper-session.test.ts index e37b82ac3f..1388bbe61a 100644 --- a/src/platforms/android/__tests__/snapshot-helper-session.test.ts +++ b/src/platforms/android/__tests__/snapshot-helper-session.test.ts @@ -300,6 +300,46 @@ test('a quit acknowledged and followed by process exit skips the force-stop roun assert.equal(calls.some(isHelperRuntimeForceStop), false); }); +test('a quit acknowledged by a helper the host then killed still force-stops the runtime', async () => { + const calls: string[][] = []; + const processes: FakeAndroidProcess[] = []; + const provider = createSessionProvider({ + calls, + processes, + quitExit: { code: null, signal: 'SIGKILL' }, + }); + + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + }); + await resetAndroidSnapshotHelperSessions(); + + // The ack alone only says the helper heard us. A host process that ended on a signal says the + // transport died, not that the instrumentation finished releasing UiAutomation — so the second + // half of the release evidence is missing and the device-side stop must still run. + assert.equal(calls.some(isHelperRuntimeForceStop), true); +}); + +test('a quit acknowledged after the host process already died still force-stops the runtime', async () => { + const calls: string[][] = []; + const processes: FakeAndroidProcess[] = []; + const provider = createSessionProvider({ calls, processes }); + + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + }); + // The host `am instrument` child is gone before the teardown starts, yet the device-side helper + // answers `quit` through the still-open forward: positive evidence that it OUTLIVED its host. + processes[0]?.emitExit(0, null); + await resetAndroidSnapshotHelperSessions(); + + assert.equal(calls.some(isHelperRuntimeForceStop), true); +}); + test('force terminates the helper when quit is not acknowledged', async () => { const calls: string[][] = []; const processes: FakeAndroidProcess[] = []; @@ -397,6 +437,7 @@ type SessionProviderOptions = { calls: string[][]; cleanupAborts?: string[][]; processes?: FakeAndroidProcess[]; + quitExit?: { code: number | null; signal: NodeJS.Signals | null }; quitExitDelayMs?: number; quitResponseMode?: 'ok' | 'malformed'; spawnArgs?: string[][]; @@ -431,8 +472,12 @@ function createSessionProvider(options: SessionProviderOptions): AndroidAdbProvi return; } socket.end(sessionResponse({ requestId, body: '' })); + const quitExit = options.quitExit ?? { code: 0, signal: null }; server.close(() => { - setTimeout(() => process.emitExit(0, null), options.quitExitDelayMs ?? 0); + setTimeout( + () => process.emitExit(quitExit.code, quitExit.signal), + options.quitExitDelayMs ?? 0, + ); }); return; } diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index 63fb5e7502..ab9cf07d9d 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -607,6 +607,33 @@ test('snapshotAndroid retires content-invalid daemon helper before the next requ ); }); +test('content-invalid daemon helper retirement force-stops the helper runtime', async () => { + // Retirement after a content failure is a recovery path, not a release: the helper answered with + // output we could not trust, so the next capture must meet a runtime that was reset. A clean quit + // is evidence the helper let go of UiAutomation, never evidence that it was healthy. + const adbCalls: string[][] = []; + const spawnArgs: string[][] = []; + const processes: FakeAndroidProcess[] = []; + const provider = createPersistentSnapshotHelperProvider({ + calls: adbCalls, + spawnArgs, + processes, + sessionXml: () => androidSystemWindowOnlyXml(), + }); + + await assert.rejects( + snapshotAndroid(device, { + helperAdb: provider, + helperArtifact, + helperSessionScope: 'daemon-session', + }), + /Android snapshot helper returned only non-application windows/, + ); + + assert.equal(processes[0]?.exitCode, 0, 'the session quit cleanly'); + assert.equal(adbCalls.some(isHelperRuntimeReset), true); +}); + test('snapshotAndroid falls back to one-shot capture after retiring a failed session', async () => { const adbCalls: string[][] = []; const spawnArgs: string[][] = []; diff --git a/src/platforms/android/snapshot-helper-retirement.ts b/src/platforms/android/snapshot-helper-retirement.ts index e62775ddb5..755e941491 100644 --- a/src/platforms/android/snapshot-helper-retirement.ts +++ b/src/platforms/android/snapshot-helper-retirement.ts @@ -102,9 +102,10 @@ export async function settleAndroidSnapshotHelperSessionCleanup(params: { timeoutMs: number; /** * Whether the device runtime still needs `am force-stop`. Only a quit the helper acknowledged - * AND whose process exit was observed proves UiAutomation was released; every forced, timed-out, - * or aborted teardown must still stop the runtime, because a daemon that dies without sending - * `quit` would otherwise leave the helper squatting UiAutomation for the next command. + * AND then completed cleanly proves UiAutomation was released; every forced, timed-out, or + * aborted teardown must still stop the runtime, because a daemon that dies without sending + * `quit` would otherwise leave the helper squatting UiAutomation for the next command. Recovery + * paths that distrust the helper's output require the stop regardless of that evidence. */ forceStopRuntime: boolean; }): Promise<{ timedOut: boolean; runtimeForceStopped: boolean }> { @@ -128,14 +129,44 @@ export async function settleAndroidSnapshotHelperSessionCleanup(params: { }; } -export function observeAndroidSnapshotHelperProcessExit(process: AndroidAdbProcess): Promise { - if (process.exitCode != null || process.signalCode != null) { - return Promise.resolve(); - } - return new Promise((resolve) => { - process.once('close', () => resolve()); - process.once('exit', () => resolve()); - }); +/** + * Watches one host `adb shell am instrument` child, and remembers whether the end it saw is + * evidence that the instrumentation FINISHED rather than that the transport merely died. + */ +export type AndroidSnapshotHelperProcessExit = { + /** Resolves when the process is gone — already resolved when it was gone before observation. */ + ended: Promise; + /** Whether the process is gone, by any cause. */ + hasEnded(): boolean; + /** + * Whether this observation started on a live process that then exited with code 0 and no + * terminating signal. A signal, a non-zero code, or a process that was already gone says the + * host child died: adb restarted, the transport dropped, something killed it. None of those say + * the device-side helper released UiAutomation — it can outlive its host through an open forward. + */ + completedCleanly(): boolean; +}; + +export function observeAndroidSnapshotHelperProcessExit( + process: AndroidAdbProcess, +): AndroidSnapshotHelperProcessExit { + const endedBeforeObservation = hasAndroidSnapshotHelperProcessEnded(process); + const ended = endedBeforeObservation + ? Promise.resolve() + : new Promise((resolve) => { + process.once('close', () => resolve()); + process.once('exit', () => resolve()); + }); + return { + ended, + hasEnded: () => hasAndroidSnapshotHelperProcessEnded(process), + completedCleanly: () => + !endedBeforeObservation && process.exitCode === 0 && process.signalCode == null, + }; +} + +function hasAndroidSnapshotHelperProcessEnded(process: AndroidAdbProcess): boolean { + return process.exitCode != null || process.signalCode != null; } export async function waitForAndroidSnapshotHelperProcessExit( @@ -167,17 +198,16 @@ export async function waitForAndroidSnapshotHelperProcessExit( export async function stopAndroidSnapshotHelperHostProcess(params: { process: AndroidAdbProcess; - processExit: Promise; - alreadyExited: boolean; + processExit: AndroidSnapshotHelperProcessExit; timeoutMs: number; }): Promise { - if (params.alreadyExited) return true; + if (params.processExit.hasEnded()) return true; try { params.process.kill('SIGTERM'); } catch { // A completed instrumentation process can reject or ignore the signal. } - return await waitForAndroidSnapshotHelperProcessExit(params.processExit, params.timeoutMs); + return await waitForAndroidSnapshotHelperProcessExit(params.processExit.ended, params.timeoutMs); } export function resetAndroidSnapshotHelperRetirements(): void { diff --git a/src/platforms/android/snapshot-helper-runtime.ts b/src/platforms/android/snapshot-helper-runtime.ts index ba02a096a0..cf74db73b1 100644 --- a/src/platforms/android/snapshot-helper-runtime.ts +++ b/src/platforms/android/snapshot-helper-runtime.ts @@ -15,11 +15,16 @@ export async function retireAndroidSnapshotHelperAfterContentFailure(params: { cause: unknown; }): Promise { const retiredPersistentSession = await stopAndroidSnapshotHelperSession(params.deviceKey, { + // Content failure is a recovery path, not a clean release: the helper answered with output we + // could not trust, so the next capture must meet a runtime that was reset. The session stop + // owns that reset, so it is required here rather than layered on afterwards. + resetRuntime: true, signal: params.signal, cause: params.cause, }); params.signal?.throwIfAborted(); if (!retiredPersistentSession) { + // The suspect helper ran one-shot, so no session stop reset the runtime for us. await resetAndroidSnapshotHelperRuntime(params.adb, params.packageName); } } diff --git a/src/platforms/android/snapshot-helper-session.ts b/src/platforms/android/snapshot-helper-session.ts index 14731b11dc..6cc2914412 100644 --- a/src/platforms/android/snapshot-helper-session.ts +++ b/src/platforms/android/snapshot-helper-session.ts @@ -22,6 +22,7 @@ import { waitForAndroidSnapshotHelperSessionReady, } from './snapshot-helper-session-protocol.ts'; import { + type AndroidSnapshotHelperProcessExit, ANDROID_SNAPSHOT_HELPER_DEVICE_RETIREMENT_TIMEOUT_MS, ANDROID_SNAPSHOT_HELPER_HOST_PROCESS_EXIT_GRACE_MS, getAndroidSnapshotHelperSessionDeviceKey, @@ -312,7 +313,18 @@ function matchesWhenBothDefined(a: Value | undefined, b: Value | undefine export async function stopAndroidSnapshotHelperSession( deviceKey: string, - options: { force?: boolean; signal?: AbortSignal; cause?: unknown } = {}, + options: { + /** Skip the graceful quit entirely: kill the host process and stop the device runtime. */ + force?: boolean; + /** + * Stop the device runtime even when the quit proved release. Recovery paths — a helper whose + * output failed content validation — restart the helper on purpose, so they cannot read "it + * quit politely" as a reason to leave a suspect process owning the runtime. + */ + resetRuntime?: boolean; + signal?: AbortSignal; + cause?: unknown; + } = {}, ): Promise { const session = sessions.get(deviceKey); if (!session) return false; @@ -320,9 +332,14 @@ export async function stopAndroidSnapshotHelperSession( const processExit = observeAndroidSnapshotHelperProcessExit(session.process); const force = options.force === true || options.signal?.aborted === true; const graceful = await requestGracefulSessionExit(session, processExit, force, options.signal); - // The helper releases UiAutomation inside its own quit handling, so an acknowledged quit whose - // process exit was then observed IS the release evidence. Anything less — including an ack whose - // exit never arrived — still needs the device-side stop. + const hostProcessEnded = processExit.hasEnded(); + // The helper releases UiAutomation inside its own quit handling, so a quit it acknowledged and + // then completed IS the release evidence. Both halves are load-bearing: `exited` is what + // separates "the helper said it would quit" from "the helper finished quitting", and it counts + // only a clean exit of the process this teardown watched (see AndroidSnapshotHelperProcessExit). + // Anything less still needs the device-side stop. Declared residual: this is host-side evidence, + // so it is only as strong as adb's exit-code forwarding — a device without shell protocol v2 can + // report 0 for an instrumentation that did not finish, and the stop is then wrongly skipped. const runtimeReleaseConfirmed = graceful.acknowledged && graceful.exited; const cleanupTimeoutMs = !force ? FORWARD_TIMEOUT_MS @@ -336,7 +353,6 @@ export async function stopAndroidSnapshotHelperSession( stopAndroidSnapshotHelperHostProcess({ process: session.process, processExit, - alreadyExited: graceful.exited, timeoutMs: processExitTimeoutMs, }), settleAndroidSnapshotHelperSessionCleanup({ @@ -345,7 +361,7 @@ export async function stopAndroidSnapshotHelperSession( port: session.port, packageName: session.helper.packageName, timeoutMs: cleanupTimeoutMs, - forceStopRuntime: !runtimeReleaseConfirmed, + forceStopRuntime: options.resetRuntime === true || !runtimeReleaseConfirmed, }), ]); emitDiagnostic({ @@ -357,7 +373,7 @@ export async function stopAndroidSnapshotHelperSession( lifetimeMs: Date.now() - session.startedAtMs, quitAcknowledged: graceful.acknowledged, runtimeReleaseConfirmed, - forceKilled: !graceful.exited && processStopped, + forceKilled: !hostProcessEnded && processStopped, forced: force || options.signal?.aborted === true, runtimeForceStopped: cleanup.runtimeForceStopped, externalCleanupTimedOut: cleanup.timedOut, @@ -375,10 +391,14 @@ export async function stopAndroidSnapshotHelperSession( async function requestGracefulSessionExit( session: AndroidSnapshotHelperSession, - processExit: Promise, + processExit: AndroidSnapshotHelperProcessExit, force: boolean, signal: AbortSignal | undefined, -): Promise<{ acknowledged: boolean; exited: boolean }> { +): Promise<{ + acknowledged: boolean; + /** The instrumentation this teardown asked to quit then finished on its own, cleanly. */ + exited: boolean; +}> { if (force) return { acknowledged: false, exited: false }; const requestId = `quit-${Date.now()}`; try { @@ -392,10 +412,11 @@ async function requestGracefulSessionExit( const exited = acknowledged && (await waitForAndroidSnapshotHelperProcessExit( - processExit, + processExit.ended, SESSION_GRACEFUL_EXIT_TIMEOUT_MS, signal, - )); + )) && + processExit.completedCleanly(); return { acknowledged, exited }; } catch { return { acknowledged: false, exited: false }; @@ -490,7 +511,7 @@ async function startAndroidSnapshotHelperSession(params: { } const [, cleanup] = await Promise.all([ waitForAndroidSnapshotHelperProcessExit( - processExit, + processExit.ended, ANDROID_SNAPSHOT_HELPER_HOST_PROCESS_EXIT_GRACE_MS, ), settleAndroidSnapshotHelperSessionCleanup({ From 89f2c0cf5817f42e60f367d993a5dc6bf715fcb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 22 Aug 2026 15:45:34 +0200 Subject: [PATCH 3/4] test: lower the snapshot.test.ts ratchet pin to 1,495 (PR #1974 CI) This branch extracted the helper-session, retirement, and touch-helper cases out of src/platforms/android/__tests__/snapshot.test.ts into sibling files, taking it from 1,658 to 1,495 lines. The test-file size ratchet fails an un-banked shrink ("lower its pin in this PR so the ratchet keeps the gain"), so bank it. The pin only moves down; nothing grew into it. --- scripts/__tests__/test-file-size-ratchet.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/__tests__/test-file-size-ratchet.test.ts b/scripts/__tests__/test-file-size-ratchet.test.ts index c2dc784c3a..16c3bb36fb 100644 --- a/scripts/__tests__/test-file-size-ratchet.test.ts +++ b/scripts/__tests__/test-file-size-ratchet.test.ts @@ -40,7 +40,7 @@ const PINNED_TEST_FILE_LINES: Readonly> = Object.freeze({ 'src/platforms/apple/core/__tests__/runner-session.test.ts': 2001, 'src/utils/__tests__/daemon-client.test.ts': 1910, 'src/utils/__tests__/output.test.ts': 1861, - 'src/platforms/android/__tests__/snapshot.test.ts': 1658, + 'src/platforms/android/__tests__/snapshot.test.ts': 1495, 'src/platforms/apple/core/__tests__/runner-client.test.ts': 1615, 'src/__tests__/client.test.ts': 1598, 'test/integration/provider-scenarios/android-lifecycle.test.ts': 1559, From 8d28c1d6113f823232912ebd62700ad3d3111bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 22 Aug 2026 18:45:40 +0200 Subject: [PATCH 4/4] refactor(android): own helper-session lifecycle and text input in their own modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #1974 review, blocker 1. Both files the branch grew were already at the extraction threshold, so the behavior landed on top of a boundary that should have moved first. `snapshot-helper-session.ts` (563) carried two concepts: who owns the device's UiAutomation, and what runs over that ownership. Ownership — the live-session registry, the enable gate, session identity, start, reuse, and retirement — moves to `snapshot-helper-session-lifecycle.ts`. What is left is 170 lines of snapshot capture and touch commands that acquire through it and never reach the registry themselves; the touch path reads ownership through `getLiveAndroidSnapshotHelperSession` instead of the map. The stale re-export block for the retirement symbols is gone with its last importer. `input-actions.ts` (508, pre-branch 501) sheds every text-entry path to `text-input.ts`: provider injection, the test IME, the adb-shell writer, and the fill orchestration this branch changed. What is left is 172 lines of pointer, key, and gesture actions — below the pre-branch length. Tests follow their owners. `snapshot-helper-session.test.ts` splits around the same seam into a lifecycle file (start, reuse, identity, teardown, quarantine) and a capture file (budgets, cancellation, fallback), and its session fake becomes a named export of the sibling fixture module, which drops the duplicated process double. `input-actions-fill.test.ts` and `input-actions-test-ime.test.ts` are renamed to `text-input-*`, and the `typeAndroid`/`fillAndroid` cases in `input-actions.test.ts` move to a new `text-input.test.ts`. Blocker 2. The teardown skipped `am force-stop` on an acknowledged quit plus a clean host exit, and its own comment admitted the hole: adb without shell protocol v2 exits 0 whenever the connection closed cleanly, including for instrumentation that never finished, so that exit status was never proof the device released UiAutomation. A residual-risk paragraph is not an invariant. `adb-shell-protocol.ts` asks the transport instead. `adb features` lists only the features both ends negotiated, so `shell_v2` there is positive proof that a host `adb shell` child's exit status is the device command's. The skip now requires it; an unsupported transport, a failed probe, and an adb too old to know `features` all keep the device-side stop. The probe is host-side and cached per device, so it costs one query per device rather than one per teardown, and it clears with the session registry. Quarantine keys on the same proven-release signal, so an unproven quit whose stop also failed now reports unknown ownership instead of trusting the quit. Red first: a transport without `shell_v2` and an unanswerable probe both skipped the stop against the pre-fix gate, and the cache fence saw no probe at all. Rebased onto origin/main, which had moved the fill files. --- src/core/interactors/android.test.ts | 7 +- src/core/interactors/android.ts | 6 +- .../__tests__/adb-shell-protocol.test.ts | 101 ++++ .../__tests__/fill-verification.test.ts | 2 +- .../android/__tests__/input-actions.test.ts | 254 +-------- .../__tests__/snapshot-clickability.test.ts | 2 +- .../__tests__/snapshot-helper-runtime.test.ts | 2 +- .../snapshot-helper-session-lifecycle.test.ts | 317 ++++++++++++ .../snapshot-helper-session.fixtures.ts | 203 ++++++++ .../__tests__/snapshot-helper-session.test.ts | 480 +----------------- .../android/__tests__/snapshot.test.ts | 2 +- ...s-fill.test.ts => text-input-fill.test.ts} | 4 +- ...me.test.ts => text-input-test-ime.test.ts} | 2 +- .../android/__tests__/text-input.test.ts | 424 ++++++++++++++++ .../__tests__/touch-helper-session.test.ts | 8 +- .../android/__tests__/touch-helper.test.ts | 2 +- src/platforms/android/adb-shell-protocol.ts | 59 +++ src/platforms/android/input-actions.ts | 347 +------------ .../android/snapshot-helper-runtime.ts | 2 +- .../snapshot-helper-session-lifecycle.ts | 449 ++++++++++++++++ .../android/snapshot-helper-session.ts | 431 +--------------- src/platforms/android/snapshot-helper.ts | 10 +- src/platforms/android/text-input.ts | 350 +++++++++++++ src/platforms/android/touch-helper.ts | 8 +- 24 files changed, 1962 insertions(+), 1510 deletions(-) create mode 100644 src/platforms/android/__tests__/adb-shell-protocol.test.ts create mode 100644 src/platforms/android/__tests__/snapshot-helper-session-lifecycle.test.ts rename src/platforms/android/__tests__/{input-actions-fill.test.ts => text-input-fill.test.ts} (99%) rename src/platforms/android/__tests__/{input-actions-test-ime.test.ts => text-input-test-ime.test.ts} (98%) create mode 100644 src/platforms/android/__tests__/text-input.test.ts create mode 100644 src/platforms/android/adb-shell-protocol.ts create mode 100644 src/platforms/android/snapshot-helper-session-lifecycle.ts create mode 100644 src/platforms/android/text-input.ts diff --git a/src/core/interactors/android.test.ts b/src/core/interactors/android.test.ts index 70ff109863..ca79e86ae4 100644 --- a/src/core/interactors/android.test.ts +++ b/src/core/interactors/android.test.ts @@ -6,13 +6,13 @@ import { import type { DeviceInfo } from '@agent-device/kernel/device'; import { createAndroidInteractor } from './android.ts'; import { snapshotAndroid } from '../../platforms/android/snapshot.ts'; -import { fillAndroid, scrollAndroid } from '../../platforms/android/input-actions.ts'; +import { scrollAndroid } from '../../platforms/android/input-actions.ts'; +import { fillAndroid } from '../../platforms/android/text-input.ts'; vi.mock('../../platforms/android/snapshot.ts', () => ({ snapshotAndroid: vi.fn(), })); vi.mock('../../platforms/android/input-actions.ts', () => ({ - fillAndroid: vi.fn(), scrollAndroid: vi.fn(), appSwitcherAndroid: vi.fn(), backAndroid: vi.fn(), @@ -22,6 +22,9 @@ vi.mock('../../platforms/android/input-actions.ts', () => ({ pressAndroid: vi.fn(), pressAndroidTvRemote: vi.fn(), setAndroidOrientation: vi.fn(), +})); +vi.mock('../../platforms/android/text-input.ts', () => ({ + fillAndroid: vi.fn(), typeAndroid: vi.fn(), })); diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index 77f021d66f..08ed9cba03 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -6,7 +6,6 @@ import { import { appSwitcherAndroid, backAndroid, - fillAndroid, focusAndroid, homeAndroid, longPressAndroid, @@ -14,8 +13,8 @@ import { pressAndroidTvRemote, scrollAndroid, setAndroidOrientation, - typeAndroid, } from '../../platforms/android/input-actions.ts'; +import { fillAndroid, typeAndroid } from '../../platforms/android/text-input.ts'; import { executeAndroidTouchPlan, readAndroidGestureViewport, @@ -82,7 +81,8 @@ export function createAndroidInteractor( screenshot: (outPath, options) => screenshotAndroid(device, outPath, options), // uiautomator reads the node covering a point; `undefined` means nothing covers it. readTextAtPoint: async (point, options) => { - const { readAndroidTextAtPoint } = await import('../../platforms/android/input-actions.ts'); + const { readAndroidTextAtPoint } = + await import('../../platforms/android/fill-verification.ts'); const read = await readAndroidTextAtPoint(device, point.x, point.y, { helperSessionScope: androidHelperSessionScope(options?.appBundleId), }); diff --git a/src/platforms/android/__tests__/adb-shell-protocol.test.ts b/src/platforms/android/__tests__/adb-shell-protocol.test.ts new file mode 100644 index 0000000000..a910703637 --- /dev/null +++ b/src/platforms/android/__tests__/adb-shell-protocol.test.ts @@ -0,0 +1,101 @@ +import assert from 'node:assert/strict'; +import { beforeEach, test } from 'vitest'; +import { + androidAdbForwardsDeviceExitStatus, + resetAndroidAdbShellProtocolProbes, +} from '../adb-shell-protocol.ts'; +import type { AndroidAdbExecutor } from '../adb-executor.ts'; + +beforeEach(() => { + resetAndroidAdbShellProtocolProbes(); +}); + +test('reads the negotiated feature set as proof that adb forwards device exit status', async () => { + const calls: string[][] = []; + const adb = featuresAdb(calls, { + exitCode: 0, + stdout: 'sendrecv_v2\nstat_v2\nshell_v2\ncmd\n', + stderr: '', + }); + + assert.equal( + await androidAdbForwardsDeviceExitStatus({ adb, deviceKey: 'android:emulator-5554' }), + true, + ); + assert.deepEqual(calls, [['features']]); +}); + +test('treats a transport without shell protocol v2 as unable to prove a device exit', async () => { + const adb = featuresAdb([], { exitCode: 0, stdout: 'stat_v2\ncmd\n', stderr: '' }); + + assert.equal( + await androidAdbForwardsDeviceExitStatus({ adb, deviceKey: 'android:emulator-5554' }), + false, + ); +}); + +test('treats an unanswered probe as unknown rather than caching it as unsupported', async () => { + const calls: string[][] = []; + let answered = false; + const adb: AndroidAdbExecutor = async (args) => { + calls.push(args); + if (!answered) { + answered = true; + return { exitCode: 1, stdout: '', stderr: 'adb: unknown command features' }; + } + return { exitCode: 0, stdout: 'shell_v2\n', stderr: '' }; + }; + + assert.equal( + await androidAdbForwardsDeviceExitStatus({ adb, deviceKey: 'android:emulator-5554' }), + false, + ); + // The failed probe proved nothing, so it is not remembered as an answer. + assert.equal( + await androidAdbForwardsDeviceExitStatus({ adb, deviceKey: 'android:emulator-5554' }), + true, + ); + assert.equal(calls.length, 2); +}); + +test('reports no proof when the probe itself throws', async () => { + const adb: AndroidAdbExecutor = async () => { + throw new Error('device offline'); + }; + + assert.equal( + await androidAdbForwardsDeviceExitStatus({ adb, deviceKey: 'android:emulator-5554' }), + false, + ); +}); + +test('answers each device from its own negotiated feature set', async () => { + const adb: AndroidAdbExecutor = async () => ({ + exitCode: 0, + stdout: 'shell_v2\n', + stderr: '', + }); + const legacyAdb: AndroidAdbExecutor = async () => ({ exitCode: 0, stdout: 'cmd\n', stderr: '' }); + + assert.equal( + await androidAdbForwardsDeviceExitStatus({ adb, deviceKey: 'android:emulator-5554' }), + true, + ); + assert.equal( + await androidAdbForwardsDeviceExitStatus({ + adb: legacyAdb, + deviceKey: 'android:legacy-device', + }), + false, + ); +}); + +function featuresAdb( + calls: string[][], + result: { exitCode: number; stdout: string; stderr: string }, +): AndroidAdbExecutor { + return async (args) => { + calls.push(args); + return result; + }; +} diff --git a/src/platforms/android/__tests__/fill-verification.test.ts b/src/platforms/android/__tests__/fill-verification.test.ts index 003d1e3ff4..4dfb10a2a7 100644 --- a/src/platforms/android/__tests__/fill-verification.test.ts +++ b/src/platforms/android/__tests__/fill-verification.test.ts @@ -13,7 +13,7 @@ import { readAndroidFillTargetBeforeMutation, verifyAndroidFilledText, } from '../fill-verification.ts'; -import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session.ts'; +import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts'; import { createPersistentSnapshotHelperProvider, isAndroidHelperForwardRemoval, diff --git a/src/platforms/android/__tests__/input-actions.test.ts b/src/platforms/android/__tests__/input-actions.test.ts index 1349153b86..aa0cf0a28c 100644 --- a/src/platforms/android/__tests__/input-actions.test.ts +++ b/src/platforms/android/__tests__/input-actions.test.ts @@ -1,18 +1,7 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; -import { - fillAndroid, - longPressAndroid, - scrollAndroid, - setAndroidOrientation, - typeAndroid, -} from '../input-actions.ts'; -import { assertRejectsAppError } from '../../../__tests__/test-utils/app-error.ts'; +import { longPressAndroid, scrollAndroid, setAndroidOrientation } from '../input-actions.ts'; import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/device-fixtures.ts'; -import { - ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT, - androidSnapshotHelperScriptResponse, -} from '../../../__tests__/test-utils/android-snapshot-helper.ts'; import { withFakeAdb } from '../../../__tests__/test-utils/fake-adb.ts'; import { withAndroidAdbProvider, type AndroidTouchInjector } from '../adb-executor.ts'; @@ -145,244 +134,3 @@ test('setAndroidOrientation locks auto-rotate and sets user rotation', async () }, ); }); - -test('typeAndroid chunks ASCII input text for shell fallback', async () => { - await withFakeAdb( - () => undefined, - async ({ calls, device }) => { - await typeAndroid(device, 'filed the expense'); - assert.deepEqual(shellInputTextCalls(calls), [ - ['shell', 'input', 'text', 'filed%sth'], - ['shell', 'input', 'text', 'e%sexpens'], - ['shell', 'input', 'text', 'e'], - ]); - }, - ); -}); - -test('typeAndroid passes shell-sensitive ascii text to adb input text', async () => { - await withFakeAdb( - () => undefined, - async ({ calls, device }) => { - await typeAndroid(device, 'curtis.layne+test+73kmc@uber.com'); - assert.deepEqual(shellInputTextCalls(calls), [ - ['shell', 'input', 'text', 'curtis.l'], - ['shell', 'input', 'text', 'ayne+tes'], - ['shell', 'input', 'text', 't+73kmc@'], - ['shell', 'input', 'text', 'uber.com'], - ]); - }, - ); -}); - -test('typeAndroid preserves percent signs while encoding spaces', async () => { - await withFakeAdb( - () => undefined, - async ({ calls, device }) => { - await typeAndroid(device, '50% complete'); - assert.deepEqual(shellInputTextCalls(calls), [ - ['shell', 'input', 'text', '50%%scomp'], - ['shell', 'input', 'text', 'lete'], - ]); - }, - ); -}); - -test('typeAndroid sends one character at a time when delay is requested', async () => { - await withFakeAdb( - () => undefined, - async ({ calls, device }) => { - await typeAndroid(device, 'hey', 1); - assert.deepEqual(shellInputTextCalls(calls), [ - ['shell', 'input', 'text', 'h'], - ['shell', 'input', 'text', 'e'], - ['shell', 'input', 'text', 'y'], - ]); - }, - ); -}); - -test('typeAndroid shell-quotes text containing shell metacharacters', async () => { - await withFakeAdb( - () => undefined, - async ({ calls, device }) => { - await typeAndroid(device, 'otp; echo pwned'); - // The chunk carrying `;` is single-quoted so the device shell cannot - // re-tokenize it into a second command. - assert.deepEqual(shellInputTextCalls(calls), [ - ['shell', 'input', 'text', "'otp;%sech'"], - ['shell', 'input', 'text', 'o%spwned'], - ]); - }, - ); -}); - -test('typeAndroid leaves safe text unquoted', async () => { - await withFakeAdb( - () => undefined, - async ({ calls, device }) => { - await typeAndroid(device, 'hello'); - assert.deepEqual(shellInputTextCalls(calls), [['shell', 'input', 'text', 'hello']]); - }, - ); -}); - -test('fillAndroid uses chunk-safe shell input and retries when verification still fails', async () => { - // First `input text` writes a wrong partial value, so attempt 1 fails - // verification and production retries with the smaller chunk size. - let state = ''; - let inputTextCount = 0; - await withFakeAdb( - (args) => { - const helperResponse = snapshotHelperResponse(args, () => state); - if (helperResponse !== undefined) return helperResponse; - if (isShellInput(args, 'tap')) return undefined; - if (isShellKeyevent(args, 'KEYCODE_MOVE_END')) return undefined; - if (isShellKeyevent(args, 'KEYCODE_DEL')) { - state = ''; - return undefined; - } - if (isShellInput(args, 'text')) { - inputTextCount += 1; - state = inputTextCount === 1 ? 'curti' : state + (args[3] ?? ''); - return undefined; - } - return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; - }, - async ({ calls, device }) => { - await fillAndroid(device, 10, 10, 'curtis.layne+test+73kmc@uber.com'); - assert.equal( - calls.some((args) => args.join(' ').startsWith('shell cmd clipboard set text')), - false, - ); - assert.equal( - calls.some((args) => args.includes('KEYCODE_PASTE')), - false, - ); - assert.ok(shellInputTextCalls(calls).length > 1); - }, - { - provider: { snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT }, - }, - ); -}, 15_000); - -test('fillAndroid keeps delayed typing in typed-input mode', async () => { - let state = ''; - await withFakeAdb( - (args) => { - const helperResponse = snapshotHelperResponse(args, () => state); - if (helperResponse !== undefined) return helperResponse; - if (isShellInput(args, 'tap')) return undefined; - if (isShellKeyevent(args, 'KEYCODE_MOVE_END')) return undefined; - if (isShellKeyevent(args, 'KEYCODE_DEL')) { - state = ''; - return undefined; - } - if (isShellInput(args, 'text')) { - state += args[3] ?? ''; - return undefined; - } - return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; - }, - async ({ calls, device }) => { - await fillAndroid(device, 10, 10, 'go', 1); - assert.equal(shellInputTextCalls(calls).length, 2); - assert.equal( - calls.some((args) => args.join(' ').startsWith('shell cmd clipboard set text')), - false, - ); - assert.equal( - calls.some((args) => args.includes('KEYCODE_PASTE')), - false, - ); - }, - { - provider: { snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT }, - }, - ); -}, 15_000); - -test('fillAndroid tolerates delayed React Native text verification', async () => { - // The first hierarchy dump reports a stale truncated value (React Native - // committing late); the later stability dumps report the real text. - let state = ''; - let dumpCount = 0; - await withFakeAdb( - (args) => { - const helperResponse = snapshotHelperResponse(args, () => { - dumpCount += 1; - return dumpCount === 1 ? 'sent the updat' : state; - }); - if (helperResponse !== undefined) return helperResponse; - if (isShellInput(args, 'tap')) return undefined; - if (isShellKeyevent(args, 'KEYCODE_MOVE_END')) return undefined; - if (isShellKeyevent(args, 'KEYCODE_DEL')) { - state = ''; - return undefined; - } - if (isShellInput(args, 'text')) { - state += (args[3] ?? '').replace(/%s/g, ' '); - return undefined; - } - return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; - }, - async ({ device }) => { - await fillAndroid(device, 10, 10, 'sent the update'); - }, - { - provider: { snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT }, - }, - ); -}, 10_000); - -test('typeAndroid reports clear error when unicode input is unsupported', async () => { - await withFakeAdb( - (args) => { - if (args.join(' ').startsWith('shell cmd clipboard set text')) { - return 'No shell command implementation.'; - } - if (isShellInput(args, 'text')) { - return { - stderr: "Exception occurred while executing 'text':\njava.lang.NullPointerException\n", - exitCode: 255, - }; - } - return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; - }, - async ({ device }) => { - await assertRejectsAppError(() => typeAndroid(device, '很'), { - code: 'COMMAND_FAILED', - message: /provider-native text injection/i, - }); - }, - ); -}); - -function shellInputTextCalls(calls: string[][]): string[][] { - return calls.filter((args) => isShellInput(args, 'text')); -} - -function isShellInput(args: string[], subcommand: 'tap' | 'text'): boolean { - return args[0] === 'shell' && args[1] === 'input' && args[2] === subcommand; -} - -function isShellKeyevent(args: string[], keycode: string): boolean { - return ( - args[0] === 'shell' && args[1] === 'input' && args[2] === 'keyevent' && args[3] === keycode - ); -} - -/** - * Answers the snapshot-helper version probe and `am instrument` capture with a - * one-EditText hierarchy holding `resolveText()`, mirroring the PATH-stub - * helper script this file used before provider injection. Returns undefined for - * every other invocation so the caller's script keeps handling input actions. - */ -function snapshotHelperResponse(args: string[], resolveText: () => string): string | undefined { - return androidSnapshotHelperScriptResponse( - args, - () => - ``, - ); -} diff --git a/src/platforms/android/__tests__/snapshot-clickability.test.ts b/src/platforms/android/__tests__/snapshot-clickability.test.ts index 67a61ced1a..2a2075279e 100644 --- a/src/platforms/android/__tests__/snapshot-clickability.test.ts +++ b/src/platforms/android/__tests__/snapshot-clickability.test.ts @@ -11,7 +11,7 @@ import type { DeviceInfo } from '@agent-device/kernel/device'; import type { AndroidAdbExecutor } from '../snapshot-helper.ts'; import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from '../../../__tests__/test-utils/android-snapshot-helper.ts'; import { resetAndroidSnapshotHelperInstallCache } from '../snapshot-helper-install.ts'; -import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session.ts'; +import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts'; vi.mock('../../../utils/exec.ts', async (importOriginal) => { const actual = await importOriginal(); diff --git a/src/platforms/android/__tests__/snapshot-helper-runtime.test.ts b/src/platforms/android/__tests__/snapshot-helper-runtime.test.ts index 285f6043ba..0bf099f7cb 100644 --- a/src/platforms/android/__tests__/snapshot-helper-runtime.test.ts +++ b/src/platforms/android/__tests__/snapshot-helper-runtime.test.ts @@ -4,7 +4,7 @@ import type { AndroidAdbExecutor } from '../adb-executor.ts'; const { stopSession } = vi.hoisted(() => ({ stopSession: vi.fn() })); -vi.mock('../snapshot-helper-session.ts', () => ({ +vi.mock('../snapshot-helper-session-lifecycle.ts', () => ({ stopAndroidSnapshotHelperSession: stopSession, })); diff --git a/src/platforms/android/__tests__/snapshot-helper-session-lifecycle.test.ts b/src/platforms/android/__tests__/snapshot-helper-session-lifecycle.test.ts new file mode 100644 index 0000000000..46067dd462 --- /dev/null +++ b/src/platforms/android/__tests__/snapshot-helper-session-lifecycle.test.ts @@ -0,0 +1,317 @@ +import assert from 'node:assert/strict'; +import { afterEach, beforeEach, test } from 'vitest'; +import { captureAndroidSnapshotWithHelperSession } from '../snapshot-helper-session.ts'; +import { + resetAndroidSnapshotHelperSessions, + stopAndroidSnapshotHelperSession, +} from '../snapshot-helper-session-lifecycle.ts'; +import { recoverAndroidSnapshotHelperRetirement } from '../snapshot-helper-retirement.ts'; +import { + createSessionProvider, + FakeAndroidProcess, + type SessionProviderOptions, +} from './snapshot-helper-session.fixtures.ts'; +import type { AndroidAdbExecutor, AndroidAdbProvider } from '../adb-executor.ts'; + +beforeEach(async () => { + delete process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION; + await resetAndroidSnapshotHelperSessions(); +}); + +afterEach(async () => { + delete process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION; + await resetAndroidSnapshotHelperSessions(); +}); + +test('returns undefined when persistent sessions are disabled', async () => { + process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION = '0'; + const calls: string[][] = []; + const provider = createSessionProvider({ calls }); + + const output = await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + }); + + assert.equal(output, undefined); + assert.deepEqual(calls, []); +}); + +test('returns undefined when the adb provider cannot spawn a helper process', async () => { + const calls: string[][] = []; + const adb: AndroidAdbExecutor = async (args) => { + calls.push(args); + return { exitCode: 0, stdout: '', stderr: '' }; + }; + + const output = await captureAndroidSnapshotWithHelperSession({ adb }); + + assert.equal(output, undefined); + assert.deepEqual(calls, []); +}); + +test('disables repeated persistent session attempts after startup failure', async () => { + const calls: string[][] = []; + const spawnArgs: string[][] = []; + const provider: AndroidAdbProvider = { + exec: async (args) => { + calls.push(args); + return { exitCode: 0, stdout: '', stderr: '' }; + }, + spawn: (args) => { + spawnArgs.push(args); + const process = new FakeAndroidProcess(); + queueMicrotask(() => process.emitExit(0, null)); + return process; + }, + }; + + const first = await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + }); + const second = await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + }); + + assert.equal(first, undefined); + assert.equal(second, undefined); + assert.equal(spawnArgs.length, 1); + assert.equal(readSessionArgument(spawnArgs[0]!, 'timeoutMs'), '2000'); + assert.equal(calls.filter((args) => args[0] === 'forward').length, 2); +}); + +test('starts and reuses a persistent Android snapshot helper session', async () => { + const calls: string[][] = []; + const spawnArgs: string[][] = []; + const provider = createSessionProvider({ calls, spawnArgs }); + + const first = await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + helperVersion: '0.16.2', + helperVersionCode: 16002, + }); + const second = await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + helperVersion: '0.16.2', + helperVersionCode: 16002, + }); + + assert.match(first?.xml ?? '', /snapshot 1/); + assert.equal(first?.metadata.transport, 'persistent-session'); + assert.equal(first?.metadata.sessionReused, false); + assert.equal(first?.metadata.elapsedMs, 7); + assert.match(second?.xml ?? '', /snapshot 2/); + assert.equal(second?.metadata.transport, 'persistent-session'); + assert.equal(second?.metadata.sessionReused, true); + assert.equal(spawnArgs.length, 1); + assert.equal( + calls.filter((args) => args[0] === 'forward' && args[1]?.startsWith('tcp:')).length, + 1, + ); +}); + +test('restarts the helper session when capture options change', async () => { + const calls: string[][] = []; + const spawnArgs: string[][] = []; + const provider = createSessionProvider({ calls, spawnArgs }); + + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + waitForIdleTimeoutMs: 25, + }); + const restarted = await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + waitForIdleTimeoutMs: 50, + }); + + assert.equal(restarted?.metadata.sessionReused, false); + assert.equal(spawnArgs.length, 2); + assert.equal( + calls.some((args) => args[0] === 'forward' && args[1] === '--remove'), + true, + ); +}); + +test('a quit acknowledged and followed by process exit skips the force-stop round trip', async () => { + const calls: string[][] = []; + const processes: FakeAndroidProcess[] = []; + const provider = createSessionProvider({ calls, processes, quitExitDelayMs: 25 }); + + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + }); + await resetAndroidSnapshotHelperSessions(); + + assert.equal(processes.length, 1); + assert.equal(processes[0]?.killed, false); + // Acknowledged quit plus an observed exit IS the release evidence, so the extra adb round trip + // buys nothing. + assert.equal(calls.some(isHelperRuntimeForceStop), false); +}); + +test('a quit acknowledged by a helper the host then killed still force-stops the runtime', async () => { + const calls: string[][] = []; + const processes: FakeAndroidProcess[] = []; + const provider = createSessionProvider({ + calls, + processes, + quitExit: { code: null, signal: 'SIGKILL' }, + }); + + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + }); + await resetAndroidSnapshotHelperSessions(); + + // The ack alone only says the helper heard us. A host process that ended on a signal says the + // transport died, not that the instrumentation finished releasing UiAutomation — so the second + // half of the release evidence is missing and the device-side stop must still run. + assert.equal(calls.some(isHelperRuntimeForceStop), true); +}); + +test('a quit acknowledged after the host process already died still force-stops the runtime', async () => { + const calls: string[][] = []; + const processes: FakeAndroidProcess[] = []; + const provider = createSessionProvider({ calls, processes }); + + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + }); + // The host `am instrument` child is gone before the teardown starts, yet the device-side helper + // answers `quit` through the still-open forward: positive evidence that it OUTLIVED its host. + processes[0]?.emitExit(0, null); + await resetAndroidSnapshotHelperSessions(); + + assert.equal(calls.some(isHelperRuntimeForceStop), true); +}); + +test('force terminates the helper when quit is not acknowledged', async () => { + const calls: string[][] = []; + const processes: FakeAndroidProcess[] = []; + const provider = createSessionProvider({ calls, processes, quitResponseMode: 'malformed' }); + + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + }); + await resetAndroidSnapshotHelperSessions(); + + assert.equal(processes.length, 1); + assert.equal(processes[0]?.killed, true); + // Nothing proved the helper released UiAutomation, so the device-side stop must still run. + assert.equal(calls.some(isHelperRuntimeForceStop), true); +}); + +test('keeps the device-side stop when the transport cannot prove the device exit status', async () => { + const calls: string[][] = []; + const processes: FakeAndroidProcess[] = []; + const provider = createSessionProvider({ calls, processes, shellProtocolV2: false }); + + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + }); + await resetAndroidSnapshotHelperSessions(); + + // Same host-side evidence as the skip above — acknowledged quit, host child exited 0 — but this + // transport has no shell protocol v2, so adb reports 0 for instrumentation that never finished. + // That 0 is not release evidence, and the device-side stop stays. + assert.equal(calls.some(isHelperRuntimeForceStop), true); +}); + +test('keeps the device-side stop when the transport capability is unknown', async () => { + const calls: string[][] = []; + const processes: FakeAndroidProcess[] = []; + const provider = createSessionProvider({ calls, processes, featureProbeFailure: true }); + + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: 'android:emulator-5554', + }); + await resetAndroidSnapshotHelperSessions(); + + // An adb that cannot answer the probe never proved anything either. + assert.equal(calls.some(isHelperRuntimeForceStop), true); +}); + +test('probes the adb transport once per device instead of once per teardown', async () => { + const calls: string[][] = []; + const provider = createSessionProvider({ calls }); + const deviceKey = 'android:emulator-5554'; + + for (let teardown = 0; teardown < 2; teardown += 1) { + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey, + }); + await stopAndroidSnapshotHelperSession(deviceKey); + } + + assert.equal(calls.filter((args) => args[0] === 'features').length, 1); + assert.equal(calls.some(isHelperRuntimeForceStop), false); +}); + +test('failed whole-module reset preserves quarantine until recovery is confirmed', async () => { + const options: SessionProviderOptions = { + calls: [], + quitResponseMode: 'malformed', + recoveryFailure: true, + }; + const provider = createSessionProvider(options); + const deviceKey = 'android:emulator-5554'; + + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey, + }); + await assert.rejects( + resetAndroidSnapshotHelperSessions(), + /Failed to retire every Android snapshot helper session/, + ); + const forceStopsBeforeRecovery = options.calls.filter((args) => + args.join(' ').includes('am force-stop'), + ).length; + + options.recoveryFailure = false; + await recoverAndroidSnapshotHelperRetirement({ + deviceKey, + adb: provider.exec, + }); + + assert.equal( + options.calls.filter((args) => args.join(' ').includes('am force-stop')).length, + forceStopsBeforeRecovery + 1, + ); +}); + +function isHelperRuntimeForceStop(args: string[]): boolean { + return args.join(' ') === 'shell am force-stop com.callstack.agentdevice.snapshothelper'; +} + +function readSessionArgument(args: string[], name: string): string | undefined { + const index = args.indexOf(name); + return index < 0 ? undefined : args[index + 1]; +} diff --git a/src/platforms/android/__tests__/snapshot-helper-session.fixtures.ts b/src/platforms/android/__tests__/snapshot-helper-session.fixtures.ts index 513bd58414..c49d8ba52f 100644 --- a/src/platforms/android/__tests__/snapshot-helper-session.fixtures.ts +++ b/src/platforms/android/__tests__/snapshot-helper-session.fixtures.ts @@ -2,6 +2,10 @@ // plus a local TCP server standing in for the helper's session socket. Every test that observes // session lifetime (snapshot capture, fill verification samples, gesture/viewport transport) reads // the same fake, so "one warm session" means the same thing in all of them. +// +// `createSessionProvider` answers the session protocol alone and is where lifetime and teardown +// evidence are steered from; `createPersistentSnapshotHelperProvider` adds the install probe, +// viewport, and one-shot instrumentation the callers above it need. import assert from 'node:assert/strict'; import { EventEmitter } from 'node:events'; @@ -129,6 +133,201 @@ export function createPersistentSnapshotHelperProvider( }; } +export type SessionProviderOptions = { + calls: string[][]; + cleanupAborts?: string[][]; + processes?: FakeAndroidProcess[]; + quitExit?: { code: number | null; signal: NodeJS.Signals | null }; + quitExitDelayMs?: number; + quitResponseMode?: 'ok' | 'malformed'; + spawnArgs?: string[][]; + responseMode?: 'ok' | 'malformed' | 'ui-automation-timeout'; + responseDelayMs?: number; + forceStopDelayMs?: number; + recoveryFailure?: boolean; + stalledCleanup?: boolean; + stalledSnapshots?: number; + /** Whether `adb features` advertises the shell protocol that forwards device exit status. */ + shellProtocolV2?: boolean; + /** Make the `adb features` probe fail the way an adb too old to know the command does. */ + featureProbeFailure?: boolean; +}; + +export function createSessionProvider(options: SessionProviderOptions): AndroidAdbProvider { + let stalledSnapshots = options.stalledSnapshots ?? 0; + return { + exec: createSessionExec(options), + spawn: (args) => { + options.spawnArgs?.push(args); + const port = readSessionPort(args); + const process = new FakeAndroidProcess(); + options.processes?.push(process); + let snapshotCount = 0; + const sockets = new Set(); + const server = net.createServer((socket) => { + sockets.add(socket); + socket.once('close', () => sockets.delete(socket)); + socket.once('data', (chunk) => { + const command = chunk.toString('utf8').trim(); + const [, requestId = ''] = command.split(/\s+/, 2); + if (command.startsWith('quit')) { + if (options.quitResponseMode === 'malformed') { + socket.end('not a session response'); + return; + } + socket.end(sessionResponse({ requestId, body: '' })); + const quitExit = options.quitExit ?? { code: 0, signal: null }; + server.close(() => { + setTimeout( + () => process.emitExit(quitExit.code, quitExit.signal), + options.quitExitDelayMs ?? 0, + ); + }); + return; + } + if (options.responseMode === 'malformed') { + socket.end('not a session response'); + return; + } + if (options.responseMode === 'ui-automation-timeout') { + socket.end( + sessionResponse({ + requestId, + body: '', + metadata: { + ok: 'false', + errorType: 'java.util.concurrent.TimeoutException', + message: 'Timed out waiting for Android UiAutomation to connect', + }, + }), + ); + return; + } + snapshotCount += 1; + if (stalledSnapshots > 0) { + stalledSnapshots -= 1; + return; + } + const body = ``; + setTimeout(() => { + socket.end( + sessionResponse({ + requestId, + body, + metadata: { + waitForIdleTimeoutMs: '25', + waitForIdleQuietMs: '25', + timeoutMs: '5000', + maxDepth: '128', + maxNodes: '5000', + rootPresent: 'true', + captureMode: 'interactive-windows', + windowCount: '1', + nodeCount: '1', + truncated: 'false', + elapsedMs: '7', + }, + }), + ); + }, options.responseDelayMs ?? 0); + }); + }); + server.listen(port, '127.0.0.1', () => { + process.stdout.write( + [ + 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', + 'INSTRUMENTATION_STATUS: sessionReady=true', + 'INSTRUMENTATION_STATUS_CODE: 2', + '', + ].join('\n'), + ); + }); + process.onKill = () => { + for (const socket of sockets) socket.destroy(); + if (server.listening) { + server.close(() => process.emitExit(0, null)); + } else { + process.emitExit(0, null); + } + }; + return process; + }, + }; +} + +function createSessionExec(options: SessionProviderOptions): AndroidAdbExecutor { + return async (args, execOptions) => { + options.calls.push(args); + if (args[0] === 'features') return adbFeaturesResult(options); + const forceStopsRuntime = args.join(' ').includes('am force-stop'); + await stallSessionCleanupIfConfigured(options, args, execOptions?.signal, forceStopsRuntime); + if (options.recoveryFailure && forceStopsRuntime) { + return { exitCode: 1, stdout: '', stderr: 'runtime still busy' }; + } + await delayForceStopIfConfigured(options, execOptions?.signal, forceStopsRuntime); + return { exitCode: 0, stdout: '', stderr: '' }; + }; +} + +function adbFeaturesResult(options: SessionProviderOptions): { + exitCode: number; + stdout: string; + stderr: string; +} { + if (options.featureProbeFailure) { + return { exitCode: 1, stdout: '', stderr: 'adb: unknown command features' }; + } + const features = [ + 'cmd', + 'stat_v2', + 'abb', + ...(options.shellProtocolV2 === false ? [] : ['shell_v2']), + ]; + return { exitCode: 0, stdout: `${features.join('\n')}\n`, stderr: '' }; +} + +async function stallSessionCleanupIfConfigured( + options: SessionProviderOptions, + args: string[], + signal: AbortSignal | undefined, + forceStopsRuntime: boolean, +): Promise { + const removesForward = args[0] === 'forward' && args[1] === '--remove'; + if (!options.stalledCleanup || !signal || (!removesForward && !forceStopsRuntime)) return; + await new Promise((_resolve, reject) => { + const onAbort = () => { + options.cleanupAborts?.push(args); + reject(signal.reason); + }; + signal.addEventListener('abort', onAbort, { once: true }); + if (signal.aborted) onAbort(); + }); +} + +async function delayForceStopIfConfigured( + options: SessionProviderOptions, + signal: AbortSignal | undefined, + forceStopsRuntime: boolean, +): Promise { + if (!forceStopsRuntime || !options.forceStopDelayMs) return; + await waitForDelay(options.forceStopDelayMs, signal); +} + +async function waitForDelay(delayMs: number, signal: AbortSignal | undefined): Promise { + await new Promise((resolve, reject) => { + const onAbort = () => { + clearTimeout(timer); + reject(signal?.reason); + }; + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, delayMs); + signal?.addEventListener('abort', onAbort, { once: true }); + if (signal?.aborted) onAbort(); + }); +} + export function isAndroidHelperRuntimeForceStop(args: readonly string[]): boolean { return args[0] === 'shell' && args[1] === 'am' && args[2] === 'force-stop'; } @@ -199,6 +398,10 @@ function persistentSnapshotExecResult( if (args.includes('--show-versioncode')) { return Promise.resolve(ANDROID_HELPER_INSTALLED_VERSION_PROBE); } + // A current adb: it negotiated shell protocol v2, so a host exit status is the device's. + if (args[0] === 'features') { + return Promise.resolve({ exitCode: 0, stdout: 'cmd\nstat_v2\nshell_v2\n', stderr: '' }); + } if (args[0] === 'forward' || isAndroidHelperRuntimeForceStop(args)) { return Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }); } diff --git a/src/platforms/android/__tests__/snapshot-helper-session.test.ts b/src/platforms/android/__tests__/snapshot-helper-session.test.ts index 1388bbe61a..a5450a47c3 100644 --- a/src/platforms/android/__tests__/snapshot-helper-session.test.ts +++ b/src/platforms/android/__tests__/snapshot-helper-session.test.ts @@ -1,121 +1,21 @@ import assert from 'node:assert/strict'; -import { EventEmitter } from 'node:events'; -import net from 'node:net'; -import { PassThrough } from 'node:stream'; import { afterEach, beforeEach, test } from 'vitest'; -import { - captureAndroidSnapshotWithHelperSession, - resetAndroidSnapshotHelperSessions, -} from '../snapshot-helper-session.ts'; +import { captureAndroidSnapshotWithHelperSession } from '../snapshot-helper-session.ts'; +import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts'; import { resolveAndroidSnapshotHelperSessionRequestTimeoutMs } from '../snapshot-helper-session-protocol.ts'; -import { recoverAndroidSnapshotHelperRetirement } from '../snapshot-helper-retirement.ts'; -import type { AndroidAdbExecutor, AndroidAdbProcess, AndroidAdbProvider } from '../adb-executor.ts'; +import { + createSessionProvider, + type FakeAndroidProcess, +} from './snapshot-helper-session.fixtures.ts'; beforeEach(async () => { - delete process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION; await resetAndroidSnapshotHelperSessions(); }); afterEach(async () => { - delete process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION; await resetAndroidSnapshotHelperSessions(); }); -test('returns undefined when persistent sessions are disabled', async () => { - process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION = '0'; - const calls: string[][] = []; - const provider = createSessionProvider({ calls }); - - const output = await captureAndroidSnapshotWithHelperSession({ - adb: provider.exec, - adbProvider: provider, - }); - - assert.equal(output, undefined); - assert.deepEqual(calls, []); -}); - -test('returns undefined when the adb provider cannot spawn a helper process', async () => { - const calls: string[][] = []; - const adb: AndroidAdbExecutor = async (args) => { - calls.push(args); - return { exitCode: 0, stdout: '', stderr: '' }; - }; - - const output = await captureAndroidSnapshotWithHelperSession({ adb }); - - assert.equal(output, undefined); - assert.deepEqual(calls, []); -}); - -test('disables repeated persistent session attempts after startup failure', async () => { - const calls: string[][] = []; - const spawnArgs: string[][] = []; - const provider: AndroidAdbProvider = { - exec: async (args) => { - calls.push(args); - return { exitCode: 0, stdout: '', stderr: '' }; - }, - spawn: (args) => { - spawnArgs.push(args); - const process = new FakeAndroidProcess(); - queueMicrotask(() => process.emitExit(0, null)); - return process; - }, - }; - - const first = await captureAndroidSnapshotWithHelperSession({ - adb: provider.exec, - adbProvider: provider, - deviceKey: 'android:emulator-5554', - }); - const second = await captureAndroidSnapshotWithHelperSession({ - adb: provider.exec, - adbProvider: provider, - deviceKey: 'android:emulator-5554', - }); - - assert.equal(first, undefined); - assert.equal(second, undefined); - assert.equal(spawnArgs.length, 1); - assert.equal(readSessionArgument(spawnArgs[0]!, 'timeoutMs'), '2000'); - assert.equal(calls.filter((args) => args[0] === 'forward').length, 2); -}); - -test('starts and reuses a persistent Android snapshot helper session', async () => { - const calls: string[][] = []; - const spawnArgs: string[][] = []; - const provider = createSessionProvider({ calls, spawnArgs }); - - const first = await captureAndroidSnapshotWithHelperSession({ - adb: provider.exec, - adbProvider: provider, - deviceKey: 'android:emulator-5554', - helperVersion: '0.16.2', - helperVersionCode: 16002, - }); - const second = await captureAndroidSnapshotWithHelperSession({ - adb: provider.exec, - adbProvider: provider, - deviceKey: 'android:emulator-5554', - helperVersion: '0.16.2', - helperVersionCode: 16002, - }); - - assert.match(first?.xml ?? '', /snapshot 1/); - assert.equal(first?.metadata.transport, 'persistent-session'); - assert.equal(first?.metadata.sessionReused, false); - assert.equal(first?.metadata.elapsedMs, 7); - assert.match(second?.xml ?? '', /snapshot 2/); - assert.equal(second?.metadata.transport, 'persistent-session'); - assert.equal(second?.metadata.sessionReused, true); - assert.equal(spawnArgs.length, 1); - assert.equal( - calls.filter((args) => args[0] === 'forward' && args[1]?.startsWith('tcp:')).length, - 1, - ); -}); - test('allows a persistent session snapshot to use the helper command budget', async () => { const calls: string[][] = []; const provider = createSessionProvider({ calls, responseDelayMs: 25 }); @@ -255,147 +155,6 @@ test('failed capture does not fall back when device runtime retirement is unconf assert.equal(processes.length, 1); }); -test('restarts the helper session when capture options change', async () => { - const calls: string[][] = []; - const spawnArgs: string[][] = []; - const provider = createSessionProvider({ calls, spawnArgs }); - - await captureAndroidSnapshotWithHelperSession({ - adb: provider.exec, - adbProvider: provider, - deviceKey: 'android:emulator-5554', - waitForIdleTimeoutMs: 25, - }); - const restarted = await captureAndroidSnapshotWithHelperSession({ - adb: provider.exec, - adbProvider: provider, - deviceKey: 'android:emulator-5554', - waitForIdleTimeoutMs: 50, - }); - - assert.equal(restarted?.metadata.sessionReused, false); - assert.equal(spawnArgs.length, 2); - assert.equal( - calls.some((args) => args[0] === 'forward' && args[1] === '--remove'), - true, - ); -}); - -test('a quit acknowledged and followed by process exit skips the force-stop round trip', async () => { - const calls: string[][] = []; - const processes: FakeAndroidProcess[] = []; - const provider = createSessionProvider({ calls, processes, quitExitDelayMs: 25 }); - - await captureAndroidSnapshotWithHelperSession({ - adb: provider.exec, - adbProvider: provider, - deviceKey: 'android:emulator-5554', - }); - await resetAndroidSnapshotHelperSessions(); - - assert.equal(processes.length, 1); - assert.equal(processes[0]?.killed, false); - // Acknowledged quit plus an observed exit IS the release evidence, so the extra adb round trip - // buys nothing. - assert.equal(calls.some(isHelperRuntimeForceStop), false); -}); - -test('a quit acknowledged by a helper the host then killed still force-stops the runtime', async () => { - const calls: string[][] = []; - const processes: FakeAndroidProcess[] = []; - const provider = createSessionProvider({ - calls, - processes, - quitExit: { code: null, signal: 'SIGKILL' }, - }); - - await captureAndroidSnapshotWithHelperSession({ - adb: provider.exec, - adbProvider: provider, - deviceKey: 'android:emulator-5554', - }); - await resetAndroidSnapshotHelperSessions(); - - // The ack alone only says the helper heard us. A host process that ended on a signal says the - // transport died, not that the instrumentation finished releasing UiAutomation — so the second - // half of the release evidence is missing and the device-side stop must still run. - assert.equal(calls.some(isHelperRuntimeForceStop), true); -}); - -test('a quit acknowledged after the host process already died still force-stops the runtime', async () => { - const calls: string[][] = []; - const processes: FakeAndroidProcess[] = []; - const provider = createSessionProvider({ calls, processes }); - - await captureAndroidSnapshotWithHelperSession({ - adb: provider.exec, - adbProvider: provider, - deviceKey: 'android:emulator-5554', - }); - // The host `am instrument` child is gone before the teardown starts, yet the device-side helper - // answers `quit` through the still-open forward: positive evidence that it OUTLIVED its host. - processes[0]?.emitExit(0, null); - await resetAndroidSnapshotHelperSessions(); - - assert.equal(calls.some(isHelperRuntimeForceStop), true); -}); - -test('force terminates the helper when quit is not acknowledged', async () => { - const calls: string[][] = []; - const processes: FakeAndroidProcess[] = []; - const provider = createSessionProvider({ calls, processes, quitResponseMode: 'malformed' }); - - await captureAndroidSnapshotWithHelperSession({ - adb: provider.exec, - adbProvider: provider, - deviceKey: 'android:emulator-5554', - }); - await resetAndroidSnapshotHelperSessions(); - - assert.equal(processes.length, 1); - assert.equal(processes[0]?.killed, true); - // Nothing proved the helper released UiAutomation, so the device-side stop must still run. - assert.equal(calls.some(isHelperRuntimeForceStop), true); -}); - -function isHelperRuntimeForceStop(args: string[]): boolean { - return args.join(' ') === 'shell am force-stop com.callstack.agentdevice.snapshothelper'; -} - -test('failed whole-module reset preserves quarantine until recovery is confirmed', async () => { - const options: SessionProviderOptions = { - calls: [], - quitResponseMode: 'malformed', - recoveryFailure: true, - }; - const provider = createSessionProvider(options); - const deviceKey = 'android:emulator-5554'; - - await captureAndroidSnapshotWithHelperSession({ - adb: provider.exec, - adbProvider: provider, - deviceKey, - }); - await assert.rejects( - resetAndroidSnapshotHelperSessions(), - /Failed to retire every Android snapshot helper session/, - ); - const forceStopsBeforeRecovery = options.calls.filter((args) => - args.join(' ').includes('am force-stop'), - ).length; - - options.recoveryFailure = false; - await recoverAndroidSnapshotHelperRetirement({ - deviceKey, - adb: provider.exec, - }); - - assert.equal( - options.calls.filter((args) => args.join(' ').includes('am force-stop')).length, - forceStopsBeforeRecovery + 1, - ); -}); - test('allows device retirement beyond host-process grace before falling back', async () => { const calls: string[][] = []; const processes: FakeAndroidProcess[] = []; @@ -432,230 +191,3 @@ test('invalidates and falls back from the helper session after a malformed respo true, ); }); - -type SessionProviderOptions = { - calls: string[][]; - cleanupAborts?: string[][]; - processes?: FakeAndroidProcess[]; - quitExit?: { code: number | null; signal: NodeJS.Signals | null }; - quitExitDelayMs?: number; - quitResponseMode?: 'ok' | 'malformed'; - spawnArgs?: string[][]; - responseMode?: 'ok' | 'malformed' | 'ui-automation-timeout'; - responseDelayMs?: number; - forceStopDelayMs?: number; - recoveryFailure?: boolean; - stalledCleanup?: boolean; - stalledSnapshots?: number; -}; - -function createSessionProvider(options: SessionProviderOptions): AndroidAdbProvider { - let stalledSnapshots = options.stalledSnapshots ?? 0; - return { - exec: createSessionExec(options), - spawn: (args) => { - options.spawnArgs?.push(args); - const port = readSessionPort(args); - const process = new FakeAndroidProcess(); - options.processes?.push(process); - let snapshotCount = 0; - const sockets = new Set(); - const server = net.createServer((socket) => { - sockets.add(socket); - socket.once('close', () => sockets.delete(socket)); - socket.once('data', (chunk) => { - const command = chunk.toString('utf8').trim(); - const [, requestId = ''] = command.split(/\s+/, 2); - if (command.startsWith('quit')) { - if (options.quitResponseMode === 'malformed') { - socket.end('not a session response'); - return; - } - socket.end(sessionResponse({ requestId, body: '' })); - const quitExit = options.quitExit ?? { code: 0, signal: null }; - server.close(() => { - setTimeout( - () => process.emitExit(quitExit.code, quitExit.signal), - options.quitExitDelayMs ?? 0, - ); - }); - return; - } - if (options.responseMode === 'malformed') { - socket.end('not a session response'); - return; - } - if (options.responseMode === 'ui-automation-timeout') { - socket.end( - sessionResponse({ - requestId, - body: '', - metadata: { - ok: 'false', - errorType: 'java.util.concurrent.TimeoutException', - message: 'Timed out waiting for Android UiAutomation to connect', - }, - }), - ); - return; - } - snapshotCount += 1; - if (stalledSnapshots > 0) { - stalledSnapshots -= 1; - return; - } - const body = ``; - setTimeout(() => { - socket.end( - sessionResponse({ - requestId, - body, - metadata: { - waitForIdleTimeoutMs: '25', - waitForIdleQuietMs: '25', - timeoutMs: '5000', - maxDepth: '128', - maxNodes: '5000', - rootPresent: 'true', - captureMode: 'interactive-windows', - windowCount: '1', - nodeCount: '1', - truncated: 'false', - elapsedMs: '7', - }, - }), - ); - }, options.responseDelayMs ?? 0); - }); - }); - server.listen(port, '127.0.0.1', () => { - process.stdout.write( - [ - 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', - 'INSTRUMENTATION_STATUS: sessionReady=true', - 'INSTRUMENTATION_STATUS_CODE: 2', - '', - ].join('\n'), - ); - }); - process.onKill = () => { - for (const socket of sockets) socket.destroy(); - if (server.listening) { - server.close(() => process.emitExit(0, null)); - } else { - process.emitExit(0, null); - } - }; - return process; - }, - }; -} - -function createSessionExec(options: SessionProviderOptions): AndroidAdbExecutor { - return async (args, execOptions) => { - options.calls.push(args); - const forceStopsRuntime = args.join(' ').includes('am force-stop'); - await stallSessionCleanupIfConfigured(options, args, execOptions?.signal, forceStopsRuntime); - if (options.recoveryFailure && forceStopsRuntime) { - return { exitCode: 1, stdout: '', stderr: 'runtime still busy' }; - } - await delayForceStopIfConfigured(options, execOptions?.signal, forceStopsRuntime); - return { exitCode: 0, stdout: '', stderr: '' }; - }; -} - -async function stallSessionCleanupIfConfigured( - options: SessionProviderOptions, - args: string[], - signal: AbortSignal | undefined, - forceStopsRuntime: boolean, -): Promise { - const removesForward = args[0] === 'forward' && args[1] === '--remove'; - if (!options.stalledCleanup || !signal || (!removesForward && !forceStopsRuntime)) return; - await new Promise((_resolve, reject) => { - const onAbort = () => { - options.cleanupAborts?.push(args); - reject(signal.reason); - }; - signal.addEventListener('abort', onAbort, { once: true }); - if (signal.aborted) onAbort(); - }); -} - -async function delayForceStopIfConfigured( - options: SessionProviderOptions, - signal: AbortSignal | undefined, - forceStopsRuntime: boolean, -): Promise { - if (!forceStopsRuntime || !options.forceStopDelayMs) return; - await waitForDelay(options.forceStopDelayMs, signal); -} - -async function waitForDelay(delayMs: number, signal: AbortSignal | undefined): Promise { - await new Promise((resolve, reject) => { - const onAbort = () => { - clearTimeout(timer); - reject(signal?.reason); - }; - const timer = setTimeout(() => { - signal?.removeEventListener('abort', onAbort); - resolve(); - }, delayMs); - signal?.addEventListener('abort', onAbort, { once: true }); - if (signal?.aborted) onAbort(); - }); -} - -function sessionResponse(params: { - requestId: string; - body: string; - metadata?: Record; -}): string { - const bodyLength = Buffer.byteLength(params.body, 'utf8'); - const headers = { - agentDeviceProtocol: 'android-snapshot-helper-v1', - helperApiVersion: '1', - outputFormat: 'uiautomator-xml', - requestId: params.requestId, - ok: 'true', - byteLength: String(bodyLength), - ...params.metadata, - }; - return `${Object.entries(headers) - .map(([key, value]) => `${key}=${value}`) - .join('\n')}\n\n${params.body}`; -} - -function readSessionPort(args: string[]): number { - const index = args.indexOf('sessionPort'); - assert.notEqual(index, -1); - return Number(args[index + 1]); -} - -function readSessionArgument(args: string[], name: string): string | undefined { - const index = args.indexOf(name); - return index < 0 ? undefined : args[index + 1]; -} - -class FakeAndroidProcess extends EventEmitter implements AndroidAdbProcess { - stdin = new PassThrough(); - stdout = new PassThrough(); - stderr = new PassThrough(); - exitCode: number | null = null; - signalCode: NodeJS.Signals | null = null; - killed = false; - onKill: (() => void) | undefined; - - kill(): boolean { - this.killed = true; - this.onKill?.(); - return true; - } - - emitExit(code: number | null, signal: NodeJS.Signals | null): void { - this.exitCode = code; - this.signalCode = signal; - this.emit('exit', code, signal); - this.emit('close', code, signal); - } -} diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index ab9cf07d9d..4004e88df7 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -24,7 +24,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { runCmd } from '../../../utils/exec.ts'; import { sleep } from '../adb.ts'; import { resetAndroidSnapshotHelperInstallCache } from '../snapshot-helper-install.ts'; -import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session.ts'; +import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts'; import { type AndroidAdbExecutor } from '../snapshot-helper.ts'; import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from '../../../__tests__/test-utils/android-snapshot-helper.ts'; import { diff --git a/src/platforms/android/__tests__/input-actions-fill.test.ts b/src/platforms/android/__tests__/text-input-fill.test.ts similarity index 99% rename from src/platforms/android/__tests__/input-actions-fill.test.ts rename to src/platforms/android/__tests__/text-input-fill.test.ts index 34b2979627..2702ceb4bd 100644 --- a/src/platforms/android/__tests__/input-actions-fill.test.ts +++ b/src/platforms/android/__tests__/text-input-fill.test.ts @@ -6,7 +6,7 @@ import { createAndroidSnapshotHelperExecutor, } from '../../../__tests__/test-utils/android-snapshot-helper.ts'; import { AppError } from '@agent-device/kernel/errors'; -import { fillAndroid, typeAndroid } from '../input-actions.ts'; +import { fillAndroid, typeAndroid } from '../text-input.ts'; import { withAndroidAdbProvider, type AndroidAdbExecutor } from '../adb-executor.ts'; import { androidFillFailureDetails, @@ -14,7 +14,7 @@ import { readAndroidTextAtPointInHierarchy, verifyAndroidFilledTextInHierarchy, } from '../fill-verification.ts'; -import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session.ts'; +import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts'; import { createPersistentSnapshotHelperProvider, isAndroidHelperForwardRemoval, diff --git a/src/platforms/android/__tests__/input-actions-test-ime.test.ts b/src/platforms/android/__tests__/text-input-test-ime.test.ts similarity index 98% rename from src/platforms/android/__tests__/input-actions-test-ime.test.ts rename to src/platforms/android/__tests__/text-input-test-ime.test.ts index 477d9a0a17..1e8aff53d0 100644 --- a/src/platforms/android/__tests__/input-actions-test-ime.test.ts +++ b/src/platforms/android/__tests__/text-input-test-ime.test.ts @@ -34,7 +34,7 @@ import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT, createAndroidSnapshotHelperExecutor, } from '../../../__tests__/test-utils/android-snapshot-helper.ts'; -import { fillAndroid, typeAndroid } from '../input-actions.ts'; +import { fillAndroid, typeAndroid } from '../text-input.ts'; import { withAndroidAdbProvider, type AndroidAdbExecutor } from '../adb-executor.ts'; import { resetAndroidTestImeActivationCacheForTests, diff --git a/src/platforms/android/__tests__/text-input.test.ts b/src/platforms/android/__tests__/text-input.test.ts new file mode 100644 index 0000000000..f0525f84cd --- /dev/null +++ b/src/platforms/android/__tests__/text-input.test.ts @@ -0,0 +1,424 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { fillAndroid, typeAndroid } from '../text-input.ts'; +import { assertRejectsAppError } from '../../../__tests__/test-utils/app-error.ts'; +import { + ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT, + androidSnapshotHelperScriptResponse, +} from '../../../__tests__/test-utils/android-snapshot-helper.ts'; +import { withFakeAdb } from '../../../__tests__/test-utils/fake-adb.ts'; + +// The fake adb provider installs through the production withAndroidAdbProvider +// scope, so `calls` records device-scoped args without a leading `-s `. + +function isShellInput(args: string[], subcommand: 'tap' | 'text'): boolean { + return args[0] === 'shell' && args[1] === 'input' && args[2] === subcommand; +} + +function isShellKeyevent(args: string[], keycode: string): boolean { + return ( + args[0] === 'shell' && args[1] === 'input' && args[2] === 'keyevent' && args[3] === keycode + ); +} + +test('typeAndroid chunks ASCII input text for shell fallback', async () => { + await withFakeAdb( + () => undefined, + async ({ calls, device }) => { + await typeAndroid(device, 'filed the expense'); + assert.deepEqual(shellInputTextCalls(calls), [ + ['shell', 'input', 'text', 'filed%sth'], + ['shell', 'input', 'text', 'e%sexpens'], + ['shell', 'input', 'text', 'e'], + ]); + }, + ); +}); + +test('typeAndroid passes shell-sensitive ascii text to adb input text', async () => { + await withFakeAdb( + () => undefined, + async ({ calls, device }) => { + await typeAndroid(device, 'curtis.layne+test+73kmc@uber.com'); + assert.deepEqual(shellInputTextCalls(calls), [ + ['shell', 'input', 'text', 'curtis.l'], + ['shell', 'input', 'text', 'ayne+tes'], + ['shell', 'input', 'text', 't+73kmc@'], + ['shell', 'input', 'text', 'uber.com'], + ]); + }, + ); +}); + +test('typeAndroid preserves percent signs while encoding spaces', async () => { + await withFakeAdb( + () => undefined, + async ({ calls, device }) => { + await typeAndroid(device, '50% complete'); + assert.deepEqual(shellInputTextCalls(calls), [ + ['shell', 'input', 'text', '50%%scomp'], + ['shell', 'input', 'text', 'lete'], + ]); + }, + ); +}); + +test('typeAndroid sends one character at a time when delay is requested', async () => { + await withFakeAdb( + () => undefined, + async ({ calls, device }) => { + await typeAndroid(device, 'hey', 1); + assert.deepEqual(shellInputTextCalls(calls), [ + ['shell', 'input', 'text', 'h'], + ['shell', 'input', 'text', 'e'], + ['shell', 'input', 'text', 'y'], + ]); + }, + ); +}); + +test('typeAndroid shell-quotes text containing shell metacharacters', async () => { + await withFakeAdb( + () => undefined, + async ({ calls, device }) => { + await typeAndroid(device, 'otp; echo pwned'); + // The chunk carrying `;` is single-quoted so the device shell cannot + // re-tokenize it into a second command. + assert.deepEqual(shellInputTextCalls(calls), [ + ['shell', 'input', 'text', "'otp;%sech'"], + ['shell', 'input', 'text', 'o%spwned'], + ]); + }, + ); +}); + +test('typeAndroid leaves safe text unquoted', async () => { + await withFakeAdb( + () => undefined, + async ({ calls, device }) => { + await typeAndroid(device, 'hello'); + assert.deepEqual(shellInputTextCalls(calls), [['shell', 'input', 'text', 'hello']]); + }, + ); +}); + +test('fillAndroid uses chunk-safe shell input and retries when verification still fails', async () => { + // First `input text` writes a wrong partial value, so attempt 1 fails + // verification and production retries with the smaller chunk size. + let state = ''; + let inputTextCount = 0; + await withFakeAdb( + (args) => { + const helperResponse = snapshotHelperResponse(args, () => state); + if (helperResponse !== undefined) return helperResponse; + if (isShellInput(args, 'tap')) return undefined; + if (isShellKeyevent(args, 'KEYCODE_MOVE_END')) return undefined; + if (isShellKeyevent(args, 'KEYCODE_DEL')) { + state = ''; + return undefined; + } + if (isShellInput(args, 'text')) { + inputTextCount += 1; + state = inputTextCount === 1 ? 'curti' : state + (args[3] ?? ''); + return undefined; + } + return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; + }, + async ({ calls, device }) => { + await fillAndroid(device, 10, 10, 'curtis.layne+test+73kmc@uber.com'); + assert.equal( + calls.some((args) => args.join(' ').startsWith('shell cmd clipboard set text')), + false, + ); + assert.equal( + calls.some((args) => args.includes('KEYCODE_PASTE')), + false, + ); + assert.ok(shellInputTextCalls(calls).length > 1); + }, + { + provider: { snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT }, + }, + ); +}, 15_000); + +test('fillAndroid keeps delayed typing in typed-input mode', async () => { + let state = ''; + await withFakeAdb( + (args) => { + const helperResponse = snapshotHelperResponse(args, () => state); + if (helperResponse !== undefined) return helperResponse; + if (isShellInput(args, 'tap')) return undefined; + if (isShellKeyevent(args, 'KEYCODE_MOVE_END')) return undefined; + if (isShellKeyevent(args, 'KEYCODE_DEL')) { + state = ''; + return undefined; + } + if (isShellInput(args, 'text')) { + state += args[3] ?? ''; + return undefined; + } + return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; + }, + async ({ calls, device }) => { + await fillAndroid(device, 10, 10, 'go', 1); + assert.equal(shellInputTextCalls(calls).length, 2); + assert.equal( + calls.some((args) => args.join(' ').startsWith('shell cmd clipboard set text')), + false, + ); + assert.equal( + calls.some((args) => args.includes('KEYCODE_PASTE')), + false, + ); + }, + { + provider: { snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT }, + }, + ); +}, 15_000); + +test('fillAndroid tolerates delayed React Native text verification', async () => { + // The first hierarchy dump reports a stale truncated value (React Native + // committing late); the later stability dumps report the real text. + let state = ''; + let dumpCount = 0; + await withFakeAdb( + (args) => { + const helperResponse = snapshotHelperResponse(args, () => { + dumpCount += 1; + return dumpCount === 1 ? 'sent the updat' : state; + }); + if (helperResponse !== undefined) return helperResponse; + if (isShellInput(args, 'tap')) return undefined; + if (isShellKeyevent(args, 'KEYCODE_MOVE_END')) return undefined; + if (isShellKeyevent(args, 'KEYCODE_DEL')) { + state = ''; + return undefined; + } + if (isShellInput(args, 'text')) { + state += (args[3] ?? '').replace(/%s/g, ' '); + return undefined; + } + return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; + }, + async ({ device }) => { + await fillAndroid(device, 10, 10, 'sent the update'); + }, + { + provider: { snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT }, + }, + ); +}, 10_000); + +test('typeAndroid reports clear error when unicode input is unsupported', async () => { + await withFakeAdb( + (args) => { + if (args.join(' ').startsWith('shell cmd clipboard set text')) { + return 'No shell command implementation.'; + } + if (isShellInput(args, 'text')) { + return { + stderr: "Exception occurred while executing 'text':\njava.lang.NullPointerException\n", + exitCode: 255, + }; + } + return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; + }, + async ({ device }) => { + await assertRejectsAppError(() => typeAndroid(device, '很'), { + code: 'COMMAND_FAILED', + message: /provider-native text injection/i, + }); + }, + ); +}); + +test('fillAndroid keeps delayed typing in typed-input mode', async () => { + let state = ''; + await withFakeAdb( + (args) => { + const helperResponse = snapshotHelperResponse(args, () => state); + if (helperResponse !== undefined) return helperResponse; + if (isShellInput(args, 'tap')) return undefined; + if (isShellKeyevent(args, 'KEYCODE_MOVE_END')) return undefined; + if (isShellKeyevent(args, 'KEYCODE_DEL')) { + state = ''; + return undefined; + } + if (isShellInput(args, 'text')) { + state += args[3] ?? ''; + return undefined; + } + return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; + }, + async ({ calls, device }) => { + await fillAndroid(device, 10, 10, 'go', 1); + assert.equal(shellInputTextCalls(calls).length, 2); + assert.equal( + calls.some((args) => args.join(' ').startsWith('shell cmd clipboard set text')), + false, + ); + assert.equal( + calls.some((args) => args.includes('KEYCODE_PASTE')), + false, + ); + }, + { + provider: { snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT }, + }, + ); +}, 15_000); + +test('fillAndroid tolerates delayed React Native text verification', async () => { + // The first hierarchy dump reports a stale truncated value (React Native + // committing late); the later stability dumps report the real text. + let state = ''; + let dumpCount = 0; + await withFakeAdb( + (args) => { + const helperResponse = snapshotHelperResponse(args, () => { + dumpCount += 1; + return dumpCount === 1 ? 'sent the updat' : state; + }); + if (helperResponse !== undefined) return helperResponse; + if (isShellInput(args, 'tap')) return undefined; + if (isShellKeyevent(args, 'KEYCODE_MOVE_END')) return undefined; + if (isShellKeyevent(args, 'KEYCODE_DEL')) { + state = ''; + return undefined; + } + if (isShellInput(args, 'text')) { + state += (args[3] ?? '').replace(/%s/g, ' '); + return undefined; + } + return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; + }, + async ({ device }) => { + await fillAndroid(device, 10, 10, 'sent the update'); + }, + { + provider: { snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT }, + }, + ); +}, 10_000); + +test('typeAndroid reports clear error when unicode input is unsupported', async () => { + await withFakeAdb( + (args) => { + if (args.join(' ').startsWith('shell cmd clipboard set text')) { + return 'No shell command implementation.'; + } + if (isShellInput(args, 'text')) { + return { + stderr: "Exception occurred while executing 'text':\njava.lang.NullPointerException\n", + exitCode: 255, + }; + } + return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; + }, + async ({ device }) => { + await assertRejectsAppError(() => typeAndroid(device, '很'), { + code: 'COMMAND_FAILED', + message: /provider-native text injection/i, + }); + }, + ); +}); + +test('fillAndroid tolerates delayed React Native text verification', async () => { + // The first hierarchy dump reports a stale truncated value (React Native + // committing late); the later stability dumps report the real text. + let state = ''; + let dumpCount = 0; + await withFakeAdb( + (args) => { + const helperResponse = snapshotHelperResponse(args, () => { + dumpCount += 1; + return dumpCount === 1 ? 'sent the updat' : state; + }); + if (helperResponse !== undefined) return helperResponse; + if (isShellInput(args, 'tap')) return undefined; + if (isShellKeyevent(args, 'KEYCODE_MOVE_END')) return undefined; + if (isShellKeyevent(args, 'KEYCODE_DEL')) { + state = ''; + return undefined; + } + if (isShellInput(args, 'text')) { + state += (args[3] ?? '').replace(/%s/g, ' '); + return undefined; + } + return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; + }, + async ({ device }) => { + await fillAndroid(device, 10, 10, 'sent the update'); + }, + { + provider: { snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT }, + }, + ); +}, 10_000); + +test('typeAndroid reports clear error when unicode input is unsupported', async () => { + await withFakeAdb( + (args) => { + if (args.join(' ').startsWith('shell cmd clipboard set text')) { + return 'No shell command implementation.'; + } + if (isShellInput(args, 'text')) { + return { + stderr: "Exception occurred while executing 'text':\njava.lang.NullPointerException\n", + exitCode: 255, + }; + } + return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; + }, + async ({ device }) => { + await assertRejectsAppError(() => typeAndroid(device, '很'), { + code: 'COMMAND_FAILED', + message: /provider-native text injection/i, + }); + }, + ); +}); + +test('typeAndroid reports clear error when unicode input is unsupported', async () => { + await withFakeAdb( + (args) => { + if (args.join(' ').startsWith('shell cmd clipboard set text')) { + return 'No shell command implementation.'; + } + if (isShellInput(args, 'text')) { + return { + stderr: "Exception occurred while executing 'text':\njava.lang.NullPointerException\n", + exitCode: 255, + }; + } + return { stderr: `unexpected args: ${args.join(' ')}`, exitCode: 1 }; + }, + async ({ device }) => { + await assertRejectsAppError(() => typeAndroid(device, '很'), { + code: 'COMMAND_FAILED', + message: /provider-native text injection/i, + }); + }, + ); +}); + +function shellInputTextCalls(calls: string[][]): string[][] { + return calls.filter((args) => isShellInput(args, 'text')); +} + +/** + * Answers the snapshot-helper version probe and `am instrument` capture with a + * one-EditText hierarchy holding `resolveText()`, mirroring the PATH-stub + * helper script this file used before provider injection. Returns undefined for + * every other invocation so the caller's script keeps handling input actions. + */ +function snapshotHelperResponse(args: string[], resolveText: () => string): string | undefined { + return androidSnapshotHelperScriptResponse( + args, + () => + ``, + ); +} diff --git a/src/platforms/android/__tests__/touch-helper-session.test.ts b/src/platforms/android/__tests__/touch-helper-session.test.ts index 81f32b0094..2513a9e7ff 100644 --- a/src/platforms/android/__tests__/touch-helper-session.test.ts +++ b/src/platforms/android/__tests__/touch-helper-session.test.ts @@ -17,11 +17,9 @@ import { type AndroidAdbProcess, type AndroidAdbProvider, } from '../adb-executor.ts'; -import { - captureAndroidSnapshotWithHelperSession, - getAndroidSnapshotHelperSessionDeviceKey, - resetAndroidSnapshotHelperSessions, -} from '../snapshot-helper-session.ts'; +import { captureAndroidSnapshotWithHelperSession } from '../snapshot-helper-session.ts'; +import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts'; +import { getAndroidSnapshotHelperSessionDeviceKey } from '../snapshot-helper-retirement.ts'; import { lowerAndroidTouchPlan } from '../touch-plan.ts'; import { executeAndroidTouchHelperPlan, readAndroidTouchHelperViewport } from '../touch-helper.ts'; import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from '../../../__tests__/test-utils/android-snapshot-helper.ts'; diff --git a/src/platforms/android/__tests__/touch-helper.test.ts b/src/platforms/android/__tests__/touch-helper.test.ts index 1c1ebd9672..46a100c8fb 100644 --- a/src/platforms/android/__tests__/touch-helper.test.ts +++ b/src/platforms/android/__tests__/touch-helper.test.ts @@ -3,7 +3,7 @@ import { afterEach, beforeEach, test, vi } from 'vitest'; import { buildGesturePlan } from '@agent-device/contracts/gesture-plan'; import { AppError } from '@agent-device/kernel/errors'; import { withAndroidAdbProvider } from '../adb-executor.ts'; -import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session.ts'; +import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts'; import { executeAndroidTouchPlan } from '../touch-executor.ts'; import { lowerAndroidTouchPlan } from '../touch-plan.ts'; import { diff --git a/src/platforms/android/adb-shell-protocol.ts b/src/platforms/android/adb-shell-protocol.ts new file mode 100644 index 0000000000..38d4568e8a --- /dev/null +++ b/src/platforms/android/adb-shell-protocol.ts @@ -0,0 +1,59 @@ +import type { AndroidAdbExecutor } from './adb-executor.ts'; + +/** + * adb only forwards a device command's exit status over shell protocol v2. Both ends must support + * it: adb reports the features it can actually use with the connected device, so this name in that + * list is the negotiated result rather than a client-side claim. + */ +const SHELL_PROTOCOL_V2_FEATURE = 'shell_v2'; +const FEATURE_PROBE_TIMEOUT_MS = 2_000; + +const deviceExitStatusForwarding = new Map(); + +/** + * Whether the exit status of a host `adb shell` child is the DEVICE command's exit status. + * + * Without shell protocol v2 adb exits 0 whenever the connection closed cleanly — including when + * the device-side command was killed or never finished — so a host exit code says nothing about + * the device. Callers that would skip a device-side confirmation on the strength of a host exit + * code must skip only when this answers `true`: an unsupported transport, a failed probe, and an + * adb too old to know `features` all answer `false`, because none of them proves anything. + * + * The negotiated feature set is fixed for a device's connection, so it is probed once per device. + */ +export async function androidAdbForwardsDeviceExitStatus(params: { + adb: AndroidAdbExecutor; + deviceKey: string; + signal?: AbortSignal; +}): Promise { + const cached = deviceExitStatusForwarding.get(params.deviceKey); + if (cached !== undefined) return cached; + let stdout: string; + try { + const result = await params.adb(['features'], { + allowFailure: true, + timeoutMs: FEATURE_PROBE_TIMEOUT_MS, + ...(params.signal ? { signal: params.signal } : {}), + }); + // A probe that did not run leaves the transport unknown rather than unsupported: nothing is + // cached, so the next teardown asks again instead of inheriting a transient failure. + if (result.exitCode !== 0) return false; + stdout = result.stdout; + } catch { + return false; + } + const forwards = parseAndroidAdbFeatures(stdout).includes(SHELL_PROTOCOL_V2_FEATURE); + deviceExitStatusForwarding.set(params.deviceKey, forwards); + return forwards; +} + +export function resetAndroidAdbShellProtocolProbes(): void { + deviceExitStatusForwarding.clear(); +} + +function parseAndroidAdbFeatures(stdout: string): string[] { + return stdout + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); +} diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index 86601f7236..5b1822c18a 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -1,7 +1,10 @@ +/** + * Pointer, key, and gesture actions on an Android device. Text entry — provider injection, the test + * IME, and the adb-shell writer — is `text-input.ts`. + */ import { DEVICE_ROTATION_SURFACE_INDEX, type DeviceRotation } from '@agent-device/contracts/device'; import { buildGesturePlan } from '@agent-device/contracts/gesture-plan'; import { GESTURE_DURATION_MIN_MS } from '@agent-device/contracts/gesture-plan-types'; -import type { FillUnconfirmedVerification } from '@agent-device/contracts/interactor-types'; import { DEFAULT_MOBILE_SCROLL_DURATION_MS } from '@agent-device/contracts/scroll-command'; import { type ScrollDirection, @@ -10,34 +13,10 @@ import { import { type TvRemoteButton, toAndroidTvRemoteKeyevent } from '@agent-device/contracts/tv-remote'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; -import { emitDiagnostic } from '../../utils/diagnostics.ts'; -import { shellQuoteIfNeeded } from '../../utils/shell-quote.ts'; -import { - resolveAndroidAdbExecutor, - resolveAndroidAdbProvider, - resolveAndroidTextInjector, - type AndroidTextInputAction, -} from './adb-executor.ts'; -import { runAndroidAdb, sleep } from './adb.ts'; -import { getAndroidKeyboardState, type AndroidKeyboardState } from './device-input-state.ts'; -import { - buildAndroidFillUnconfirmedVerification, - completeAndroidFillVerification, - readAndroidFillTargetBeforeMutation, - verifyAndroidFilledText, - type AndroidFillVerification, -} from './fill-verification.ts'; -import { - clearAndroidImeHelperText, - selectAndroidImeHelperArtifact, - sendAndroidImeHelperText, -} from './ime-helper.ts'; -import { isAndroidTestImeActive } from './ime-lifecycle.ts'; +import { runAndroidAdb } from './adb.ts'; import { executeAndroidTouchPlan, readAndroidGestureViewport } from './touch-executor.ts'; import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; -export { readAndroidTextAtPoint } from './fill-verification.ts'; - export async function pressAndroid(device: DeviceInfo, x: number, y: number): Promise { await runAndroidAdb(device, ['shell', 'input', 'tap', String(x), String(y)]); } @@ -114,161 +93,10 @@ export async function longPressAndroid( }); } -export async function typeAndroid(device: DeviceInfo, text: string, delayMs = 0): Promise { - const providerText = resolveAndroidTextInjector(device); - if (providerText) { - await providerText({ action: 'type', text, delayMs }); - emitAndroidTextDiagnostic('type', 'provider-native', text); - return; - } - if (isAndroidTestImeActive(device)) { - await typeAndroidTestIme(device, text, delayMs); - return; - } - assertAndroidShellTextSupported(text); - await assertAndroidShellInputIsAppOwned(device, 'type'); - if (delayMs > 0 && Array.from(text).length > 1) { - await typeAndroidShell(device, { action: 'type', text, chunkSize: 1, delayMs }); - return; - } - await typeAndroidShell(device, { - action: 'type', - text, - chunkSize: ANDROID_INPUT_TEXT_CHUNK_SIZE, - delayMs: 0, - }); -} - export async function focusAndroid(device: DeviceInfo, x: number, y: number): Promise { await pressAndroid(device, x, y); } -export async function fillAndroid( - device: DeviceInfo, - x: number, - y: number, - text: string, - delayMs = 0, - helper: AndroidHelperSessionOptions = {}, -): Promise { - const beforeTarget = await readAndroidFillTargetBeforeMutation(device, x, y, helper); - const providerText = resolveAndroidTextInjector(device); - if (providerText) { - await providerText({ action: 'fill', target: { x, y }, text, delayMs }); - emitAndroidTextDiagnostic('fill', 'provider-native', text); - const verification = await verifyAndroidFilledText(device, x, y, text, helper); - return completeAndroidFillVerification(text, beforeTarget, verification); - } - if (isAndroidTestImeActive(device)) { - const verification = await fillAndroidTestIme(device, x, y, text, beforeTarget, helper); - return completeAndroidFillVerification(text, beforeTarget, verification); - } - assertAndroidShellTextSupported(text); - - const textCodePointLength = Array.from(text).length; - const attempts: Array<{ - clearPadding: number; - minClear: number; - maxClear: number; - chunkSize: number; - inputDelayMs: number; - }> = [ - { - clearPadding: 12, - minClear: 8, - maxClear: 48, - chunkSize: delayMs > 0 ? 1 : ANDROID_INPUT_TEXT_CHUNK_SIZE, - inputDelayMs: delayMs, - }, - { - clearPadding: 24, - minClear: 16, - maxClear: 96, - chunkSize: delayMs > 0 ? 1 : 4, - inputDelayMs: delayMs > 0 ? delayMs : 15, - }, - ]; - - let lastVerification: AndroidFillVerification | null = null; - - for (const attempt of attempts) { - await focusAndroid(device, x, y); - await assertAndroidShellInputIsAppOwned(device, 'fill'); - const clearCount = clampCount( - textCodePointLength + attempt.clearPadding, - attempt.minClear, - attempt.maxClear, - ); - await clearFocusedText(device, clearCount); - await typeAndroidShell(device, { - action: 'fill', - text, - chunkSize: attempt.chunkSize, - delayMs: attempt.inputDelayMs, - }); - const verification = await verifyAndroidFilledText(device, x, y, text, helper); - lastVerification = verification; - if (verification.ok) return; - if (verification.reason === 'ime_capture') { - return completeAndroidFillVerification(text, beforeTarget, verification); - } - const unconfirmed = buildAndroidFillUnconfirmedVerification(text, beforeTarget, verification); - if (unconfirmed) return unconfirmed; - } - - return completeAndroidFillVerification(text, beforeTarget, lastVerification); -} - -async function typeAndroidTestIme( - device: DeviceInfo, - text: string, - delayMs: number, -): Promise { - const adb = resolveAndroidAdbExecutor(device); - const artifact = await selectAndroidImeHelperArtifact(resolveAndroidAdbProvider(device)); - const packageName = artifact.manifest.packageName; - const parts = text.split('\n'); - for (const [partIndex, part] of parts.entries()) { - const chunks = delayMs > 0 ? chunkAndroidInputText(part, 1) : [part]; - for (const [chunkIndex, chunk] of chunks.entries()) { - if (chunk) await sendAndroidImeHelperText(adb, packageName, chunk); - if (delayMs > 0 && (chunkIndex + 1 < chunks.length || partIndex + 1 < parts.length)) { - await sleep(delayMs); - } - } - if (partIndex + 1 < parts.length) { - await runAndroidAdb(device, ['shell', 'input', 'keyevent', 'ENTER']); - } - } - emitAndroidTextDiagnostic('type', 'test-ime', text); -} - -async function fillAndroidTestIme( - device: DeviceInfo, - x: number, - y: number, - text: string, - beforeTarget: AndroidFillVerification['targetInput'], - helper: AndroidHelperSessionOptions, -): Promise { - const adb = resolveAndroidAdbExecutor(device); - const artifact = await selectAndroidImeHelperArtifact(resolveAndroidAdbProvider(device)); - const packageName = artifact.manifest.packageName; - let lastVerification: AndroidFillVerification | null = null; - // One retry covers the rare not-yet-bound InputConnection right after focus. - for (let attempt = 0; attempt < 2; attempt += 1) { - await focusAndroid(device, x, y); - await clearAndroidImeHelperText(adb, packageName); - if (text) await sendAndroidImeHelperText(adb, packageName, text); - const verification = await verifyAndroidFilledText(device, x, y, text, helper); - lastVerification = verification; - if (verification.ok) break; - if (buildAndroidFillUnconfirmedVerification(text, beforeTarget, verification)) break; - } - emitAndroidTextDiagnostic('fill', 'test-ime', text); - return lastVerification as AndroidFillVerification; -} - export async function scrollAndroid( device: DeviceInfo, direction: ScrollDirection, @@ -333,43 +161,6 @@ function resolveAndroidUserRotation(orientation: DeviceRotation): string { return String(index); } -async function assertAndroidShellInputIsAppOwned( - device: DeviceInfo, - action: AndroidTextInputAction, -): Promise { - let state: AndroidKeyboardState; - try { - state = await getAndroidKeyboardState(device); - } catch (error) { - emitDiagnostic({ - level: 'warn', - phase: 'android_input_ownership_probe_failed', - data: { - action, - error: error instanceof Error ? error.message : String(error), - }, - }); - return; - } - if (state.inputOwner !== 'ime') return; - throw new AppError( - 'COMMAND_FAILED', - 'KEYBOARD_OVERLAY_BLOCKING: Android text input is blocked because the focused input belongs to the active keyboard/IME.', - { - failureReason: 'ime_capture', - action, - inputOwner: state.inputOwner, - inputType: state.inputType, - type: state.type, - inputMethodPackage: state.inputMethodPackage, - focusedPackage: state.focusedPackage, - focusedResourceId: state.focusedResourceId, - nextAction: - 'Focused input appears to be owned by the keyboard/IME; dismiss or change the IME before retrying text entry.', - }, - ); -} - export async function getAndroidScreenSize( device: DeviceInfo, ): Promise<{ width: number; height: number }> { @@ -378,131 +169,3 @@ export async function getAndroidScreenSize( if (!match) throw new AppError('COMMAND_FAILED', 'Unable to read screen size'); return { width: Number(match[1]), height: Number(match[2]) }; } - -const ANDROID_INPUT_TEXT_CHUNK_SIZE = 8; - -async function typeAndroidShell( - device: DeviceInfo, - options: { action: AndroidTextInputAction; text: string; chunkSize: number; delayMs: number }, -): Promise { - const parts = options.text.split('\n'); - for (const [partIndex, part] of parts.entries()) { - const chunks = chunkAndroidInputText(part, options.chunkSize); - for (const [chunkIndex, chunk] of chunks.entries()) { - await typeAndroidShellChunk(device, chunk); - if (options.delayMs > 0 && (chunkIndex + 1 < chunks.length || partIndex + 1 < parts.length)) { - await sleep(options.delayMs); - } - } - if (partIndex + 1 < parts.length) { - await runAndroidAdb(device, ['shell', 'input', 'keyevent', 'ENTER']); - } - } - emitAndroidTextDiagnostic(options.action, 'adb-shell', options.text); -} - -async function typeAndroidShellChunk(device: DeviceInfo, text: string): Promise { - if (!text) return; - try { - await runAndroidAdb(device, [ - 'shell', - 'input', - 'text', - shellQuoteIfNeeded(encodeAndroidInputText(text)), - ]); - } catch (error) { - if (isAndroidInputTextUnsupported(error)) { - throw unsupportedAndroidShellTextError(text, error); - } - throw error; - } -} - -function assertAndroidShellTextSupported(text: string): void { - if (isAndroidShellTextSupported(text)) return; - throw unsupportedAndroidShellTextError(text); -} - -function isAndroidShellTextSupported(text: string): boolean { - for (const char of text) { - const code = char.codePointAt(0); - if (code === undefined) continue; - if (char === '\n') continue; - if (code < 0x20 || code > 0x7e) { - return false; - } - } - return true; -} - -function encodeAndroidInputText(text: string): string { - // Android shell input uses `%s` as the escaped token for spaces. - return text.replace(/ /g, '%s'); -} - -function isAndroidInputTextUnsupported(error: unknown): boolean { - if (!(error instanceof AppError)) return false; - if (error.code !== 'COMMAND_FAILED') return false; - const rawStderr = error.details?.stderr; - const stderr = (typeof rawStderr === 'string' ? rawStderr : '').toLowerCase(); - if (stderr.includes("exception occurred while executing 'text'")) return true; - if (stderr.includes('nullpointerexception') && stderr.includes('inputshellcommand.sendtext')) - return true; - return false; -} - -function unsupportedAndroidShellTextError(text: string, cause?: unknown): AppError { - return new AppError( - 'COMMAND_FAILED', - 'Android text input requires provider-native text injection or the bundled test IME helper for non-ASCII/control characters; the adb-shell fallback supports ASCII text only. On emulators the test IME activates automatically; on real devices pass `open --test-ime` to enable it (see `agent-device doctor` for the current IME state).', - { - backend: 'adb-shell', - textLength: Array.from(text).length, - textPreview: text.slice(0, 32), - }, - cause instanceof Error ? cause : undefined, - ); -} - -function chunkAndroidInputText(text: string, chunkSize: number): string[] { - const size = Math.max(1, Math.floor(chunkSize)); - const chunks: string[] = []; - const chars = Array.from(text); - for (let i = 0; i < chars.length; i += size) { - chunks.push(chars.slice(i, i + size).join('')); - } - return chunks.length > 0 ? chunks : ['']; -} - -function emitAndroidTextDiagnostic( - action: AndroidTextInputAction, - backend: 'provider-native' | 'adb-shell' | 'test-ime', - text: string, -): void { - emitDiagnostic({ - phase: 'android_text_injection', - data: { action, backend, textLength: Array.from(text).length }, - }); -} - -async function clearFocusedText(device: DeviceInfo, count: number): Promise { - const deletes = Math.max(0, count); - await runAndroidAdb(device, ['shell', 'input', 'keyevent', 'KEYCODE_MOVE_END'], { - allowFailure: true, - }); - const batchSize = 24; - for (let i = 0; i < deletes; i += batchSize) { - const size = Math.min(batchSize, deletes - i); - await runAndroidAdb( - device, - ['shell', 'input', 'keyevent', ...Array(size).fill('KEYCODE_DEL')], - { - allowFailure: true, - }, - ); - } -} - -function clampCount(value: number, min: number, max: number): number { - return Math.max(min, Math.min(max, value)); -} diff --git a/src/platforms/android/snapshot-helper-runtime.ts b/src/platforms/android/snapshot-helper-runtime.ts index cf74db73b1..3a7bf1397b 100644 --- a/src/platforms/android/snapshot-helper-runtime.ts +++ b/src/platforms/android/snapshot-helper-runtime.ts @@ -2,7 +2,7 @@ import { normalizeError } from '@agent-device/kernel/errors'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import { sleep } from './adb.ts'; import type { AndroidAdbExecutor } from './adb-executor.ts'; -import { stopAndroidSnapshotHelperSession } from './snapshot-helper-session.ts'; +import { stopAndroidSnapshotHelperSession } from './snapshot-helper-session-lifecycle.ts'; const HELPER_RUNTIME_RESET_DELAY_MS = 150; const HELPER_RUNTIME_RESET_TIMEOUT_MS = 2_000; diff --git a/src/platforms/android/snapshot-helper-session-lifecycle.ts b/src/platforms/android/snapshot-helper-session-lifecycle.ts new file mode 100644 index 0000000000..f01b6d6040 --- /dev/null +++ b/src/platforms/android/snapshot-helper-session-lifecycle.ts @@ -0,0 +1,449 @@ +/** + * Who owns the device's UiAutomation right now, and how that ownership starts and ends. + * + * Android permits ONE UiAutomation owner, so a live helper session is device-exclusive state: this + * module is the only place that starts one, hands it out, and retires it. Commands run OVER a + * session (snapshot capture, gestures) live in `snapshot-helper-session.ts`; they acquire through + * here and never reach the registry themselves. + */ +import type { AndroidAdbProcess } from './adb-executor.ts'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; +import { emitDiagnostic } from '../../utils/diagnostics.ts'; +import { + androidAdbForwardsDeviceExitStatus, + resetAndroidAdbShellProtocolProbes, +} from './adb-shell-protocol.ts'; +import type { + AndroidAdbExecutor, + AndroidSnapshotHelperCaptureOptions, +} from './snapshot-helper-types.ts'; +import { + buildAndroidSnapshotHelperArgs, + resolveAndroidSnapshotHelperCaptureOptions, + type AndroidSnapshotHelperResolvedCaptureOptions, +} from './snapshot-helper-capture.ts'; +import { + allocateAndroidSnapshotHelperSessionPort, + isAndroidSnapshotHelperSessionCommandAcknowledged, + sendAndroidSnapshotHelperSessionCommand, + waitForAndroidSnapshotHelperSessionReady, +} from './snapshot-helper-session-protocol.ts'; +import { + type AndroidSnapshotHelperProcessExit, + ANDROID_SNAPSHOT_HELPER_DEVICE_RETIREMENT_TIMEOUT_MS, + ANDROID_SNAPSHOT_HELPER_HOST_PROCESS_EXIT_GRACE_MS, + getAndroidSnapshotHelperSessionDeviceKey, + isAndroidSnapshotHelperRetirementUnconfirmedError, + observeAndroidSnapshotHelperProcessExit, + quarantineAndroidSnapshotHelperRetirement, + recoverAndroidSnapshotHelperRetirement, + resetAndroidSnapshotHelperRetirements, + settleAndroidSnapshotHelperSessionCleanup, + stopAndroidSnapshotHelperHostProcess, + waitForAndroidSnapshotHelperProcessExit, +} from './snapshot-helper-retirement.ts'; + +const SESSION_READY_TIMEOUT_MS = 10_000; +const SESSION_STOP_TIMEOUT_MS = 1_000; +// SnapshotInstrumentation.finishSafely can spend up to 10 seconds waiting for Android to finish +// connecting UiAutomation. Let an acknowledged quit complete that release before force-killing adb. +const SESSION_GRACEFUL_EXIT_TIMEOUT_MS = 11_000; +const SESSION_PROCESS_EXIT_TIMEOUT_MS = 2_000; +// Persistent capture is an optimization before the required one-shot path. Keep its native and +// transport budgets shorter so a wedged UiAutomation connection leaves time for a clean fallback. +const SESSION_CAPTURE_TIMEOUT_MS = 2_000; +const SESSION_REQUEST_OVERHEAD_MS = 3_000; +const FORWARD_TIMEOUT_MS = 5_000; + +export type AndroidSnapshotHelperSessionHelperIdentity = { + packageName: string; + runner: string; + helperVersion?: string; + helperVersionCode?: number; + sha256?: string; +}; + +export type AndroidSnapshotHelperSession = { + identity: string; + deviceKey: string; + helper: AndroidSnapshotHelperSessionHelperIdentity; + port: number; + adb: AndroidAdbExecutor; + process: AndroidAdbProcess; + startedAtMs: number; + capturedCount: number; +}; + +/** A session this caller may run commands on, with the budgets it was started under. */ +export type AndroidSnapshotHelperSessionAcquisition = { + session: AndroidSnapshotHelperSession; + resolved: AndroidSnapshotHelperResolvedCaptureOptions; + deviceKey: string; +}; + +const sessions = new Map(); +const disabledSessionIdentities = new Map(); + +/** + * Starts (or reuses) the session without capturing, so a helper-backed read that is not a snapshot + * — the gesture viewport — can leave a warm session behind for the gesture that follows instead of + * paying its own one-shot instrumentation. Answers whether a session is live; `false` means the + * caller must use the one-shot transport, exactly as when a capture cannot use the session. + */ +export async function ensureAndroidSnapshotHelperSession( + options: AndroidSnapshotHelperCaptureOptions, +): Promise { + return (await acquireAndroidSnapshotHelperSession(options)) !== undefined; +} + +export async function acquireAndroidSnapshotHelperSession( + options: AndroidSnapshotHelperCaptureOptions, +): Promise { + const deviceKey = options.deviceKey ?? 'android:default'; + await recoverAndroidSnapshotHelperRetirement({ + deviceKey, + adb: options.adb, + signal: options.signal, + }); + if (!isAndroidSnapshotHelperSessionEnabled() || !options.adbProvider?.spawn) { + return undefined; + } + const resolved = resolvePersistentSessionCaptureOptions( + resolveAndroidSnapshotHelperCaptureOptions(options), + ); + const identity = createSessionIdentity(deviceKey, resolved, options); + const session = await resolveAndroidSnapshotHelperSession({ + deviceKey, + identity, + options, + resolved, + }); + return session ? { session, resolved, deviceKey } : undefined; +} + +/** + * The live session for a device, or `undefined` when none is running. Commands that may only + * piggyback on an existing session — never start one — read ownership through this. + */ +export function getLiveAndroidSnapshotHelperSession( + deviceKey: string, +): AndroidSnapshotHelperSession | undefined { + return sessions.get(deviceKey); +} + +async function resolveAndroidSnapshotHelperSession(params: { + deviceKey: string; + identity: string; + options: AndroidSnapshotHelperCaptureOptions; + resolved: AndroidSnapshotHelperResolvedCaptureOptions; +}): Promise { + const { deviceKey, identity, options, resolved } = params; + if (disabledSessionIdentities.get(deviceKey) === identity) { + return undefined; + } + let session = sessions.get(deviceKey); + if (session && session.identity !== identity) { + await stopAndroidSnapshotHelperSession(deviceKey); + session = undefined; + } + if (!session) { + try { + session = await startAndroidSnapshotHelperSession({ + deviceKey, + identity, + options, + resolved, + }); + } catch (error) { + options.signal?.throwIfAborted(); + disabledSessionIdentities.set(deviceKey, identity); + emitDiagnostic({ + level: 'warn', + phase: 'android_snapshot_helper_session_disabled', + data: { + deviceKey, + reason: error instanceof Error ? error.message : String(error), + }, + }); + if (isAndroidSnapshotHelperRetirementUnconfirmedError(error)) { + throw error; + } + return undefined; + } + } + return session; +} + +async function startAndroidSnapshotHelperSession(params: { + deviceKey: string; + identity: string; + options: AndroidSnapshotHelperCaptureOptions; + resolved: AndroidSnapshotHelperResolvedCaptureOptions; +}): Promise { + const port = await allocateAndroidSnapshotHelperSessionPort(); + await params.options.adb(['forward', `tcp:${port}`, `tcp:${port}`], { + allowFailure: false, + timeoutMs: FORWARD_TIMEOUT_MS, + signal: params.options.signal, + }); + const args = buildAndroidSnapshotHelperArgs({ + ...params.resolved, + outputPath: undefined, + emitChunks: false, + }); + const runner = args[args.length - 1]; + if (!runner) { + throw new AppError('INVALID_ARGS', 'Android snapshot helper runner was not resolved'); + } + const sessionArgs = [...args.slice(0, -1), '-e', 'sessionPort', String(port), runner]; + const process = params.options.adbProvider!.spawn!(sessionArgs, { + allowFailure: true, + captureOutput: false, + }); + const session: AndroidSnapshotHelperSession = { + identity: params.identity, + deviceKey: params.deviceKey, + helper: { + packageName: params.resolved.packageName, + runner: params.resolved.runner, + helperVersion: params.options.helperVersion, + helperVersionCode: params.options.helperVersionCode, + sha256: params.options.helperSha256, + }, + port, + adb: params.options.adb, + process, + startedAtMs: Date.now(), + capturedCount: 0, + }; + try { + await waitForAndroidSnapshotHelperSessionReady( + process, + SESSION_READY_TIMEOUT_MS, + params.options.signal, + ); + sessions.set(params.deviceKey, session); + emitDiagnostic({ + phase: 'android_snapshot_helper_session_ready', + data: { + deviceKey: params.deviceKey, + port, + packageName: params.resolved.packageName, + runner: params.resolved.runner, + }, + }); + return session; + } catch (error) { + const processExit = observeAndroidSnapshotHelperProcessExit(process); + try { + process.kill('SIGTERM'); + } catch { + // Best effort after startup failure. + } + const [, cleanup] = await Promise.all([ + waitForAndroidSnapshotHelperProcessExit( + processExit.ended, + ANDROID_SNAPSHOT_HELPER_HOST_PROCESS_EXIT_GRACE_MS, + ), + settleAndroidSnapshotHelperSessionCleanup({ + adb: session.adb, + process: session.process, + port: session.port, + packageName: session.helper.packageName, + timeoutMs: ANDROID_SNAPSHOT_HELPER_DEVICE_RETIREMENT_TIMEOUT_MS, + // Startup failed before the helper could acknowledge anything, so nothing proves it + // released UiAutomation. + forceStopRuntime: true, + }), + ]); + if (!cleanup.runtimeForceStopped) { + quarantineAndroidSnapshotHelperRetirement({ + deviceKey: params.deviceKey, + packageName: session.helper.packageName, + cause: error, + }); + } + throw error; + } +} + +function createSessionIdentity( + deviceKey: string, + resolved: AndroidSnapshotHelperResolvedCaptureOptions, + options: AndroidSnapshotHelperCaptureOptions, +): string { + const identity = JSON.stringify({ + deviceKey, + packageName: resolved.packageName, + runner: resolved.runner, + helperVersion: options.helperVersion, + helperVersionCode: options.helperVersionCode, + helperSha256: options.helperSha256, + waitForIdleTimeoutMs: resolved.waitForIdleTimeoutMs, + waitForIdleQuietMs: resolved.waitForIdleQuietMs, + timeoutMs: resolved.timeoutMs, + maxDepth: resolved.maxDepth, + maxNodes: resolved.maxNodes, + }); + return identity; +} + +function resolvePersistentSessionCaptureOptions( + resolved: AndroidSnapshotHelperResolvedCaptureOptions, +): AndroidSnapshotHelperResolvedCaptureOptions { + const timeoutMs = Math.min(resolved.timeoutMs, SESSION_CAPTURE_TIMEOUT_MS); + return { + ...resolved, + timeoutMs, + commandTimeoutMs: Math.min(resolved.commandTimeoutMs, timeoutMs + SESSION_REQUEST_OVERHEAD_MS), + }; +} + +function isAndroidSnapshotHelperSessionEnabled(): boolean { + const value = process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION; + return value === undefined || !/^(0|false|no|off)$/i.test(value); +} + +export async function stopAndroidSnapshotHelperSession( + deviceKey: string, + options: { + /** Skip the graceful quit entirely: kill the host process and stop the device runtime. */ + force?: boolean; + /** + * Stop the device runtime even when the quit proved release. Recovery paths — a helper whose + * output failed content validation — restart the helper on purpose, so they cannot read "it + * quit politely" as a reason to leave a suspect process owning the runtime. + */ + resetRuntime?: boolean; + signal?: AbortSignal; + cause?: unknown; + } = {}, +): Promise { + const session = sessions.get(deviceKey); + if (!session) return false; + sessions.delete(deviceKey); + const processExit = observeAndroidSnapshotHelperProcessExit(session.process); + const force = options.force === true || options.signal?.aborted === true; + const graceful = await requestGracefulSessionExit(session, processExit, force, options.signal); + const hostProcessEnded = processExit.hasEnded(); + // The helper releases UiAutomation inside its own quit handling, so a quit it acknowledged and + // then completed IS the release evidence — but only where the host exit code it is read from + // belongs to the device. `exited` separates "the helper said it would quit" from "the helper + // finished quitting" (see AndroidSnapshotHelperProcessExit); the transport probe separates an + // exit status adb forwarded from the device from one adb invented for a closed connection. + // Anything less is not evidence, and the device-side stop runs. + const deviceExitObserved = graceful.acknowledged && graceful.exited; + const runtimeReleaseConfirmed = + deviceExitObserved && + (await androidAdbForwardsDeviceExitStatus({ + adb: session.adb, + deviceKey, + signal: options.signal, + })); + const cleanupTimeoutMs = !force + ? FORWARD_TIMEOUT_MS + : options.signal?.aborted === true + ? ANDROID_SNAPSHOT_HELPER_HOST_PROCESS_EXIT_GRACE_MS + : ANDROID_SNAPSHOT_HELPER_DEVICE_RETIREMENT_TIMEOUT_MS; + const processExitTimeoutMs = force + ? ANDROID_SNAPSHOT_HELPER_HOST_PROCESS_EXIT_GRACE_MS + : SESSION_PROCESS_EXIT_TIMEOUT_MS; + const [processStopped, cleanup] = await Promise.all([ + stopAndroidSnapshotHelperHostProcess({ + process: session.process, + processExit, + timeoutMs: processExitTimeoutMs, + }), + settleAndroidSnapshotHelperSessionCleanup({ + adb: session.adb, + process: session.process, + port: session.port, + packageName: session.helper.packageName, + timeoutMs: cleanupTimeoutMs, + forceStopRuntime: options.resetRuntime === true || !runtimeReleaseConfirmed, + }), + ]); + emitDiagnostic({ + phase: 'android_snapshot_helper_session_stop', + data: { + deviceKey, + port: session.port, + capturedCount: session.capturedCount, + lifetimeMs: Date.now() - session.startedAtMs, + quitAcknowledged: graceful.acknowledged, + // With the exit observed but the release unconfirmed, the transport is what failed to prove it. + quitExitObserved: deviceExitObserved, + runtimeReleaseConfirmed, + forceKilled: !hostProcessEnded && processStopped, + forced: force || options.signal?.aborted === true, + runtimeForceStopped: cleanup.runtimeForceStopped, + externalCleanupTimedOut: cleanup.timedOut, + }, + }); + // Either the release was proven or the device-side stop confirmed it. An unproven quit whose + // stop also failed leaves ownership unknown, which is what quarantine exists to report. + if (!runtimeReleaseConfirmed && !cleanup.runtimeForceStopped) { + quarantineAndroidSnapshotHelperRetirement({ + deviceKey, + packageName: session.helper.packageName, + cause: options.cause, + }); + } + return true; +} + +async function requestGracefulSessionExit( + session: AndroidSnapshotHelperSession, + processExit: AndroidSnapshotHelperProcessExit, + force: boolean, + signal: AbortSignal | undefined, +): Promise<{ + acknowledged: boolean; + /** The instrumentation this teardown asked to quit then finished on its own, cleanly. */ + exited: boolean; +}> { + if (force) return { acknowledged: false, exited: false }; + const requestId = `quit-${Date.now()}`; + try { + const response = await sendAndroidSnapshotHelperSessionCommand( + session.port, + `quit ${requestId}`, + SESSION_STOP_TIMEOUT_MS, + signal, + ); + const acknowledged = isAndroidSnapshotHelperSessionCommandAcknowledged(response, requestId); + const exited = + acknowledged && + (await waitForAndroidSnapshotHelperProcessExit( + processExit.ended, + SESSION_GRACEFUL_EXIT_TIMEOUT_MS, + signal, + )) && + processExit.completedCleanly(); + return { acknowledged, exited }; + } catch { + return { acknowledged: false, exited: false }; + } +} + +export async function stopAndroidSnapshotHelperSessionForDevice( + device: Pick, +): Promise { + await stopAndroidSnapshotHelperSession(getAndroidSnapshotHelperSessionDeviceKey(device)); +} + +export async function resetAndroidSnapshotHelperSessions(): Promise { + const retirements = await Promise.allSettled( + [...sessions.keys()].map((deviceKey) => stopAndroidSnapshotHelperSession(deviceKey)), + ); + disabledSessionIdentities.clear(); + const failures = retirements + .filter((result): result is PromiseRejectedResult => result.status === 'rejected') + .map((result) => result.reason); + if (failures.length > 0) { + throw new AggregateError(failures, 'Failed to retire every Android snapshot helper session'); + } + resetAndroidSnapshotHelperRetirements(); + resetAndroidAdbShellProtocolProbes(); +} diff --git a/src/platforms/android/snapshot-helper-session.ts b/src/platforms/android/snapshot-helper-session.ts index 6cc2914412..662e6ce870 100644 --- a/src/platforms/android/snapshot-helper-session.ts +++ b/src/platforms/android/snapshot-helper-session.ts @@ -1,78 +1,28 @@ -import type { AndroidAdbProcess } from './adb-executor.ts'; -import type { DeviceInfo } from '@agent-device/kernel/device'; +/** + * Commands that run OVER a live automation-helper session: snapshot capture and the touch commands + * that piggyback on it. Session ownership itself — starting, reusing, retiring — belongs to + * `snapshot-helper-session-lifecycle.ts`, which this module acquires through. + */ import { AppError } from '@agent-device/kernel/errors'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; -import { - type AndroidAdbExecutor, - type AndroidSnapshotHelperCaptureOptions, - type AndroidSnapshotHelperOutput, +import type { + AndroidSnapshotHelperCaptureOptions, + AndroidSnapshotHelperOutput, } from './snapshot-helper-types.ts'; +import type { AndroidSnapshotHelperResolvedCaptureOptions } from './snapshot-helper-capture.ts'; import { - buildAndroidSnapshotHelperArgs, - resolveAndroidSnapshotHelperCaptureOptions, - type AndroidSnapshotHelperResolvedCaptureOptions, -} from './snapshot-helper-capture.ts'; -import { - allocateAndroidSnapshotHelperSessionPort, assertAndroidSnapshotHelperTouchSessionHeaders, - isAndroidSnapshotHelperSessionCommandAcknowledged, parseAndroidSnapshotHelperSessionHeaders, requestAndroidSnapshotHelperSessionSnapshot, sendAndroidSnapshotHelperSessionCommand, - waitForAndroidSnapshotHelperSessionReady, } from './snapshot-helper-session-protocol.ts'; import { - type AndroidSnapshotHelperProcessExit, - ANDROID_SNAPSHOT_HELPER_DEVICE_RETIREMENT_TIMEOUT_MS, - ANDROID_SNAPSHOT_HELPER_HOST_PROCESS_EXIT_GRACE_MS, - getAndroidSnapshotHelperSessionDeviceKey, - isAndroidSnapshotHelperRetirementUnconfirmedError, - observeAndroidSnapshotHelperProcessExit, - quarantineAndroidSnapshotHelperRetirement, - recoverAndroidSnapshotHelperRetirement, - resetAndroidSnapshotHelperRetirements, - settleAndroidSnapshotHelperSessionCleanup, - stopAndroidSnapshotHelperHostProcess, - waitForAndroidSnapshotHelperProcessExit, -} from './snapshot-helper-retirement.ts'; -export { - getAndroidSnapshotHelperSessionDeviceKey, - isAndroidSnapshotHelperRetirementUnconfirmedError, - recoverAndroidSnapshotHelperRetirement, -} from './snapshot-helper-retirement.ts'; -const SESSION_READY_TIMEOUT_MS = 10_000; -const SESSION_STOP_TIMEOUT_MS = 1_000; -// SnapshotInstrumentation.finishSafely can spend up to 10 seconds waiting for Android to finish -// connecting UiAutomation. Let an acknowledged quit complete that release before force-killing adb. -const SESSION_GRACEFUL_EXIT_TIMEOUT_MS = 11_000; -const SESSION_PROCESS_EXIT_TIMEOUT_MS = 2_000; -// Persistent capture is an optimization before the required one-shot path. Keep its native and -// transport budgets shorter so a wedged UiAutomation connection leaves time for a clean fallback. -const SESSION_CAPTURE_TIMEOUT_MS = 2_000; -const SESSION_REQUEST_OVERHEAD_MS = 3_000; -const FORWARD_TIMEOUT_MS = 5_000; - -type AndroidSnapshotHelperSessionHelperIdentity = { - packageName: string; - runner: string; - helperVersion?: string; - helperVersionCode?: number; - sha256?: string; -}; - -type AndroidSnapshotHelperSession = { - identity: string; - deviceKey: string; - helper: AndroidSnapshotHelperSessionHelperIdentity; - port: number; - adb: AndroidAdbExecutor; - process: AndroidAdbProcess; - startedAtMs: number; - capturedCount: number; -}; - -const sessions = new Map(); -const disabledSessionIdentities = new Map(); + acquireAndroidSnapshotHelperSession, + getLiveAndroidSnapshotHelperSession, + stopAndroidSnapshotHelperSession, + type AndroidSnapshotHelperSession, + type AndroidSnapshotHelperSessionHelperIdentity, +} from './snapshot-helper-session-lifecycle.ts'; export async function captureAndroidSnapshotWithHelperSession( options: AndroidSnapshotHelperCaptureOptions, @@ -87,93 +37,6 @@ export async function captureAndroidSnapshotWithHelperSession( }); } -/** - * Starts (or reuses) the session without capturing, so a helper-backed read that is not a snapshot - * — the gesture viewport — can leave a warm session behind for the gesture that follows instead of - * paying its own one-shot instrumentation. Answers whether a session is live; `false` means the - * caller must use the one-shot transport, exactly as when a capture cannot use the session. - */ -export async function ensureAndroidSnapshotHelperSession( - options: AndroidSnapshotHelperCaptureOptions, -): Promise { - return (await acquireAndroidSnapshotHelperSession(options)) !== undefined; -} - -async function acquireAndroidSnapshotHelperSession( - options: AndroidSnapshotHelperCaptureOptions, -): Promise< - | { - session: AndroidSnapshotHelperSession; - resolved: AndroidSnapshotHelperResolvedCaptureOptions; - deviceKey: string; - } - | undefined -> { - const deviceKey = options.deviceKey ?? 'android:default'; - await recoverAndroidSnapshotHelperRetirement({ - deviceKey, - adb: options.adb, - signal: options.signal, - }); - if (!isAndroidSnapshotHelperSessionEnabled() || !options.adbProvider?.spawn) { - return undefined; - } - const resolved = resolvePersistentSessionCaptureOptions( - resolveAndroidSnapshotHelperCaptureOptions(options), - ); - const identity = createSessionIdentity(deviceKey, resolved, options); - const session = await resolveAndroidSnapshotHelperSession({ - deviceKey, - identity, - options, - resolved, - }); - return session ? { session, resolved, deviceKey } : undefined; -} - -async function resolveAndroidSnapshotHelperSession(params: { - deviceKey: string; - identity: string; - options: AndroidSnapshotHelperCaptureOptions; - resolved: AndroidSnapshotHelperResolvedCaptureOptions; -}): Promise { - const { deviceKey, identity, options, resolved } = params; - if (disabledSessionIdentities.get(deviceKey) === identity) { - return undefined; - } - let session = sessions.get(deviceKey); - if (session && session.identity !== identity) { - await stopAndroidSnapshotHelperSession(deviceKey); - session = undefined; - } - if (!session) { - try { - session = await startAndroidSnapshotHelperSession({ - deviceKey, - identity, - options, - resolved, - }); - } catch (error) { - options.signal?.throwIfAborted(); - disabledSessionIdentities.set(deviceKey, identity); - emitDiagnostic({ - level: 'warn', - phase: 'android_snapshot_helper_session_disabled', - data: { - deviceKey, - reason: error instanceof Error ? error.message : String(error), - }, - }); - if (isAndroidSnapshotHelperRetirementUnconfirmedError(error)) { - throw error; - } - return undefined; - } - } - return session; -} - async function captureFromAndroidSnapshotHelperSession(params: { session: AndroidSnapshotHelperSession; deviceKey: string; @@ -218,17 +81,6 @@ async function captureFromAndroidSnapshotHelperSession(params: { } } -function resolvePersistentSessionCaptureOptions( - resolved: AndroidSnapshotHelperResolvedCaptureOptions, -): AndroidSnapshotHelperResolvedCaptureOptions { - const timeoutMs = Math.min(resolved.timeoutMs, SESSION_CAPTURE_TIMEOUT_MS); - return { - ...resolved, - timeoutMs, - commandTimeoutMs: Math.min(resolved.commandTimeoutMs, timeoutMs + SESSION_REQUEST_OVERHEAD_MS), - }; -} - function isUiAutomationConnectionTimeoutResponse(error: unknown): boolean { if (!(error instanceof AppError)) return false; const helper = error.details?.helper; @@ -246,7 +98,7 @@ export async function runAndroidSnapshotHelperSessionTouchCommand(params: { payloadBase64?: string; timeoutMs: number; }): Promise | undefined> { - const session = sessions.get(params.deviceKey); + const session = getLiveAndroidSnapshotHelperSession(params.deviceKey); if (!session) return undefined; if (!matchesSessionHelperIdentity(session.helper, params.helper)) { // A different helper binary was selected for this device (e.g. a provider-supplied artifact). @@ -310,254 +162,3 @@ function matchesSessionHelperIdentity( function matchesWhenBothDefined(a: Value | undefined, b: Value | undefined): boolean { return a === undefined || b === undefined || a === b; } - -export async function stopAndroidSnapshotHelperSession( - deviceKey: string, - options: { - /** Skip the graceful quit entirely: kill the host process and stop the device runtime. */ - force?: boolean; - /** - * Stop the device runtime even when the quit proved release. Recovery paths — a helper whose - * output failed content validation — restart the helper on purpose, so they cannot read "it - * quit politely" as a reason to leave a suspect process owning the runtime. - */ - resetRuntime?: boolean; - signal?: AbortSignal; - cause?: unknown; - } = {}, -): Promise { - const session = sessions.get(deviceKey); - if (!session) return false; - sessions.delete(deviceKey); - const processExit = observeAndroidSnapshotHelperProcessExit(session.process); - const force = options.force === true || options.signal?.aborted === true; - const graceful = await requestGracefulSessionExit(session, processExit, force, options.signal); - const hostProcessEnded = processExit.hasEnded(); - // The helper releases UiAutomation inside its own quit handling, so a quit it acknowledged and - // then completed IS the release evidence. Both halves are load-bearing: `exited` is what - // separates "the helper said it would quit" from "the helper finished quitting", and it counts - // only a clean exit of the process this teardown watched (see AndroidSnapshotHelperProcessExit). - // Anything less still needs the device-side stop. Declared residual: this is host-side evidence, - // so it is only as strong as adb's exit-code forwarding — a device without shell protocol v2 can - // report 0 for an instrumentation that did not finish, and the stop is then wrongly skipped. - const runtimeReleaseConfirmed = graceful.acknowledged && graceful.exited; - const cleanupTimeoutMs = !force - ? FORWARD_TIMEOUT_MS - : options.signal?.aborted === true - ? ANDROID_SNAPSHOT_HELPER_HOST_PROCESS_EXIT_GRACE_MS - : ANDROID_SNAPSHOT_HELPER_DEVICE_RETIREMENT_TIMEOUT_MS; - const processExitTimeoutMs = force - ? ANDROID_SNAPSHOT_HELPER_HOST_PROCESS_EXIT_GRACE_MS - : SESSION_PROCESS_EXIT_TIMEOUT_MS; - const [processStopped, cleanup] = await Promise.all([ - stopAndroidSnapshotHelperHostProcess({ - process: session.process, - processExit, - timeoutMs: processExitTimeoutMs, - }), - settleAndroidSnapshotHelperSessionCleanup({ - adb: session.adb, - process: session.process, - port: session.port, - packageName: session.helper.packageName, - timeoutMs: cleanupTimeoutMs, - forceStopRuntime: options.resetRuntime === true || !runtimeReleaseConfirmed, - }), - ]); - emitDiagnostic({ - phase: 'android_snapshot_helper_session_stop', - data: { - deviceKey, - port: session.port, - capturedCount: session.capturedCount, - lifetimeMs: Date.now() - session.startedAtMs, - quitAcknowledged: graceful.acknowledged, - runtimeReleaseConfirmed, - forceKilled: !hostProcessEnded && processStopped, - forced: force || options.signal?.aborted === true, - runtimeForceStopped: cleanup.runtimeForceStopped, - externalCleanupTimedOut: cleanup.timedOut, - }, - }); - if (!graceful.exited && !cleanup.runtimeForceStopped) { - quarantineAndroidSnapshotHelperRetirement({ - deviceKey, - packageName: session.helper.packageName, - cause: options.cause, - }); - } - return true; -} - -async function requestGracefulSessionExit( - session: AndroidSnapshotHelperSession, - processExit: AndroidSnapshotHelperProcessExit, - force: boolean, - signal: AbortSignal | undefined, -): Promise<{ - acknowledged: boolean; - /** The instrumentation this teardown asked to quit then finished on its own, cleanly. */ - exited: boolean; -}> { - if (force) return { acknowledged: false, exited: false }; - const requestId = `quit-${Date.now()}`; - try { - const response = await sendAndroidSnapshotHelperSessionCommand( - session.port, - `quit ${requestId}`, - SESSION_STOP_TIMEOUT_MS, - signal, - ); - const acknowledged = isAndroidSnapshotHelperSessionCommandAcknowledged(response, requestId); - const exited = - acknowledged && - (await waitForAndroidSnapshotHelperProcessExit( - processExit.ended, - SESSION_GRACEFUL_EXIT_TIMEOUT_MS, - signal, - )) && - processExit.completedCleanly(); - return { acknowledged, exited }; - } catch { - return { acknowledged: false, exited: false }; - } -} - -export async function stopAndroidSnapshotHelperSessionForDevice( - device: Pick, -): Promise { - await stopAndroidSnapshotHelperSession(getAndroidSnapshotHelperSessionDeviceKey(device)); -} - -export async function resetAndroidSnapshotHelperSessions(): Promise { - const retirements = await Promise.allSettled( - [...sessions.keys()].map((deviceKey) => stopAndroidSnapshotHelperSession(deviceKey)), - ); - disabledSessionIdentities.clear(); - const failures = retirements - .filter((result): result is PromiseRejectedResult => result.status === 'rejected') - .map((result) => result.reason); - if (failures.length > 0) { - throw new AggregateError(failures, 'Failed to retire every Android snapshot helper session'); - } - resetAndroidSnapshotHelperRetirements(); -} - -async function startAndroidSnapshotHelperSession(params: { - deviceKey: string; - identity: string; - options: AndroidSnapshotHelperCaptureOptions; - resolved: AndroidSnapshotHelperResolvedCaptureOptions; -}): Promise { - const port = await allocateAndroidSnapshotHelperSessionPort(); - await params.options.adb(['forward', `tcp:${port}`, `tcp:${port}`], { - allowFailure: false, - timeoutMs: FORWARD_TIMEOUT_MS, - signal: params.options.signal, - }); - const args = buildAndroidSnapshotHelperArgs({ - ...params.resolved, - outputPath: undefined, - emitChunks: false, - }); - const runner = args[args.length - 1]; - if (!runner) { - throw new AppError('INVALID_ARGS', 'Android snapshot helper runner was not resolved'); - } - const sessionArgs = [...args.slice(0, -1), '-e', 'sessionPort', String(port), runner]; - const process = params.options.adbProvider!.spawn!(sessionArgs, { - allowFailure: true, - captureOutput: false, - }); - const session: AndroidSnapshotHelperSession = { - identity: params.identity, - deviceKey: params.deviceKey, - helper: { - packageName: params.resolved.packageName, - runner: params.resolved.runner, - helperVersion: params.options.helperVersion, - helperVersionCode: params.options.helperVersionCode, - sha256: params.options.helperSha256, - }, - port, - adb: params.options.adb, - process, - startedAtMs: Date.now(), - capturedCount: 0, - }; - try { - await waitForAndroidSnapshotHelperSessionReady( - process, - SESSION_READY_TIMEOUT_MS, - params.options.signal, - ); - sessions.set(params.deviceKey, session); - emitDiagnostic({ - phase: 'android_snapshot_helper_session_ready', - data: { - deviceKey: params.deviceKey, - port, - packageName: params.resolved.packageName, - runner: params.resolved.runner, - }, - }); - return session; - } catch (error) { - const processExit = observeAndroidSnapshotHelperProcessExit(process); - try { - process.kill('SIGTERM'); - } catch { - // Best effort after startup failure. - } - const [, cleanup] = await Promise.all([ - waitForAndroidSnapshotHelperProcessExit( - processExit.ended, - ANDROID_SNAPSHOT_HELPER_HOST_PROCESS_EXIT_GRACE_MS, - ), - settleAndroidSnapshotHelperSessionCleanup({ - adb: session.adb, - process: session.process, - port: session.port, - packageName: session.helper.packageName, - timeoutMs: ANDROID_SNAPSHOT_HELPER_DEVICE_RETIREMENT_TIMEOUT_MS, - // Startup failed before the helper could acknowledge anything, so nothing proves it - // released UiAutomation. - forceStopRuntime: true, - }), - ]); - if (!cleanup.runtimeForceStopped) { - quarantineAndroidSnapshotHelperRetirement({ - deviceKey: params.deviceKey, - packageName: session.helper.packageName, - cause: error, - }); - } - throw error; - } -} - -function createSessionIdentity( - deviceKey: string, - resolved: AndroidSnapshotHelperResolvedCaptureOptions, - options: AndroidSnapshotHelperCaptureOptions, -): string { - const identity = JSON.stringify({ - deviceKey, - packageName: resolved.packageName, - runner: resolved.runner, - helperVersion: options.helperVersion, - helperVersionCode: options.helperVersionCode, - helperSha256: options.helperSha256, - waitForIdleTimeoutMs: resolved.waitForIdleTimeoutMs, - waitForIdleQuietMs: resolved.waitForIdleQuietMs, - timeoutMs: resolved.timeoutMs, - maxDepth: resolved.maxDepth, - maxNodes: resolved.maxNodes, - }); - return identity; -} - -function isAndroidSnapshotHelperSessionEnabled(): boolean { - const value = process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION; - return value === undefined || !/^(0|false|no|off)$/i.test(value); -} diff --git a/src/platforms/android/snapshot-helper.ts b/src/platforms/android/snapshot-helper.ts index 016a85e9cd..b7e365914e 100644 --- a/src/platforms/android/snapshot-helper.ts +++ b/src/platforms/android/snapshot-helper.ts @@ -1,13 +1,15 @@ export { parseAndroidSnapshotHelperManifest } from './snapshot-helper-artifact.ts'; export { captureAndroidSnapshotWithHelper } from './snapshot-helper-capture.ts'; +export { captureAndroidSnapshotWithHelperSession } from './snapshot-helper-session.ts'; export { - captureAndroidSnapshotWithHelperSession, - getAndroidSnapshotHelperSessionDeviceKey, - isAndroidSnapshotHelperRetirementUnconfirmedError, resetAndroidSnapshotHelperSessions, stopAndroidSnapshotHelperSession, stopAndroidSnapshotHelperSessionForDevice, -} from './snapshot-helper-session.ts'; +} from './snapshot-helper-session-lifecycle.ts'; +export { + getAndroidSnapshotHelperSessionDeviceKey, + isAndroidSnapshotHelperRetirementUnconfirmedError, +} from './snapshot-helper-retirement.ts'; export { ensureAndroidSnapshotHelper, forgetAndroidSnapshotHelperInstall, diff --git a/src/platforms/android/text-input.ts b/src/platforms/android/text-input.ts new file mode 100644 index 0000000000..e6ee633aba --- /dev/null +++ b/src/platforms/android/text-input.ts @@ -0,0 +1,350 @@ +/** + * How text reaches the focused Android field: the routing between provider-native injection, the + * bundled test IME, and the adb-shell fallback, plus the shell writer all three fall back to. + * Pointer, key, and gesture actions stay in `input-actions.ts`; what the field ended up holding is + * `fill-verification.ts`. + */ +import type { FillUnconfirmedVerification } from '@agent-device/contracts/interactor-types'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; +import { emitDiagnostic } from '../../utils/diagnostics.ts'; +import { shellQuoteIfNeeded } from '../../utils/shell-quote.ts'; +import { + resolveAndroidAdbExecutor, + resolveAndroidAdbProvider, + resolveAndroidTextInjector, + type AndroidTextInputAction, +} from './adb-executor.ts'; +import { runAndroidAdb, sleep } from './adb.ts'; +import { getAndroidKeyboardState, type AndroidKeyboardState } from './device-input-state.ts'; +import { + buildAndroidFillUnconfirmedVerification, + completeAndroidFillVerification, + readAndroidFillTargetBeforeMutation, + verifyAndroidFilledText, + type AndroidFillVerification, +} from './fill-verification.ts'; +import { + clearAndroidImeHelperText, + selectAndroidImeHelperArtifact, + sendAndroidImeHelperText, +} from './ime-helper.ts'; +import { isAndroidTestImeActive } from './ime-lifecycle.ts'; +import { focusAndroid } from './input-actions.ts'; +import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; + +const ANDROID_INPUT_TEXT_CHUNK_SIZE = 8; + +export async function typeAndroid(device: DeviceInfo, text: string, delayMs = 0): Promise { + const providerText = resolveAndroidTextInjector(device); + if (providerText) { + await providerText({ action: 'type', text, delayMs }); + emitAndroidTextDiagnostic('type', 'provider-native', text); + return; + } + if (isAndroidTestImeActive(device)) { + await typeAndroidTestIme(device, text, delayMs); + return; + } + assertAndroidShellTextSupported(text); + await assertAndroidShellInputIsAppOwned(device, 'type'); + if (delayMs > 0 && Array.from(text).length > 1) { + await typeAndroidShell(device, { action: 'type', text, chunkSize: 1, delayMs }); + return; + } + await typeAndroidShell(device, { + action: 'type', + text, + chunkSize: ANDROID_INPUT_TEXT_CHUNK_SIZE, + delayMs: 0, + }); +} + +export async function fillAndroid( + device: DeviceInfo, + x: number, + y: number, + text: string, + delayMs = 0, + helper: AndroidHelperSessionOptions = {}, +): Promise { + const beforeTarget = await readAndroidFillTargetBeforeMutation(device, x, y, helper); + const providerText = resolveAndroidTextInjector(device); + if (providerText) { + await providerText({ action: 'fill', target: { x, y }, text, delayMs }); + emitAndroidTextDiagnostic('fill', 'provider-native', text); + const verification = await verifyAndroidFilledText(device, x, y, text, helper); + return completeAndroidFillVerification(text, beforeTarget, verification); + } + if (isAndroidTestImeActive(device)) { + const verification = await fillAndroidTestIme(device, x, y, text, beforeTarget, helper); + return completeAndroidFillVerification(text, beforeTarget, verification); + } + assertAndroidShellTextSupported(text); + + const textCodePointLength = Array.from(text).length; + const attempts: Array<{ + clearPadding: number; + minClear: number; + maxClear: number; + chunkSize: number; + inputDelayMs: number; + }> = [ + { + clearPadding: 12, + minClear: 8, + maxClear: 48, + chunkSize: delayMs > 0 ? 1 : ANDROID_INPUT_TEXT_CHUNK_SIZE, + inputDelayMs: delayMs, + }, + { + clearPadding: 24, + minClear: 16, + maxClear: 96, + chunkSize: delayMs > 0 ? 1 : 4, + inputDelayMs: delayMs > 0 ? delayMs : 15, + }, + ]; + + let lastVerification: AndroidFillVerification | null = null; + + for (const attempt of attempts) { + await focusAndroid(device, x, y); + await assertAndroidShellInputIsAppOwned(device, 'fill'); + const clearCount = clampCount( + textCodePointLength + attempt.clearPadding, + attempt.minClear, + attempt.maxClear, + ); + await clearFocusedText(device, clearCount); + await typeAndroidShell(device, { + action: 'fill', + text, + chunkSize: attempt.chunkSize, + delayMs: attempt.inputDelayMs, + }); + const verification = await verifyAndroidFilledText(device, x, y, text, helper); + lastVerification = verification; + if (verification.ok) return; + if (verification.reason === 'ime_capture') { + return completeAndroidFillVerification(text, beforeTarget, verification); + } + const unconfirmed = buildAndroidFillUnconfirmedVerification(text, beforeTarget, verification); + if (unconfirmed) return unconfirmed; + } + + return completeAndroidFillVerification(text, beforeTarget, lastVerification); +} + +async function typeAndroidTestIme( + device: DeviceInfo, + text: string, + delayMs: number, +): Promise { + const adb = resolveAndroidAdbExecutor(device); + const artifact = await selectAndroidImeHelperArtifact(resolveAndroidAdbProvider(device)); + const packageName = artifact.manifest.packageName; + const parts = text.split('\n'); + for (const [partIndex, part] of parts.entries()) { + const chunks = delayMs > 0 ? chunkAndroidInputText(part, 1) : [part]; + for (const [chunkIndex, chunk] of chunks.entries()) { + if (chunk) await sendAndroidImeHelperText(adb, packageName, chunk); + if (delayMs > 0 && (chunkIndex + 1 < chunks.length || partIndex + 1 < parts.length)) { + await sleep(delayMs); + } + } + if (partIndex + 1 < parts.length) { + await runAndroidAdb(device, ['shell', 'input', 'keyevent', 'ENTER']); + } + } + emitAndroidTextDiagnostic('type', 'test-ime', text); +} + +async function fillAndroidTestIme( + device: DeviceInfo, + x: number, + y: number, + text: string, + beforeTarget: AndroidFillVerification['targetInput'], + helper: AndroidHelperSessionOptions, +): Promise { + const adb = resolveAndroidAdbExecutor(device); + const artifact = await selectAndroidImeHelperArtifact(resolveAndroidAdbProvider(device)); + const packageName = artifact.manifest.packageName; + let lastVerification: AndroidFillVerification | null = null; + // One retry covers the rare not-yet-bound InputConnection right after focus. + for (let attempt = 0; attempt < 2; attempt += 1) { + await focusAndroid(device, x, y); + await clearAndroidImeHelperText(adb, packageName); + if (text) await sendAndroidImeHelperText(adb, packageName, text); + const verification = await verifyAndroidFilledText(device, x, y, text, helper); + lastVerification = verification; + if (verification.ok) break; + if (buildAndroidFillUnconfirmedVerification(text, beforeTarget, verification)) break; + } + emitAndroidTextDiagnostic('fill', 'test-ime', text); + return lastVerification as AndroidFillVerification; +} + +async function typeAndroidShell( + device: DeviceInfo, + options: { action: AndroidTextInputAction; text: string; chunkSize: number; delayMs: number }, +): Promise { + const parts = options.text.split('\n'); + for (const [partIndex, part] of parts.entries()) { + const chunks = chunkAndroidInputText(part, options.chunkSize); + for (const [chunkIndex, chunk] of chunks.entries()) { + await typeAndroidShellChunk(device, chunk); + if (options.delayMs > 0 && (chunkIndex + 1 < chunks.length || partIndex + 1 < parts.length)) { + await sleep(options.delayMs); + } + } + if (partIndex + 1 < parts.length) { + await runAndroidAdb(device, ['shell', 'input', 'keyevent', 'ENTER']); + } + } + emitAndroidTextDiagnostic(options.action, 'adb-shell', options.text); +} + +async function typeAndroidShellChunk(device: DeviceInfo, text: string): Promise { + if (!text) return; + try { + await runAndroidAdb(device, [ + 'shell', + 'input', + 'text', + shellQuoteIfNeeded(encodeAndroidInputText(text)), + ]); + } catch (error) { + if (isAndroidInputTextUnsupported(error)) { + throw unsupportedAndroidShellTextError(text, error); + } + throw error; + } +} + +async function clearFocusedText(device: DeviceInfo, count: number): Promise { + const deletes = Math.max(0, count); + await runAndroidAdb(device, ['shell', 'input', 'keyevent', 'KEYCODE_MOVE_END'], { + allowFailure: true, + }); + const batchSize = 24; + for (let i = 0; i < deletes; i += batchSize) { + const size = Math.min(batchSize, deletes - i); + await runAndroidAdb( + device, + ['shell', 'input', 'keyevent', ...Array(size).fill('KEYCODE_DEL')], + { + allowFailure: true, + }, + ); + } +} + +async function assertAndroidShellInputIsAppOwned( + device: DeviceInfo, + action: AndroidTextInputAction, +): Promise { + let state: AndroidKeyboardState; + try { + state = await getAndroidKeyboardState(device); + } catch (error) { + emitDiagnostic({ + level: 'warn', + phase: 'android_input_ownership_probe_failed', + data: { + action, + error: error instanceof Error ? error.message : String(error), + }, + }); + return; + } + if (state.inputOwner !== 'ime') return; + throw new AppError( + 'COMMAND_FAILED', + 'KEYBOARD_OVERLAY_BLOCKING: Android text input is blocked because the focused input belongs to the active keyboard/IME.', + { + failureReason: 'ime_capture', + action, + inputOwner: state.inputOwner, + inputType: state.inputType, + type: state.type, + inputMethodPackage: state.inputMethodPackage, + focusedPackage: state.focusedPackage, + focusedResourceId: state.focusedResourceId, + nextAction: + 'Focused input appears to be owned by the keyboard/IME; dismiss or change the IME before retrying text entry.', + }, + ); +} + +function assertAndroidShellTextSupported(text: string): void { + if (isAndroidShellTextSupported(text)) return; + throw unsupportedAndroidShellTextError(text); +} + +function isAndroidShellTextSupported(text: string): boolean { + for (const char of text) { + const code = char.codePointAt(0); + if (code === undefined) continue; + if (char === '\n') continue; + if (code < 0x20 || code > 0x7e) { + return false; + } + } + return true; +} + +function encodeAndroidInputText(text: string): string { + // Android shell input uses `%s` as the escaped token for spaces. + return text.replace(/ /g, '%s'); +} + +function isAndroidInputTextUnsupported(error: unknown): boolean { + if (!(error instanceof AppError)) return false; + if (error.code !== 'COMMAND_FAILED') return false; + const rawStderr = error.details?.stderr; + const stderr = (typeof rawStderr === 'string' ? rawStderr : '').toLowerCase(); + if (stderr.includes("exception occurred while executing 'text'")) return true; + if (stderr.includes('nullpointerexception') && stderr.includes('inputshellcommand.sendtext')) + return true; + return false; +} + +function unsupportedAndroidShellTextError(text: string, cause?: unknown): AppError { + return new AppError( + 'COMMAND_FAILED', + 'Android text input requires provider-native text injection or the bundled test IME helper for non-ASCII/control characters; the adb-shell fallback supports ASCII text only. On emulators the test IME activates automatically; on real devices pass `open --test-ime` to enable it (see `agent-device doctor` for the current IME state).', + { + backend: 'adb-shell', + textLength: Array.from(text).length, + textPreview: text.slice(0, 32), + }, + cause instanceof Error ? cause : undefined, + ); +} + +function chunkAndroidInputText(text: string, chunkSize: number): string[] { + const size = Math.max(1, Math.floor(chunkSize)); + const chunks: string[] = []; + const chars = Array.from(text); + for (let i = 0; i < chars.length; i += size) { + chunks.push(chars.slice(i, i + size).join('')); + } + return chunks.length > 0 ? chunks : ['']; +} + +function emitAndroidTextDiagnostic( + action: AndroidTextInputAction, + backend: 'provider-native' | 'adb-shell' | 'test-ime', + text: string, +): void { + emitDiagnostic({ + phase: 'android_text_injection', + data: { action, backend, textLength: Array.from(text).length }, + }); +} + +function clampCount(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); +} diff --git a/src/platforms/android/touch-helper.ts b/src/platforms/android/touch-helper.ts index c810d30c53..e08d7a39e5 100644 --- a/src/platforms/android/touch-helper.ts +++ b/src/platforms/android/touch-helper.ts @@ -18,13 +18,15 @@ import type { AndroidLoweredTouchPlan } from './touch-plan.ts'; import { resolveAndroidHelperArtifact } from './helper-package-install.ts'; import { parseAndroidSnapshotHelperManifest } from './snapshot-helper-artifact.ts'; import { ensureAndroidSnapshotHelper } from './snapshot-helper-install.ts'; +import { runAndroidSnapshotHelperSessionTouchCommand } from './snapshot-helper-session.ts'; import { ensureAndroidSnapshotHelperSession, + stopAndroidSnapshotHelperSession, +} from './snapshot-helper-session-lifecycle.ts'; +import { getAndroidSnapshotHelperSessionDeviceKey, recoverAndroidSnapshotHelperRetirement, - runAndroidSnapshotHelperSessionTouchCommand, - stopAndroidSnapshotHelperSession, -} from './snapshot-helper-session.ts'; +} from './snapshot-helper-retirement.ts'; import { buildAndroidSnapshotHelperCaptureOptions } from './snapshot-helper-capture.ts'; import { ANDROID_SNAPSHOT_HELPER_PROTOCOL,