diff --git a/packages/contracts/package.json b/packages/contracts/package.json index d6c1663769..5563637f7f 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -132,6 +132,10 @@ "types": "./src/focus-runtime.ts", "default": "./src/focus-runtime.ts" }, + "./gesture-admission": { + "types": "./src/gesture-admission.ts", + "default": "./src/gesture-admission.ts" + }, "./gesture-input": { "types": "./src/gesture-input.ts", "default": "./src/gesture-input.ts" @@ -148,6 +152,10 @@ "types": "./src/gesture-plan-types.ts", "default": "./src/gesture-plan-types.ts" }, + "./gesture-runtime": { + "types": "./src/gesture-runtime.ts", + "default": "./src/gesture-runtime.ts" + }, "./home-runtime": { "types": "./src/home-runtime.ts", "default": "./src/home-runtime.ts" @@ -264,6 +272,10 @@ "types": "./src/scroll-gesture.ts", "default": "./src/scroll-gesture.ts" }, + "./scroll-runtime": { + "types": "./src/scroll-runtime.ts", + "default": "./src/scroll-runtime.ts" + }, "./selector-observation-runtime": { "types": "./src/selector-observation-runtime.ts", "default": "./src/selector-observation-runtime.ts" diff --git a/packages/contracts/src/apple-multitouch-support.ts b/packages/contracts/src/apple-multitouch-support.ts index d35f6b160e..81b6c5fc90 100644 --- a/packages/contracts/src/apple-multitouch-support.ts +++ b/packages/contracts/src/apple-multitouch-support.ts @@ -4,19 +4,11 @@ import { type AppleOS, type DeviceInfo, } from '@agent-device/kernel/device'; +import { APPLE_OS_DISPLAY_NAMES } from './apple-os-display-names.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { GesturePlan } from './gesture-plan-types.ts'; -const APPLE_OS_DISPLAY_NAMES: Record = { - ios: 'iOS', - ipados: 'iPadOS', - tvos: 'tvOS', - watchos: 'watchOS', - visionos: 'visionOS', - macos: 'macOS', -}; - -const APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS: Partial> = { +export const APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS: Partial> = { visionos: 'visionOS uses spatial input and does not support two-finger touch synthesis.', tvos: 'tvOS has no touch input — this gesture is supported on Android and the iOS simulator only.', macos: diff --git a/packages/contracts/src/apple-os-display-names.ts b/packages/contracts/src/apple-os-display-names.ts new file mode 100644 index 0000000000..eb680b0187 --- /dev/null +++ b/packages/contracts/src/apple-os-display-names.ts @@ -0,0 +1,18 @@ +import type { AppleOS } from '@agent-device/kernel/device'; + +/** + * How each Apple OS names itself in agent-facing prose. + * + * Its own module because two callers need it — the defensive adapter check in + * `apple-multitouch-support.ts` and the gesture refusal subject in `gesture-admission.ts` — and + * neither the display table nor the wording it produces is a public contracts surface. Keeping it + * out of a façade-re-exported module is what lets both callers share ONE copy of the wording. + */ +export const APPLE_OS_DISPLAY_NAMES: Record = { + ios: 'iOS', + ipados: 'iPadOS', + tvos: 'tvOS', + watchos: 'watchOS', + visionos: 'visionOS', + macos: 'macOS', +}; diff --git a/packages/contracts/src/facades/platform.ts b/packages/contracts/src/facades/platform.ts index 5c6281055f..d2c0444a4c 100644 --- a/packages/contracts/src/facades/platform.ts +++ b/packages/contracts/src/facades/platform.ts @@ -224,6 +224,11 @@ export { waitSelectorCaptureRuntimePlanUses, findRuntimePlanUses, focusRuntimeUse, + gestureRuntimePlanUses, + resolveGestureRuntimePlan, + resolveScrollRuntimePlan, + scrollRuntimePlanUses, + swipeRuntimePlanUses, typeTextRuntimeUse, viewportRuntimeUse, backRuntimeUse, @@ -236,7 +241,9 @@ export { keyboardEnterUse, } from '../platform-runtime-operations.ts'; export type { + GestureRuntimePlan, ScreenshotRuntimePlan, + ScrollRuntimePlan, SelectorCaptureRuntimeIntent, SelectorCaptureRuntimePlan, SnapshotRuntimePlan, @@ -431,6 +438,7 @@ export type { LocalKeyboardInteractorResolver, ProviderKeyboardInteractorResolver, } from '../keyboard-runtime.ts'; +export { APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS } from '../apple-multitouch-support.ts'; export { viewportRuntimeOperationFacts } from '../viewport-runtime.ts'; export type { SetViewportInput, diff --git a/packages/contracts/src/gesture-admission.ts b/packages/contracts/src/gesture-admission.ts new file mode 100644 index 0000000000..884368745b --- /dev/null +++ b/packages/contracts/src/gesture-admission.ts @@ -0,0 +1,65 @@ +import { + isApplePlatform, + resolveDeviceAppleOs, + type DeviceInfo, +} from '@agent-device/kernel/device'; +import { APPLE_OS_DISPLAY_NAMES } from './apple-os-display-names.ts'; +import type { GestureCommandInput } from './gesture-plan-types.ts'; +import type { GestureRuntimeTier } from './gesture-tier.ts'; + +/** The hint an owner states when it cannot preserve a target-authored drag's timing. */ +export const TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT = + 'Target-authored drag requires an adapter that preserves source hold, timed movement, and destination hold; it is supported on Android touch devices and iOS/iPadOS.'; + +/** The hint the Android owner states for a TV target, which has no touch input at all. */ +export const ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT = + 'Android TV has no touch input — this gesture is supported on Android phones, tablets, and the iOS simulator only.'; + +/** The hint the Apple owner states for a physical iOS/iPadOS device. */ +export const PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT = + 'Two-finger gesture synthesis is iOS-simulator only — not available on physical iOS devices.'; + +/** + * How a refused cell names itself, reproducing every subject the retired admission produced. + * + * Four owner-specific subjects, then the plain platform name. The special cases resolve the + * Apple OS the way `assertAppleMultiTouchSupported` does; the default reads `appleOs` raw, the way + * the retired `gesturePlatformMessage` did — an Apple device with no declared OS therefore still + * reports `apple`, exactly as before. + */ +function gestureRefusalSubject(device: DeviceInfo, tier: GestureRuntimeTier): string { + const owned = + tier === 'multi-touch' + ? multiTouchRefusalSubject(device) + : tier === 'directional-fling' && device.platform === 'linux' + ? 'Linux' + : undefined; + return owned ?? device.appleOs ?? device.platform; +} + +/** + * The three owner-specific subjects two-contact synthesis produced, or `undefined` where the + * retired admission fell through to the plain platform name. + */ +function multiTouchRefusalSubject(device: DeviceInfo): string | undefined { + if (device.platform === 'android') return device.target === 'tv' ? 'Android TV' : undefined; + if (!isApplePlatform(device.platform)) return undefined; + const appleOs = resolveDeviceAppleOs(device); + if (appleOs === 'ios' || appleOs === 'ipados') return 'physical iOS devices'; + if (appleOs === 'macos' || appleOs === 'tvos' || appleOs === 'visionos') { + return APPLE_OS_DISPLAY_NAMES[appleOs]; + } + return undefined; +} + +/** + * The refusal one unavailable gesture cell reports. `gesture fling` on Linux keeps its bare intent + * wording because its subject is the platform's display name, so no special-casing is needed here. + */ +export function gestureRefusalMessage( + device: DeviceInfo, + tier: GestureRuntimeTier, + intent: GestureCommandInput['intent'], +): string { + return `gesture ${intent} is not supported on ${gestureRefusalSubject(device, tier)}`; +} diff --git a/packages/contracts/src/gesture-runtime.test.ts b/packages/contracts/src/gesture-runtime.test.ts new file mode 100644 index 0000000000..edec2d87e2 --- /dev/null +++ b/packages/contracts/src/gesture-runtime.test.ts @@ -0,0 +1,215 @@ +import { expect, test, vi } from 'vitest'; +import { + bindLocalGestureInteractor, + bindProviderGestureInteractor, + gestureRuntimeOperationFacts, +} from './gesture-runtime.ts'; +import type { GesturePlan } from './gesture-plan-types.ts'; +import type { Interactor } from './interactor-types.ts'; + +const device = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, +} as const; + +const available = { available: true } as const; +const unavailable = { available: false, reason: 'unsupported-platform-leaf' } as const; + +const allAvailable = gestureRuntimeOperationFacts({ + plan: available, + directionalFling: available, + multiTouch: available, + targetAuthoredDrag: available, + viewport: available, +}); + +const plan: GesturePlan = { + topology: 'single', + intent: 'pan', + executionProfile: 'timed-pan', + durationMs: 300, + viewport: { x: 0, y: 0, width: 400, height: 800 }, + pointers: [ + { + pointerId: 0, + samples: [ + { offsetMs: 0, point: { x: 10, y: 20 } }, + { offsetMs: 300, point: { x: 10, y: 220 } }, + ], + }, + ], +}; + +test('builds the exact gesture operation fact catalog', () => { + expect( + gestureRuntimeOperationFacts({ + plan: available, + directionalFling: unavailable, + multiTouch: unavailable, + targetAuthoredDrag: available, + viewport: unavailable, + }), + ).toEqual({ + performGesturePlan: available, + performDirectionalFlingPlan: unavailable, + performMultiTouchGesturePlan: unavailable, + performTargetAuthoredDrag: available, + gestureViewport: unavailable, + }); +}); + +test('a local binding executes the plan through the owner interactor', async () => { + const performGesture = vi.fn(async () => ({ backend: 'adb' })); + const resolveInteractor = vi.fn(async () => ({ performGesture }) as unknown as Interactor); + const signal = new AbortController().signal; + + const operations = bindLocalGestureInteractor({ + device, + signal, + facts: allAvailable, + resolveInteractor, + }); + await operations.performGesturePlan?.({ + plan, + options: { appBundleId: 'com.example.app' }, + execution: { logPath: '/tmp/daemon.log', requestId: 'gesture-1' }, + }); + + expect(resolveInteractor).toHaveBeenCalledWith(device, { + logPath: '/tmp/daemon.log', + requestId: 'gesture-1', + appBundleId: 'com.example.app', + signal, + }); + // The plan reaches the seam whole and unmodified — this is the sole argument, so an executor + // that dropped or rebuilt it shows up here. + expect(performGesture).toHaveBeenCalledWith(plan); +}); + +test('every admitted tier reaches the same single plan executor', async () => { + const performGesture = vi.fn(async () => ({})); + const operations = bindLocalGestureInteractor({ + device, + signal: new AbortController().signal, + facts: allAvailable, + resolveInteractor: async () => ({ performGesture }) as unknown as Interactor, + }); + + await operations.performDirectionalFlingPlan?.({ plan }); + await operations.performMultiTouchGesturePlan?.({ plan }); + await operations.performTargetAuthoredDrag?.({ plan }); + + expect(performGesture).toHaveBeenCalledTimes(3); +}); + +test('a binding exposes only the tiers its owner facts admitted', () => { + const operations = bindLocalGestureInteractor({ + device, + signal: new AbortController().signal, + facts: gestureRuntimeOperationFacts({ + plan: available, + directionalFling: unavailable, + multiTouch: unavailable, + targetAuthoredDrag: unavailable, + viewport: unavailable, + }), + resolveInteractor: async () => ({ performGesture: async () => ({}) }) as unknown as Interactor, + }); + + expect(operations.performGesturePlan).toBeTypeOf('function'); + expect(operations.performDirectionalFlingPlan).toBeUndefined(); + expect(operations.performMultiTouchGesturePlan).toBeUndefined(); + expect(operations.performTargetAuthoredDrag).toBeUndefined(); + expect(operations.gestureViewport).toBeUndefined(); +}); + +test('a local binding reads the owner frame through its interactor', async () => { + const gestureViewport = vi.fn(async () => ({ x: 0, y: 0, width: 393, height: 852 })); + const resolveInteractor = vi.fn(async () => ({ gestureViewport }) as unknown as Interactor); + const signal = new AbortController().signal; + + const operations = bindLocalGestureInteractor({ + device, + signal, + facts: allAvailable, + resolveInteractor, + }); + + await expect( + operations.gestureViewport?.({ execution: { requestId: 'gesture-2' } }), + ).resolves.toEqual({ x: 0, y: 0, width: 393, height: 852 }); + expect(resolveInteractor).toHaveBeenCalledWith(device, { + requestId: 'gesture-2', + appBundleId: undefined, + signal, + }); +}); + +test('a provider binding executes through its own resolved interactor', async () => { + const performGesture = vi.fn(async () => ({})); + const resolveInteractor = vi.fn(() => ({ performGesture }) as unknown as Interactor); + const signal = new AbortController().signal; + + const operations = bindProviderGestureInteractor({ + device, + signal, + facts: allAvailable, + resolveInteractor, + }); + await operations.performGesturePlan?.({ plan, execution: { requestId: 'gesture-3' } }); + + expect(resolveInteractor).toHaveBeenCalledWith({ + requestId: 'gesture-3', + appBundleId: undefined, + signal, + }); + expect(performGesture).toHaveBeenCalledWith(plan); +}); + +test('a provider binding fails closed when its exact owner exposes no interactor', async () => { + const operations = bindProviderGestureInteractor({ + device, + signal: new AbortController().signal, + facts: allAvailable, + resolveInteractor: () => undefined, + }); + + await expect(operations.performGesturePlan?.({ plan })).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'provider-runtime-interactor-missing', deviceId: device.id }, + }); +}); + +test('an advertised tier whose interactor cannot execute is a contract bug, not a refusal', async () => { + const operations = bindLocalGestureInteractor({ + device, + signal: new AbortController().signal, + facts: allAvailable, + resolveInteractor: async () => ({}) as unknown as Interactor, + }); + + await expect(operations.performGesturePlan?.({ plan })).rejects.toMatchObject({ + message: expect.stringContaining('advertised gesture execution'), + }); +}); + +test('an already-cancelled request never resolves an interactor', async () => { + const controller = new AbortController(); + controller.abort(); + const performGesture = vi.fn(async () => ({})); + const resolveInteractor = vi.fn(async () => ({ performGesture }) as unknown as Interactor); + + const operations = bindLocalGestureInteractor({ + device, + signal: controller.signal, + facts: allAvailable, + resolveInteractor, + }); + + await expect(operations.performGesturePlan?.({ plan })).rejects.toThrow(); + expect(resolveInteractor).not.toHaveBeenCalled(); + expect(performGesture).not.toHaveBeenCalled(); +}); diff --git a/packages/contracts/src/gesture-runtime.ts b/packages/contracts/src/gesture-runtime.ts new file mode 100644 index 0000000000..eded5e3225 --- /dev/null +++ b/packages/contracts/src/gesture-runtime.ts @@ -0,0 +1,186 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import type { GesturePlan } from './gesture-plan-types.ts'; +import { + localInteractorSource, + providerInteractorSource, + type LocalInteractorOperationResolver, + type ProviderInteractorOperationResolver, +} from './interactor-operation-binding.ts'; +import type { Interactor, RunnerContext } from './interactor-types.ts'; +import type { RuntimeOperationFact } from './platform-runtime.ts'; +import { invalidRuntimeContract } from './runtime-contract-error.ts'; +import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; + +/** + * Neutral intent for executing one typed gesture plan (ADR 0013). The plan is already built — + * from the coordinates, preset, or resolved drag targets a caller normalized — so the operation + * names no command, request, session, or CLI flag. + */ +export type GesturePlanInput = Readonly<{ + plan: GesturePlan; + options?: Readonly<{ appBundleId?: string }>; + /** Same runner metadata a capture needs; reuses that type rather than restating it. */ + execution?: SnapshotRuntimeExecution; +}>; + +/** Reading the gesture coordinate frame needs no plan — only the owner's authority. */ +export type GestureViewportInput = Readonly<{ + options?: Readonly<{ appBundleId?: string }>; + execution?: SnapshotRuntimeExecution; +}>; + +/** + * The gesture family's execution surface. + * + * The four plan operations run the SAME mechanics — one `Interactor.performGesture` call — and + * are separate keys because their **cells** differ, not their implementations. That is the shape + * `SnapshotRuntimeOperations` already uses for its three capture keys: an owner declares each + * requirement it can actually meet, and a command requires exactly the tier its input selected, + * so a device is refused where the retired `requireGestureSupported` refused it instead of + * failing mid-execution. + * + * The tiers, and the retired admission each one restates: + * - `performGesturePlan` — one-contact fling/pan, and every `swipe`. Refused where the legacy + * check refused a plain gesture (web, watchOS, visionOS). + * - `performDirectionalFlingPlan` — `gesture fling --direction`, whose speed semantics Linux + * cannot honor even though it executes coordinate flings through its drag primitive. + * - `performMultiTouchGesturePlan` — pinch/rotate/transform and two-pointer pan. On Apple this is + * the two-finger XCTest synthesis, which is iOS/iPadOS **simulator** only. + * - `performTargetAuthoredDrag` — `gesture drag`, which needs an adapter preserving source hold, + * timed movement, and destination hold. + */ +export type GestureRuntimeOperations = Readonly<{ + performGesturePlan(input: GesturePlanInput): Promise | void>; + performDirectionalFlingPlan(input: GesturePlanInput): Promise | void>; + performMultiTouchGesturePlan(input: GesturePlanInput): Promise | void>; + performTargetAuthoredDrag(input: GesturePlanInput): Promise | void>; + /** + * The owner's own gesture coordinate frame. Callers without it derive the frame from their + * admitted snapshot capture. + */ + gestureViewport(input: GestureViewportInput): Promise; +}>; + +export type GestureRuntimeOperationFacts = Readonly<{ + performGesturePlan: RuntimeOperationFact; + performDirectionalFlingPlan: RuntimeOperationFact; + performMultiTouchGesturePlan: RuntimeOperationFact; + performTargetAuthoredDrag: RuntimeOperationFact; + gestureViewport: RuntimeOperationFact; +}>; + +/** Builds the exhaustive owner claims for the five gesture requirements. */ +export function gestureRuntimeOperationFacts( + input: Readonly<{ + plan: RuntimeOperationFact; + directionalFling: RuntimeOperationFact; + multiTouch: RuntimeOperationFact; + targetAuthoredDrag: RuntimeOperationFact; + viewport: RuntimeOperationFact; + }>, +): GestureRuntimeOperationFacts { + return Object.freeze({ + performGesturePlan: input.plan, + performDirectionalFlingPlan: input.directionalFling, + performMultiTouchGesturePlan: input.multiTouch, + performTargetAuthoredDrag: input.targetAuthoredDrag, + gestureViewport: input.viewport, + }); +} + +/** + * Captures one selected owner's interactor authority for the lifetime of a request binding, and + * exposes only the tiers that owner's own facts admitted. + * + * The per-tier gating lives HERE rather than in each of the eight runtime owners: the tiers share + * one executor, so eight copies of the same five-branch spread would be duplication of mechanism + * — and the next tier added would cost eight more edits. + */ +function bindGestureOperations( + facts: GestureRuntimeOperationFacts, + signal: AbortSignal, + resolveInteractor: (runner: RunnerContext) => Promise, +): Partial { + const performPlan = async (input: GesturePlanInput) => { + const interactor = await resolveGestureInteractor(signal, resolveInteractor, input); + // Facts advertised gesture execution but the owner's interactor cannot perform it. That is a + // contract violation, not a refusal (ADR 0019 §2): degrading here would execute nothing and + // report success. + if (typeof interactor.performGesture !== 'function') { + throw invalidRuntimeContract( + 'Runtime owner advertised gesture execution without an interactor implementation', + ); + } + return await interactor.performGesture(input.plan); + }; + return Object.freeze({ + ...(facts.performGesturePlan.available ? { performGesturePlan: performPlan } : {}), + ...(facts.performDirectionalFlingPlan.available + ? { performDirectionalFlingPlan: performPlan } + : {}), + ...(facts.performMultiTouchGesturePlan.available + ? { performMultiTouchGesturePlan: performPlan } + : {}), + ...(facts.performTargetAuthoredDrag.available + ? { performTargetAuthoredDrag: performPlan } + : {}), + ...(facts.gestureViewport.available + ? { + gestureViewport: async (input: GestureViewportInput) => { + const interactor = await resolveGestureInteractor(signal, resolveInteractor, input); + if (typeof interactor.gestureViewport !== 'function') { + throw invalidRuntimeContract( + 'Runtime owner advertised gestureViewport without an interactor implementation', + ); + } + return await interactor.gestureViewport(); + }, + } + : {}), + }); +} + +async function resolveGestureInteractor( + signal: AbortSignal, + resolveInteractor: (runner: RunnerContext) => Promise, + input: GesturePlanInput | GestureViewportInput, +): Promise { + signal.throwIfAborted(); + return await resolveInteractor({ + ...input.execution, + appBundleId: input.options?.appBundleId, + signal, + }); +} + +export type LocalGestureInteractorResolver = LocalInteractorOperationResolver; + +export function bindLocalGestureInteractor( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + facts: GestureRuntimeOperationFacts; + resolveInteractor: LocalGestureInteractorResolver; + }>, +): Partial { + return bindGestureOperations(params.facts, params.signal, localInteractorSource(params)); +} + +export type ProviderGestureInteractorResolver = ProviderInteractorOperationResolver; + +/** Provider bindings fail closed when their exact owner no longer exposes its interactor. */ +export function bindProviderGestureInteractor( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + facts: GestureRuntimeOperationFacts; + resolveInteractor: ProviderGestureInteractorResolver; + }>, +): Partial { + return bindGestureOperations( + params.facts, + params.signal, + providerInteractorSource({ ...params, operation: 'gesture' }), + ); +} diff --git a/packages/contracts/src/gesture-tier.ts b/packages/contracts/src/gesture-tier.ts new file mode 100644 index 0000000000..d19e850735 --- /dev/null +++ b/packages/contracts/src/gesture-tier.ts @@ -0,0 +1,13 @@ +/** + * Which execution tier one gesture input needs. The tier comes from the input alone — never from + * the device — because a tier's *availability* is what varies by owner, and mixing the two is how + * the retired `requireGestureSupported` ended up owning a platform table inside the daemon. + * + * Type-only on purpose: the classifier that produces it lives beside the use catalog that consumes + * it, so naming a tier costs an importer no eager module evaluation. + */ +export type GestureRuntimeTier = + | 'plan' + | 'directional-fling' + | 'multi-touch' + | 'target-authored-drag'; diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index 4659eea310..43aaa3e1f2 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -17,6 +17,10 @@ import type { SnapshotRuntimeHost, SnapshotRuntimeOperations } from './snapshot- import type { SelectorObservationRuntimeOperations } from './selector-observation-runtime.ts'; import type { ViewportRuntimeOperations } from './viewport-runtime.ts'; import type { FocusRuntimeOperations } from './focus-runtime.ts'; +import type { GestureCommandInput, GestureSemanticInput } from './gesture-plan-types.ts'; +import type { GestureRuntimeTier } from './gesture-tier.ts'; +import type { GestureRuntimeOperations } from './gesture-runtime.ts'; +import type { ScrollRuntimeOperations } from './scroll-runtime.ts'; import type { TypeTextRuntimeOperations } from './type-text-runtime.ts'; import type { ElementTextRuntimeOperations } from './element-text-runtime.ts'; import type { BackRuntimeOperations } from './back-runtime.ts'; @@ -59,6 +63,8 @@ export type PlatformRuntimeOperations = AppLogRuntimeOperations & SelectorObservationRuntimeOperations & ViewportRuntimeOperations & FocusRuntimeOperations & + GestureRuntimeOperations & + ScrollRuntimeOperations & TypeTextRuntimeOperations & ElementTextRuntimeOperations & BackRuntimeOperations & @@ -168,6 +174,133 @@ export function resolveTouchRuntimePlan( : { kind: 'hover-point', use: hoverPointUse }; } } + +/** + * The gesture family's action-selected uses (ADR 0019 §9: one bind per handler). A gesture input + * needs exactly one execution tier, so each tier carries the complete requirement set and the + * handler binds once — for the whole `swipe --count N` series as much as for one `gesture pinch`. + * + * Every tier binds snapshot capture with its action because capture is the fallback when the owner + * has no direct viewport read. `gestureViewport` remains preferred. + */ +const gesturePlanUse = defineUse({ + required: ['performGesturePlan', 'captureSnapshot'], + preferred: ['gestureViewport'], +}); +const gestureDirectionalFlingUse = defineUse({ + required: ['performDirectionalFlingPlan', 'captureSnapshot'], + preferred: ['gestureViewport'], +}); +const gestureMultiTouchUse = defineUse({ + required: ['performMultiTouchGesturePlan', 'captureSnapshot'], + preferred: ['gestureViewport'], +}); +const gestureTargetAuthoredDragUse = defineUse({ + required: ['performTargetAuthoredDrag', 'captureSnapshot'], + preferred: ['gestureViewport'], +}); + +/** `scroll ` executes one pass and needs nothing else. */ +const scrollDirectionUse = defineUse({ required: ['scrollDirection'] }); +/** + * `scroll top` / `scroll bottom` verify hidden content between passes, so the capture is part of + * the tier's requirement rather than something discovered mid-run — the retired leaf's + * "requires snapshot support to verify hidden content before scrolling" refusal, moved to + * admission. + */ +const scrollEdgeUse = defineUse({ required: ['scrollDirection', 'captureSnapshot'] }); + +const gestureUsesByTier = Object.freeze({ + plan: gesturePlanUse, + 'directional-fling': gestureDirectionalFlingUse, + 'multi-touch': gestureMultiTouchUse, + 'target-authored-drag': gestureTargetAuthoredDragUse, +} as const); + +/** Every use `gesture` can select between; the descriptor declares the whole set. */ +export const gestureRuntimePlanUses = Object.freeze([ + gesturePlanUse, + gestureDirectionalFlingUse, + gestureMultiTouchUse, + gestureTargetAuthoredDragUse, +] as const); + +/** Public `swipe` always normalizes to a one-contact coordinate fling. */ +export const swipeRuntimePlanUses = Object.freeze([gesturePlanUse] as const); + +/** Every use `scroll` can select between. */ +export const scrollRuntimePlanUses = Object.freeze([scrollDirectionUse, scrollEdgeUse] as const); + +type GesturePlanFor = Readonly<{ + tier: Tier; + operation: GestureTierOperation; + use: (typeof gestureUsesByTier)[Tier]; +}>; + +type GestureTierOperation = + (typeof gestureUsesByTier)[Tier]['required'][0]; + +export type GestureRuntimePlan = { + [Tier in GestureRuntimeTier]: GesturePlanFor; +}[GestureRuntimeTier]; + +/** Selects the one owner-fact-backed gesture plan a normalized gesture input needs. */ +/** Two contacts are what pinch, rotate, transform, and an explicit two-pointer pan all need. */ +function isMultiTouchGesture(input: GestureSemanticInput): boolean { + if (input.intent === 'pan') return ('pointerCount' in input ? input.pointerCount : 1) === 2; + return input.intent === 'pinch' || input.intent === 'rotate' || input.intent === 'transform'; +} + +/** Selects the one tier a gesture input needs, so its handler binds exactly once (ADR 0019 §9). */ +function gestureRuntimeTier(input: GestureCommandInput): GestureRuntimeTier { + if (input.intent === 'drag') return 'target-authored-drag'; + if (isMultiTouchGesture(input)) return 'multi-touch'; + // A direction-authored fling carries speed semantics a coordinate fling does not, which is the + // one thing the Linux drag primitive cannot reproduce. + if (input.intent === 'fling' && 'direction' in input) return 'directional-fling'; + return 'plan'; +} + +export function resolveGestureRuntimePlan(input: GestureCommandInput): GestureRuntimePlan { + const tier = gestureRuntimeTier(input); + switch (tier) { + case 'plan': + return gesturePlan(tier); + case 'directional-fling': + return gesturePlan(tier); + case 'multi-touch': + return gesturePlan(tier); + case 'target-authored-drag': + return gesturePlan(tier); + } +} + +function gesturePlan(tier: Tier): GesturePlanFor { + const use = gestureUsesByTier[tier]; + return Object.freeze({ + tier, + operation: use.required[0] as GestureTierOperation, + use, + }); +} + +/** + * The edge travels WITH the discriminant so a caller that narrows to `edge` also has the edge + * itself — that is what lets its execution closure be typed on a binding whose `captureSnapshot` + * is non-optional, instead of widening both branches and re-proving the capture at runtime. + */ +export type ScrollRuntimePlan = + | Readonly<{ kind: 'direction'; use: typeof scrollDirectionUse }> + | Readonly<{ kind: 'edge'; edge: 'top' | 'bottom'; use: typeof scrollEdgeUse }>; + +/** `scroll top`/`scroll bottom` verify between passes; every other scroll executes one pass. */ +export function resolveScrollRuntimePlan( + input: Readonly<{ edge?: 'top' | 'bottom' }>, +): ScrollRuntimePlan { + return input.edge === undefined + ? Object.freeze({ kind: 'direction', use: scrollDirectionUse } as const) + : Object.freeze({ kind: 'edge', edge: input.edge, use: scrollEdgeUse } as const); +} const captureSnapshotWithCustomActionsUse = defineUse({ required: ['captureSnapshot', 'captureSnapshotWithCustomActions'], }); diff --git a/packages/contracts/src/platform-runtime-unavailable.test.ts b/packages/contracts/src/platform-runtime-unavailable.test.ts index 0dad51f95f..ad023a0710 100644 --- a/packages/contracts/src/platform-runtime-unavailable.test.ts +++ b/packages/contracts/src/platform-runtime-unavailable.test.ts @@ -32,6 +32,8 @@ test('generic unavailable binding preserves exact provider ownership and mode', screenshot: { available: false, reason: 'unsupported-device-kind' }, viewport: { available: false, reason: 'unsupported-platform-leaf' }, focus: { available: false, reason: 'unsupported-provider-mode' }, + gesture: { available: false, reason: 'unsupported-provider-mode' }, + scroll: { available: false, reason: 'unsupported-provider-mode' }, typeText: { available: false, reason: 'unsupported-provider-mode' }, touch: { available: false, reason: 'unsupported-provider-mode' }, elementText: { available: false, reason: 'unsupported-provider-mode' }, diff --git a/packages/contracts/src/platform-runtime-unavailable.ts b/packages/contracts/src/platform-runtime-unavailable.ts index 5eeca9c82f..88f8b3ad9f 100644 --- a/packages/contracts/src/platform-runtime-unavailable.ts +++ b/packages/contracts/src/platform-runtime-unavailable.ts @@ -15,6 +15,8 @@ import { snapshotRuntimeOperationFacts } from './snapshot-runtime.ts'; import { selectorObservationRuntimeOperationFacts } from './selector-observation-runtime.ts'; import { viewportRuntimeOperationFacts } from './viewport-runtime.ts'; import { focusRuntimeOperationFacts } from './focus-runtime.ts'; +import { gestureRuntimeOperationFacts } from './gesture-runtime.ts'; +import { scrollRuntimeOperationFacts } from './scroll-runtime.ts'; import { typeTextRuntimeOperationFacts } from './type-text-runtime.ts'; import { elementTextRuntimeOperationFacts } from './element-text-runtime.ts'; import { backRuntimeOperationFacts } from './back-runtime.ts'; @@ -39,6 +41,8 @@ export type UnavailablePlatformRuntimeFacts = Readonly<{ snapshot?: RuntimeOperationUnavailability; viewport: RuntimeOperationUnavailability; focus: RuntimeOperationUnavailability; + gesture: RuntimeOperationUnavailability; + scroll: RuntimeOperationUnavailability; typeText: RuntimeOperationUnavailability; touch: RuntimeOperationUnavailability; elementText: RuntimeOperationUnavailability; @@ -93,6 +97,8 @@ export function createUnavailablePlatformRuntimeFacts( snapshot, viewport, focus, + gesture, + scroll, typeText, touch, elementText, @@ -143,6 +149,14 @@ export function createUnavailablePlatformRuntimeFacts( }), ...viewportRuntimeOperationFacts({ setViewport: viewport }), ...focusRuntimeOperationFacts({ focus }), + ...gestureRuntimeOperationFacts({ + plan: gesture, + directionalFling: gesture, + multiTouch: gesture, + targetAuthoredDrag: gesture, + viewport: gesture, + }), + ...scrollRuntimeOperationFacts({ scroll }), ...typeTextRuntimeOperationFacts({ type: typeText }), ...touchRuntimeOperationFacts({ tap: touch, @@ -194,6 +208,8 @@ function freezeUnavailableFacts( // Interaction cells are stated by their owner: a family that can drive touch says so for its // exact kinds, and one that cannot must say why rather than inherit a transport gap. focus: Object.freeze({ ...unavailable.focus }), + gesture: Object.freeze({ ...unavailable.gesture }), + scroll: Object.freeze({ ...unavailable.scroll }), typeText: Object.freeze({ ...unavailable.typeText }), touch: Object.freeze({ ...unavailable.touch }), readiness: orNetwork(unavailable.readiness), diff --git a/packages/contracts/src/platform-runtime.ts b/packages/contracts/src/platform-runtime.ts index a9605764a8..549a82d104 100644 --- a/packages/contracts/src/platform-runtime.ts +++ b/packages/contracts/src/platform-runtime.ts @@ -110,6 +110,18 @@ export type RuntimeOperationUnavailability = Readonly<{ export type RuntimeOperationFact = Readonly<{ available: true }> | RuntimeOperationUnavailability; +/** + * An operation is present on a binding only when the owner's own facts admitted it. One helper so + * every owner's `bind` reads as a list of admitted operations rather than a chain of branches — + * and so the next operation added there costs no additional branch. + */ +export function whenAdmitted( + fact: RuntimeOperationFact, + build: () => T, +): T | Record { + return fact.available ? build() : {}; +} + export type RuntimeFacts = Readonly<{ device: RuntimeDeviceShape; operations: Readonly<{ @@ -313,11 +325,3 @@ function unsupportedRuntimeOperation(key: string, fact: RuntimeOperationUnavaila hint: fact.hint, }); } - -/** Include an operation binding only when the owning runtime's published fact admits it. */ -export function whenAdmitted( - fact: RuntimeOperationFact, - build: () => T, -): T | Record { - return fact.available ? build() : {}; -} diff --git a/packages/contracts/src/scroll-runtime.test.ts b/packages/contracts/src/scroll-runtime.test.ts new file mode 100644 index 0000000000..4b28983887 --- /dev/null +++ b/packages/contracts/src/scroll-runtime.test.ts @@ -0,0 +1,103 @@ +import { expect, test, vi } from 'vitest'; +import { + bindLocalScrollInteractor, + bindProviderScrollInteractor, + scrollRuntimeOperationFacts, +} from './scroll-runtime.ts'; +import type { Interactor } from './interactor-types.ts'; + +const device = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, +} as const; + +const options = { + amount: 0.55, + pixels: undefined, + durationMs: undefined, + releaseBehavior: 'controlled', +} as const; + +test('builds the exact scroll operation fact catalog', () => { + const scroll = { available: true } as const; + expect(scrollRuntimeOperationFacts({ scroll })).toEqual({ scrollDirection: scroll }); +}); + +test('a local binding scrolls the owner in the requested direction', async () => { + const scroll = vi.fn(async () => ({ pixels: 240 })); + const resolveInteractor = vi.fn(async () => ({ scroll }) as unknown as Interactor); + const signal = new AbortController().signal; + + const operations = bindLocalScrollInteractor({ device, signal, resolveInteractor }); + await expect( + operations.scrollDirection({ + direction: 'down', + options, + target: { appBundleId: 'com.example.app' }, + execution: { logPath: '/tmp/daemon.log', requestId: 'scroll-1' }, + }), + ).resolves.toEqual({ pixels: 240 }); + + expect(resolveInteractor).toHaveBeenCalledWith(device, { + logPath: '/tmp/daemon.log', + requestId: 'scroll-1', + appBundleId: 'com.example.app', + signal, + }); + // Positional (direction, options): the `Interactor` seam takes them in that order, and + // transposing them is the one swap an object-shaped assertion would not catch. + expect(scroll).toHaveBeenCalledWith('down', options); +}); + +test('a provider binding scrolls through its own resolved interactor', async () => { + const scroll = vi.fn(async () => undefined); + const resolveInteractor = vi.fn(() => ({ scroll }) as unknown as Interactor); + const signal = new AbortController().signal; + + const operations = bindProviderScrollInteractor({ device, signal, resolveInteractor }); + await operations.scrollDirection({ + direction: 'up', + options, + execution: { requestId: 'scroll-2' }, + }); + + expect(resolveInteractor).toHaveBeenCalledWith({ + requestId: 'scroll-2', + appBundleId: undefined, + signal, + }); + expect(scroll).toHaveBeenCalledWith('up', options); +}); + +test('a provider binding fails closed when its exact owner exposes no interactor', async () => { + const operations = bindProviderScrollInteractor({ + device, + signal: new AbortController().signal, + resolveInteractor: () => undefined, + }); + + await expect(operations.scrollDirection({ direction: 'down', options })).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'provider-runtime-interactor-missing', deviceId: device.id }, + }); +}); + +test('an already-cancelled request never resolves an interactor', async () => { + const controller = new AbortController(); + controller.abort(); + const scroll = vi.fn(async () => undefined); + const resolveInteractor = vi.fn(async () => ({ scroll }) as unknown as Interactor); + + const operations = bindLocalScrollInteractor({ + device, + signal: controller.signal, + resolveInteractor, + }); + + await expect(operations.scrollDirection({ direction: 'down', options })).rejects.toThrow(); + expect(resolveInteractor).not.toHaveBeenCalled(); + expect(scroll).not.toHaveBeenCalled(); +}); diff --git a/packages/contracts/src/scroll-runtime.ts b/packages/contracts/src/scroll-runtime.ts new file mode 100644 index 0000000000..43a363a096 --- /dev/null +++ b/packages/contracts/src/scroll-runtime.ts @@ -0,0 +1,94 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { + localInteractorSource, + providerInteractorSource, + type LocalInteractorOperationResolver, + type ProviderInteractorOperationResolver, +} from './interactor-operation-binding.ts'; +import type { Interactor, RunnerContext } from './interactor-types.ts'; +import type { RuntimeOperationFact } from './platform-runtime.ts'; +import type { ResolvedScrollExecutionOptions } from './scroll-command.ts'; +import type { ScrollDirection } from './scroll-gesture.ts'; +import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; + +/** + * Neutral intent for one directional scroll. Distance, timing, and release behavior are already + * resolved by the caller (`resolveScrollExecutionOptions`), so the owner receives them as data + * and the operation names no command, request, session, or CLI flag. + */ +export type ScrollDirectionInput = Readonly<{ + direction: ScrollDirection; + options: ResolvedScrollExecutionOptions; + target?: Readonly<{ appBundleId?: string }>; + /** Same runner metadata a capture needs; reuses that type rather than restating it. */ + execution?: SnapshotRuntimeExecution; +}>; + +/** + * One scroll pass. `scroll top` / `scroll bottom` run several, verifying between passes with the + * capture operation they additionally require — the platform-visible unit is still a single pass, + * so edge repetition stays caller-side policy rather than a second operation. + */ +export type ScrollRuntimeOperations = Readonly<{ + scrollDirection(input: ScrollDirectionInput): Promise | void>; +}>; + +export type ScrollRuntimeOperationFacts = Readonly<{ + scrollDirection: RuntimeOperationFact; +}>; + +export function scrollRuntimeOperationFacts( + input: Readonly<{ scroll: RuntimeOperationFact }>, +): ScrollRuntimeOperationFacts { + return Object.freeze({ scrollDirection: input.scroll }); +} + +/** + * Captures one selected owner's interactor authority for the lifetime of a request binding. The + * owner is already chosen by the time a binder is called, so each entry point supplies its own + * resolution and this holds only what both share: the runner context and the scroll itself. + */ +function bindScrollDirection( + signal: AbortSignal, + resolveInteractor: (runner: RunnerContext) => Promise, +): ScrollRuntimeOperations { + return Object.freeze({ + scrollDirection: async (input: ScrollDirectionInput) => { + signal.throwIfAborted(); + const interactor = await resolveInteractor({ + ...input.execution, + appBundleId: input.target?.appBundleId, + signal, + }); + return await interactor.scroll(input.direction, input.options); + }, + }); +} + +export type LocalScrollInteractorResolver = LocalInteractorOperationResolver; + +export function bindLocalScrollInteractor( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + resolveInteractor: LocalScrollInteractorResolver; + }>, +): ScrollRuntimeOperations { + return bindScrollDirection(params.signal, localInteractorSource(params)); +} + +export type ProviderScrollInteractorResolver = ProviderInteractorOperationResolver; + +/** Provider bindings fail closed when their exact owner no longer exposes its interactor. */ +export function bindProviderScrollInteractor( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + resolveInteractor: ProviderScrollInteractorResolver; + }>, +): ScrollRuntimeOperations { + return bindScrollDirection( + params.signal, + providerInteractorSource({ ...params, operation: 'scroll' }), + ); +} diff --git a/packages/platform-android/src/runtime.test.ts b/packages/platform-android/src/runtime.test.ts index 5e958bd8f5..067ca37806 100644 --- a/packages/platform-android/src/runtime.test.ts +++ b/packages/platform-android/src/runtime.test.ts @@ -443,3 +443,83 @@ function expectLifecycleFacts( } } } + +// R52/R53: the Android gesture-tier and scroll cells. The one gate the retired +// `requireGestureSupported` carried on Android was the TV target, which it applied to two-contact +// synthesis and to target-authored drag but never to a plain one-contact fling or pan. +test.each([ + // name, device, plan, multiTouch, drag, scroll + ['emulator', device, true, true, true, true], + ['physical device', { ...device, kind: 'device' as const }, true, true, true, true], + ['unknown kind', unknownKindDevice, true, true, true, true], + ['TV target', { ...device, target: 'tv' as const }, true, false, false, true], + [ + 'synthetic simulator row', + { ...device, kind: 'simulator' as const }, + false, + false, + false, + false, + ], +])( + 'declares the Android %s gesture and scroll cells', + async (_name, runtimeDevice, plan, multiTouch, drag, scroll) => { + const facts = await createAndroidPlatformRuntime(gestureHost()).inspectFacts(runtimeDevice); + expect(facts.operations.performGesturePlan.available).toBe(plan); + // Android honors a direction-authored fling's speed semantics, so it shares the plan cell. + expect(facts.operations.performDirectionalFlingPlan.available).toBe(plan); + expect(facts.operations.performMultiTouchGesturePlan.available).toBe(multiTouch); + expect(facts.operations.performTargetAuthoredDrag.available).toBe(drag); + expect(facts.operations.gestureViewport.available).toBe(plan); + expect(facts.operations.scrollDirection.available).toBe(scroll); + }, +); + +test('carries the retired Android TV hints verbatim', async () => { + const facts = await createAndroidPlatformRuntime(gestureHost()).inspectFacts({ + ...device, + target: 'tv', + }); + expect(facts.operations.performMultiTouchGesturePlan).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'Android TV has no touch input — this gesture is supported on Android phones, tablets, and the iOS simulator only.', + }); + expect(facts.operations.performTargetAuthoredDrag).toMatchObject({ + available: false, + hint: expect.stringContaining('source hold, timed movement, and destination hold'), + }); +}); + +test('binds only the Android gesture tiers the target admitted', async () => { + const bind = async (runtimeDevice: DeviceInfo) => + await createAndroidPlatformRuntime(gestureHost()).bind({ + device: runtimeDevice, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + const phone = await bind(device); + expect(phone.operations.performMultiTouchGesturePlan).toBeTypeOf('function'); + expect(phone.operations.scrollDirection).toBeTypeOf('function'); + const tv = await bind({ ...device, target: 'tv' }); + expect(tv.operations.performGesturePlan).toBeTypeOf('function'); + expect(tv.operations.performMultiTouchGesturePlan).toBeUndefined(); + expect(tv.operations.performTargetAuthoredDrag).toBeUndefined(); +}); + +function gestureHost(): PlatformRuntimeHost { + return { + processTransports: { resolve: async () => ({ mode: 'local' as const }) }, + appInventory: { + apple: { listApps: async () => [] }, + android: { listApps: async () => [] }, + harmonyos: { listApps: async () => [] }, + }, + localInteractors: { resolve: async () => ({}) }, + screenRecording: { android: { resolve: async () => ({ mode: 'local' as const }) } }, + } as unknown as PlatformRuntimeHost; +} diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index 924a9fa8aa..c834d3f7ea 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -1,6 +1,10 @@ import type { EnsureReadyInput } from '@agent-device/contracts/device-readiness-runtime'; import type { NetworkDumpInput } from '@agent-device/contracts/network-runtime'; -import type { DeviceBinding, RuntimeOperationFact } from '@agent-device/contracts/platform-runtime'; +import type { + DeviceBinding, + RuntimeFacts, + RuntimeOperationFact, +} from '@agent-device/contracts/platform-runtime'; import type { PlatformRuntimeHost, PlatformRuntimeOperations, @@ -18,6 +22,18 @@ import { bindLocalFocusInteractor, focusRuntimeOperationFacts, } from '@agent-device/contracts/focus-runtime'; +import { + ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT, + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} from '@agent-device/contracts/gesture-admission'; +import { + bindLocalGestureInteractor, + gestureRuntimeOperationFacts, +} from '@agent-device/contracts/gesture-runtime'; +import { + bindLocalScrollInteractor, + scrollRuntimeOperationFacts, +} from '@agent-device/contracts/scroll-runtime'; import { localRuntimeOwner, whenAdmitted } from '@agent-device/contracts/platform-runtime'; import { bindLocalScreenshotInteractor, @@ -168,6 +184,36 @@ function androidRuntimeHintsFact(device: DeviceInfo) { : runtimeHintsUnavailable; } +const gestureKindUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', + hint: 'Gestures are supported on Android emulators and physical devices.', +} as const); +const androidTvMultiTouchUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT, +} as const); +const androidTvDragUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} as const); + +/** + * A TV target has no touch input at all, which is the one Android gate the retired admission + * carried — it applied to two-contact synthesis and to target-authored drag, never to a plain + * one-contact fling or pan (the D-pad adapter executes those). + */ +function androidTouchTargetFact(device: DeviceInfo, refusal: RuntimeOperationFact) { + if (device.kind === 'simulator') return gestureKindUnavailable; + return device.target === 'tv' ? refusal : available; +} + +function androidGestureFact(device: DeviceInfo) { + return device.kind === 'simulator' ? gestureKindUnavailable : available; +} + /** adb drives every interaction cell the same way; only the synthetic `simulator` row lacks a device. */ function androidTouchFact(device: DeviceInfo) { return device.kind === 'simulator' ? focusKindUnavailable : available; @@ -217,6 +263,16 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), ...focusRuntimeOperationFacts({ focus: androidTouchFact(device) }), + ...gestureRuntimeOperationFacts({ + plan: androidGestureFact(device), + directionalFling: androidGestureFact(device), + multiTouch: androidTouchTargetFact(device, androidTvMultiTouchUnavailable), + targetAuthoredDrag: androidTouchTargetFact(device, androidTvDragUnavailable), + viewport: androidGestureFact(device), + }), + // `scroll` had no admission beyond its capability bucket, so its cell is that bucket + // verbatim: every Android kind but the synthetic `simulator` row. + ...scrollRuntimeOperationFacts({ scroll: androidGestureFact(device) }), // Text entry shares focus's cell: adb drives both, and only the synthetic `simulator` // row has no device behind it (parity with the retired `type` bucket). ...typeTextRuntimeOperationFacts({ type: androidTouchFact(device) }), @@ -292,51 +348,7 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor networkDump: async (input: NetworkDumpInput) => await dumpAndroidNetworkTraffic(host, request.device, input, request.scope.signal), ...recording, - ...(facts.operations.captureSnapshot.available - ? bindLocalSnapshotInteractor({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), - ...(facts.operations.captureScreenshot.available - ? bindLocalScreenshotInteractor({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), - ...(facts.operations.focusPoint.available - ? bindLocalFocusInteractor({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), - ...(facts.operations.typeText.available - ? bindLocalTypeTextInteractor({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), - ...whenAdmitted(facts.operations.tapPoint, () => - bindLocalTouchInteractor({ - facts: facts.operations, - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - pause: async (milliseconds) => - await host.clock.sleep(milliseconds, request.scope.signal), - }), - ), - ...(facts.operations.readTextAtPoint.available - ? bindElementTextRuntime({ - device: request.device, - signal: request.scope.signal, - resolveInteractor: host.localInteractors.resolve, - }) - : {}), + ...androidInteractionOperations(host, request, facts), ...bindAdmittedLocalInteractorOperations({ device: request.device, signal: request.scope.signal, @@ -398,3 +410,38 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor shutdown: async () => await appLogs.shutdown(), }); } + +/** + * The interactor-backed operations, each independently gated by its own admitted fact. Extracted + * for the same reason the Linux owner extracts its own: `bind` composes owners, it does not read + * facts one ternary at a time. + */ +function androidInteractionOperations( + host: PlatformRuntimeHost, + request: Parameters[0], + facts: RuntimeFacts, +): Partial['operations']> { + const resolver = { + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }; + return { + ...(facts.operations.captureSnapshot.available ? bindLocalSnapshotInteractor(resolver) : {}), + ...(facts.operations.captureScreenshot.available + ? bindLocalScreenshotInteractor(resolver) + : {}), + ...(facts.operations.focusPoint.available ? bindLocalFocusInteractor(resolver) : {}), + ...bindLocalGestureInteractor({ ...resolver, facts: facts.operations }), + ...(facts.operations.scrollDirection.available ? bindLocalScrollInteractor(resolver) : {}), + ...(facts.operations.typeText.available ? bindLocalTypeTextInteractor(resolver) : {}), + ...whenAdmitted(facts.operations.tapPoint, () => + bindLocalTouchInteractor({ + ...resolver, + facts: facts.operations, + pause: async (milliseconds) => await host.clock.sleep(milliseconds, request.scope.signal), + }), + ), + ...(facts.operations.readTextAtPoint.available ? bindElementTextRuntime(resolver) : {}), + }; +} diff --git a/packages/platform-apple/src/gesture-facts.test.ts b/packages/platform-apple/src/gesture-facts.test.ts new file mode 100644 index 0000000000..987398271f --- /dev/null +++ b/packages/platform-apple/src/gesture-facts.test.ts @@ -0,0 +1,108 @@ +import { expect, test } from 'vitest'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { createApplePlatformRuntime } from './runtime.ts'; +import { platformRuntimeHostFixture } from './runtime.fixtures.ts'; + +function appleDevice(overrides: Partial = {}): DeviceInfo { + return { + platform: 'apple', + appleOs: 'ios', + id: 'apple-fact', + name: 'Apple', + kind: 'simulator', + target: 'mobile', + booted: true, + ...overrides, + }; +} + +const leaves = { + ios: appleDevice(), + ipados: appleDevice({ appleOs: 'ipados' }), + tvos: appleDevice({ appleOs: 'tvos', target: 'tv' }), + macos: appleDevice({ appleOs: 'macos', kind: 'device', target: 'desktop' }), + visionos: appleDevice({ appleOs: 'visionos' }), + watchos: appleDevice({ appleOs: 'watchos' }), +}; + +// R52/R53: the gesture-tier and scroll cells the retired `requireGestureSupported` used to +// decide inside the daemon. Each row is one Apple leaf's complete gesture table, so a cell that +// silently widens (or narrows) fails here rather than on a device. +test.each([ + // leaf, plan, directionalFling, multiTouch, drag, viewport, scroll + ['iOS simulator', leaves.ios, true, true, true, true, true, true], + [ + 'iOS physical', + appleDevice({ kind: 'device', iosPhysicalDeviceBackend: 'coredevice' }), + true, + true, + false, + true, + true, + true, + ], + ['iPadOS simulator', leaves.ipados, true, true, true, true, true, true], + ['tvOS simulator', leaves.tvos, true, true, false, false, true, true], + ['macOS host', leaves.macos, true, true, false, false, true, true], + ['visionOS simulator', leaves.visionos, false, false, false, false, true, true], + ['watchOS sentinel', leaves.watchos, false, false, false, false, false, true], +])( + 'declares the %s gesture and scroll cells', + async (_name, device, plan, directionalFling, multiTouch, drag, viewport, scroll) => { + const facts = await createApplePlatformRuntime(platformRuntimeHostFixture()).inspectFacts( + device, + ); + expect(facts.operations.performGesturePlan.available).toBe(plan); + expect(facts.operations.performDirectionalFlingPlan.available).toBe(directionalFling); + expect(facts.operations.performMultiTouchGesturePlan.available).toBe(multiTouch); + expect(facts.operations.performTargetAuthoredDrag.available).toBe(drag); + expect(facts.operations.gestureViewport.available).toBe(viewport); + expect(facts.operations.scrollDirection.available).toBe(scroll); + }, +); + +test('carries the retired multi-touch hints verbatim on every Apple leaf that refused', async () => { + const runtime = createApplePlatformRuntime(platformRuntimeHostFixture()); + const physical = await runtime.inspectFacts(appleDevice({ kind: 'device' })); + expect(physical.operations.performMultiTouchGesturePlan).toEqual({ + available: false, + reason: 'unsupported-device-kind', + hint: 'Two-finger gesture synthesis is iOS-simulator only — not available on physical iOS devices.', + }); + const macos = await runtime.inspectFacts(leaves.macos); + expect(macos.operations.performMultiTouchGesturePlan).toMatchObject({ + available: false, + hint: expect.stringContaining('macOS automation has no multi-touch input'), + }); + expect(macos.operations.performTargetAuthoredDrag).toMatchObject({ + available: false, + hint: expect.stringContaining('source hold, timed movement, and destination hold'), + }); + // watchOS was caught by the retired admission's FIRST branch, before the policy that carries + // the per-OS hints ever ran, so it refuses without one. + const watchos = await runtime.inspectFacts(leaves.watchos); + expect(watchos.operations.performGesturePlan).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + }); +}); + +test('binds only the gesture tiers the leaf admitted', async () => { + const bind = async (device: DeviceInfo) => + await createApplePlatformRuntime(platformRuntimeHostFixture()).bind({ + device, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + const simulator = await bind(leaves.ios); + expect(simulator.operations.performGesturePlan).toBeTypeOf('function'); + expect(simulator.operations.performMultiTouchGesturePlan).toBeTypeOf('function'); + expect(simulator.operations.scrollDirection).toBeTypeOf('function'); + const physical = await bind(appleDevice({ kind: 'device' })); + expect(physical.operations.performGesturePlan).toBeTypeOf('function'); + expect(physical.operations.performMultiTouchGesturePlan).toBeUndefined(); +}); diff --git a/packages/platform-apple/src/gesture-facts.ts b/packages/platform-apple/src/gesture-facts.ts new file mode 100644 index 0000000000..daac83f3c8 --- /dev/null +++ b/packages/platform-apple/src/gesture-facts.ts @@ -0,0 +1,118 @@ +import { APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS } from '@agent-device/contracts/apple-multitouch-support'; +import { + PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT, + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} from '@agent-device/contracts/gesture-admission'; +import { gestureRuntimeOperationFacts } from '@agent-device/contracts/gesture-runtime'; +import type { RuntimeOperationFact } from '@agent-device/contracts/platform-runtime'; +import { scrollRuntimeOperationFacts } from '@agent-device/contracts/scroll-runtime'; +import { resolveDeviceAppleOs, type DeviceInfo } from '@agent-device/kernel/device'; + +/** + * The Apple owner's gesture-family cell table (R52/R53). + * + * This is admission the daemon used to own: `requireGestureSupported` decided Apple's gesture + * tiers from inside `core/capabilities.ts`. Every refusal below reproduces the exact cell — and + * the exact hint — that function produced, which is why the wording constants are imported rather + * than restated. `runtime.ts` composes these facts; it does not decide them. + */ +const available = Object.freeze({ available: true } as const); +const gestureLeafUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', +} as const); +const gestureKindUnavailable = unsupportedAppleDeviceKind( + 'Gestures are supported only for Apple simulators and devices.', +); +const physicalIosMultiTouchUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', + hint: PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT, +} as const); +const targetAuthoredDragUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} as const); +const scrollKindUnavailable = unsupportedAppleDeviceKind( + 'scroll is supported only for Apple simulators and devices.', +); + +function unsupportedAppleDeviceKind(hint: string) { + return Object.freeze({ available: false, reason: 'unsupported-device-kind', hint } as const); +} + +/** The gesture and scroll cells one Apple leaf declares, ready to spread into its fact catalog. */ +export function appleGestureAndScrollFacts(device: DeviceInfo) { + return { + ...gestureRuntimeOperationFacts({ + plan: appleGesturePlanFact(device), + directionalFling: appleGesturePlanFact(device), + multiTouch: appleMultiTouchGestureFact(device), + targetAuthoredDrag: appleTargetAuthoredDragFact(device), + viewport: appleGestureViewportFact(device), + }), + ...scrollRuntimeOperationFacts({ scroll: appleScrollFact(device) }), + }; +} + +/** + * One-contact gesture execution. The retired admission refused watchOS with every other + * `platform === 'web'` case and refused visionOS just after the multi-touch branch, both by + * reading `device.appleOs` RAW — an Apple device that declares no OS was admitted, so this reads + * it raw too rather than resolving a default that would newly refuse. + */ +function appleGesturePlanFact(device: DeviceInfo): RuntimeOperationFact { + if (device.appleOs === 'watchos' || device.appleOs === 'visionos') return gestureLeafUnavailable; + return appleTouchKind(device) ? available : gestureKindUnavailable; +} + +/** + * Two-contact synthesis, which on Apple is the iOS-simulator-only XCTest two-finger model. watchOS + * is refused with no hint because the retired admission caught it in its first branch, before the + * multi-touch policy that carries the per-OS hints ever ran. + */ +function appleMultiTouchGestureFact(device: DeviceInfo): RuntimeOperationFact { + if (device.appleOs === 'watchos') return gestureLeafUnavailable; + if (!appleTouchKind(device)) return gestureKindUnavailable; + const appleOs = resolveDeviceAppleOs(device); + if (appleOs !== 'ios' && appleOs !== 'ipados') { + const hint = APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS[appleOs]; + return Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + ...(hint === undefined ? {} : { hint }), + } as const); + } + return device.kind === 'simulator' ? available : physicalIosMultiTouchUnavailable; +} + +/** Target-authored drag needs source hold, timed movement, and destination hold preserved. */ +function appleTargetAuthoredDragFact(device: DeviceInfo): RuntimeOperationFact { + if (!appleTouchKind(device)) return gestureKindUnavailable; + const supported = + device.appleOs === undefined + ? device.target !== 'desktop' && device.target !== 'tv' + : device.appleOs === 'ios' || device.appleOs === 'ipados'; + return supported ? available : targetAuthoredDragUnavailable; +} + +/** The runner reads the frame for every Apple leaf that has one; watchOS has no runner at all. */ +function appleGestureViewportFact(device: DeviceInfo): RuntimeOperationFact { + if (device.appleOs === 'watchos') return gestureLeafUnavailable; + return appleTouchKind(device) ? available : gestureKindUnavailable; +} + +/** + * `scroll` had no admission beyond its capability bucket — no plugin closure, no gesture policy — + * so its cell is the bucket verbatim, watchOS included. A watchOS scroll still fails where it + * fails today: when the Apple interactor refuses to construct, not at admission. + */ +function appleScrollFact(device: DeviceInfo): RuntimeOperationFact { + return appleTouchKind(device) ? available : scrollKindUnavailable; +} + +/** The two kinds the Apple capability bucket ever admitted. */ +function appleTouchKind(device: DeviceInfo): boolean { + return device.kind === 'simulator' || device.kind === 'device'; +} diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index ab81fd5555..a80e2cdbb7 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -18,6 +18,8 @@ import { bindLocalFocusInteractor, focusRuntimeOperationFacts, } from '@agent-device/contracts/focus-runtime'; +import { bindLocalGestureInteractor } from '@agent-device/contracts/gesture-runtime'; +import { bindLocalScrollInteractor } from '@agent-device/contracts/scroll-runtime'; import { localRuntimeOwner, whenAdmitted } from '@agent-device/contracts/platform-runtime'; import { bindLocalScreenshotInteractor, @@ -40,6 +42,7 @@ import { resolveDeviceAppleOs, type DeviceInfo, } from '@agent-device/kernel/device'; +import { appleGestureAndScrollFacts } from './gesture-facts.ts'; import { createAppleAppLogRuntime } from './logs/runtime.ts'; import { dumpAppleNetworkTraffic } from './network/runtime.ts'; import { @@ -282,6 +285,7 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), ...focusRuntimeOperationFacts({ focus: appleFocusFact(device) }), + ...appleGestureAndScrollFacts(device), // Text entry rides the same interactor authority the point focus does, so it shares the // exact kind cell (parity with the retired `type` bucket, `{ simulator, device }`). ...typeTextRuntimeOperationFacts({ type: appleFocusFact(device) }), @@ -351,6 +355,19 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR resolveInteractor: host.localInteractors.resolve, }), ), + ...bindLocalGestureInteractor({ + device: request.device, + signal: request.scope.signal, + facts: facts.operations, + resolveInteractor: host.localInteractors.resolve, + }), + ...whenAdmitted(facts.operations.scrollDirection, () => + bindLocalScrollInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ), ...whenAdmitted(facts.operations.typeText, () => bindLocalTypeTextInteractor({ device: request.device, diff --git a/packages/platform-harmonyos/src/runtime.test.ts b/packages/platform-harmonyos/src/runtime.test.ts index 5cdfd05bb2..7148c03450 100644 --- a/packages/platform-harmonyos/src/runtime.test.ts +++ b/packages/platform-harmonyos/src/runtime.test.ts @@ -266,3 +266,52 @@ function expectLegacyLifecycleCell( } } } + +// R52/R53: hdc synthesizes one contact, so HarmonyOS admits the one-contact tiers on the same +// kind cell its focus/type overlay admitted, and refuses the two tiers it cannot reproduce. +test.each([ + ['device', device], + ['emulator', { ...device, kind: 'emulator' as const }], +])('declares the HarmonyOS %s gesture and scroll cells', async (_name, runtimeDevice) => { + const facts = await createHarmonyPlatformRuntime(gestureHost()).inspectFacts(runtimeDevice); + expect(facts.operations.performGesturePlan).toEqual({ available: true }); + expect(facts.operations.performDirectionalFlingPlan).toEqual({ available: true }); + expect(facts.operations.gestureViewport).toEqual({ available: true }); + expect(facts.operations.scrollDirection).toEqual({ available: true }); + // The retired admission refused two-contact synthesis on every platform that is neither + // Android nor Apple, with no hint — that is this cell, verbatim. + expect(facts.operations.performMultiTouchGesturePlan).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + }); + expect(facts.operations.performTargetAuthoredDrag).toMatchObject({ + available: false, + hint: expect.stringContaining('source hold, timed movement, and destination hold'), + }); +}); + +test('binds the HarmonyOS gesture tiers it admitted and omits the rest', async () => { + const binding = await createHarmonyPlatformRuntime(gestureHost()).bind({ + device, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + expect(binding.operations.performGesturePlan).toBeTypeOf('function'); + expect(binding.operations.gestureViewport).toBeTypeOf('function'); + expect(binding.operations.scrollDirection).toBeTypeOf('function'); + expect(binding.operations.performMultiTouchGesturePlan).toBeUndefined(); + expect(binding.operations.performTargetAuthoredDrag).toBeUndefined(); +}); + +function gestureHost(): PlatformRuntimeHost { + return { + processTransports: { resolve: async () => ({ mode: 'local' as const }) }, + appInventory: { harmonyos: { listApps: async () => [] } }, + appState: { harmonyos: { run: async () => ({ stdout: '' }) } }, + localInteractors: { resolve: async () => ({}) }, + } as unknown as PlatformRuntimeHost; +} diff --git a/packages/platform-harmonyos/src/runtime.ts b/packages/platform-harmonyos/src/runtime.ts index 190a412ff4..97c06dd2e7 100644 --- a/packages/platform-harmonyos/src/runtime.ts +++ b/packages/platform-harmonyos/src/runtime.ts @@ -14,6 +14,15 @@ import { bindLocalFocusInteractor, focusRuntimeOperationFacts, } from '@agent-device/contracts/focus-runtime'; +import { TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT } from '@agent-device/contracts/gesture-admission'; +import { + bindLocalGestureInteractor, + gestureRuntimeOperationFacts, +} from '@agent-device/contracts/gesture-runtime'; +import { + bindLocalScrollInteractor, + scrollRuntimeOperationFacts, +} from '@agent-device/contracts/scroll-runtime'; import { homeRuntimeOperationFacts } from '@agent-device/contracts/home-runtime'; import { bindAdmittedLocalInteractorOperations } from '@agent-device/contracts/interactor-operation-catalog'; import { keyboardRuntimeOperationFacts } from '@agent-device/contracts/keyboard-runtime'; @@ -147,6 +156,31 @@ function harmonyCloseTargetFact(device: DeviceInfo) { : closeTargetKindUnavailable; } +const gestureKindUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', + hint: 'Gestures are supported on HarmonyOS emulators and physical devices.', +} as const); +/** + * hdc synthesizes one contact. The retired admission refused two-contact synthesis on every + * platform that is neither Android nor Apple, with no hint — that is this cell. + */ +const multiTouchUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', +} as const); +const targetAuthoredDragUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} as const); + +function harmonyGestureFact(device: DeviceInfo): RuntimeOperationFact { + return device.kind === 'emulator' || device.kind === 'device' + ? available + : gestureKindUnavailable; +} + function harmonyFocusFact(device: DeviceInfo): RuntimeOperationFact { return device.kind === 'emulator' || device.kind === 'device' ? available : focusKindUnavailable; } @@ -209,6 +243,16 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor }), ...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }), ...focusRuntimeOperationFacts({ focus: harmonyFocusFact(device) }), + // Gestures share focus's kind cell (the overlay admitted `{emulator, device}`); only the + // two tiers hdc cannot synthesize are refused. + ...gestureRuntimeOperationFacts({ + plan: harmonyGestureFact(device), + directionalFling: harmonyGestureFact(device), + multiTouch: multiTouchUnavailable, + targetAuthoredDrag: targetAuthoredDragUnavailable, + viewport: harmonyGestureFact(device), + }), + ...scrollRuntimeOperationFacts({ scroll: harmonyGestureFact(device) }), // Text entry shares focus's cell: hdc drives both on the same two kinds. ...typeTextRuntimeOperationFacts({ type: harmonyFocusFact(device) }), ...touchRuntimeOperationFacts({ @@ -308,6 +352,19 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor resolveInteractor: host.localInteractors.resolve, }) : {}), + ...bindLocalGestureInteractor({ + device: request.device, + signal: request.scope.signal, + facts: facts.operations, + resolveInteractor: host.localInteractors.resolve, + }), + ...(facts.operations.scrollDirection.available + ? bindLocalScrollInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), ...bindAdmittedLocalInteractorOperations({ device: request.device, signal: request.scope.signal, diff --git a/packages/platform-linux/src/runtime.test.ts b/packages/platform-linux/src/runtime.test.ts index a0741c2b7b..04a808e711 100644 --- a/packages/platform-linux/src/runtime.test.ts +++ b/packages/platform-linux/src/runtime.test.ts @@ -245,3 +245,57 @@ test('the Linux surface capture composes the per-capture signal with the request scope.abort(); expect(passedSecond.aborted).toBe(true); }); + +// R52/R53: Linux is the one owner whose gesture tiers genuinely split. Its drag primitive +// preserves a coordinate fling's endpoints but not a direction-authored fling's speed semantics +// (`gesture fling is not supported on Linux`), it synthesizes one contact, and it has no frame +// read of its own — which is why `gestureViewport` is PREFERRED rather than required. +test.each([ + ['desktop device', 'device' as const, true], + ['non-desktop kind', 'emulator' as const, false], +])('declares the Linux %s gesture and scroll cells', async (_name, kind, supported) => { + const facts = await createLinuxPlatformRuntime(lifecycleHost()).inspectFacts({ + platform: 'linux', + id: 'linux', + name: 'Linux', + kind, + target: 'desktop', + booted: true, + }); + expect(facts.operations.performGesturePlan.available).toBe(supported); + expect(facts.operations.scrollDirection.available).toBe(supported); + expect(facts.operations.performDirectionalFlingPlan.available).toBe(false); + expect(facts.operations.performMultiTouchGesturePlan.available).toBe(false); + expect(facts.operations.performTargetAuthoredDrag).toMatchObject({ + available: false, + hint: expect.stringContaining('source hold, timed movement, and destination hold'), + }); + expect(facts.operations.gestureViewport.available).toBe(false); +}); + +test('binds the Linux coordinate-fling tier without a frame read', async () => { + const binding = await createLinuxPlatformRuntime(lifecycleHost()).bind({ + device: { + platform: 'linux', + id: 'linux', + name: 'Linux', + kind: 'device', + target: 'desktop', + booted: true, + }, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + expect(binding.operations.performGesturePlan).toBeTypeOf('function'); + expect(binding.operations.scrollDirection).toBeTypeOf('function'); + expect(binding.operations.performDirectionalFlingPlan).toBeUndefined(); + expect(binding.operations.performMultiTouchGesturePlan).toBeUndefined(); + expect(binding.operations.performTargetAuthoredDrag).toBeUndefined(); + // Absent, not broken: the caller derives the coordinate frame from a capture instead, which is + // exactly how a Linux gesture resolved its viewport before this migration. + expect(binding.operations.gestureViewport).toBeUndefined(); +}); diff --git a/packages/platform-linux/src/runtime.ts b/packages/platform-linux/src/runtime.ts index 1b160bf155..5f8f43949c 100644 --- a/packages/platform-linux/src/runtime.ts +++ b/packages/platform-linux/src/runtime.ts @@ -22,6 +22,15 @@ import { bindLocalFocusInteractor, focusRuntimeOperationFacts, } from '@agent-device/contracts/focus-runtime'; +import { TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT } from '@agent-device/contracts/gesture-admission'; +import { + bindLocalGestureInteractor, + gestureRuntimeOperationFacts, +} from '@agent-device/contracts/gesture-runtime'; +import { + bindLocalScrollInteractor, + scrollRuntimeOperationFacts, +} from '@agent-device/contracts/scroll-runtime'; import { homeRuntimeOperationFacts } from '@agent-device/contracts/home-runtime'; import { bindAdmittedLocalInteractorOperations } from '@agent-device/contracts/interactor-operation-catalog'; import { @@ -63,6 +72,27 @@ const typeKindUnavailable = unavailableLinuxRuntimeFact( 'unsupported-device-kind', 'type is supported only for the Linux desktop device.', ); +const gestureKindUnavailable = unavailableLinuxRuntimeFact( + 'unsupported-device-kind', + 'Gestures are supported only for the Linux desktop device.', +); +const scrollKindUnavailable = unavailableLinuxRuntimeFact( + 'unsupported-device-kind', + 'scroll is supported only for the Linux desktop device.', +); +/** + * The drag primitive preserves a coordinate fling's endpoints but not a direction-authored + * fling's speed semantics, which is the one gesture the retired admission refused BY PLATFORM + * rather than by leaf or kind. + */ +const directionalFlingUnavailable = unavailableLinuxRuntimeFact('unsupported-platform-leaf'); +const multiTouchUnavailable = unavailableLinuxRuntimeFact('unsupported-platform-leaf'); +const targetAuthoredDragUnavailable = unavailableLinuxRuntimeFact( + 'unsupported-platform-leaf', + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +); +/** No frame read of its own: a Linux gesture derives its viewport from a capture, as it does today. */ +const gestureViewportUnavailable = unavailableLinuxRuntimeFact('unsupported-platform-leaf'); const runtimeHintsUnavailable = unavailableLinuxRuntimeFact( 'unsupported-platform-leaf', 'Runtime hints are supported only for local iOS-family simulators and Android devices.', @@ -155,7 +185,7 @@ export function createLinuxPlatformRuntime(host: PlatformRuntimeHost): PlatformR }); } -/** The five desktop-interactor operations, each independently gated by its own admitted fact. */ +/** The desktop-interactor operations, each independently gated by its own admitted fact. */ function linuxInteractionOperations( host: PlatformRuntimeHost, request: Parameters[0], @@ -171,6 +201,8 @@ function linuxInteractionOperations( ? bindLocalScreenshotInteractor(resolver) : {}), ...(facts.operations.focusPoint.available ? bindLocalFocusInteractor(resolver) : {}), + ...bindLocalGestureInteractor({ ...resolver, facts: facts.operations }), + ...(facts.operations.scrollDirection.available ? bindLocalScrollInteractor(resolver) : {}), ...(facts.operations.typeText.available ? bindLocalTypeTextInteractor(resolver) : {}), ...(facts.operations.readTextAtPoint.available ? bindElementTextRuntime(resolver) : {}), ...bindAdmittedLocalInteractorOperations({ @@ -190,6 +222,8 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts snapshot: snapshotKindUnavailable, viewport: unsupportedPlatformLeaf, focus: focusKindUnavailable, + gesture: gestureKindUnavailable, + scroll: scrollKindUnavailable, typeText: typeKindUnavailable, touch: focusKindUnavailable, elementText: elementTextKindUnavailable, @@ -229,6 +263,17 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts // the only Linux cell with a pointer to drive. ...focusRuntimeOperationFacts({ focus: linuxDesktopFact(device, focusKindUnavailable) }), // Text entry shares focus's cell: ydotool drives both on the desktop device only. + // The desktop device is the only Linux cell with a pointer; three tiers are refused on + // every cell — a direction-authored fling's speed semantics, two-contact synthesis, and + // target-authored drag timing. + ...gestureRuntimeOperationFacts({ + plan: linuxDesktopFact(device, gestureKindUnavailable), + directionalFling: directionalFlingUnavailable, + multiTouch: multiTouchUnavailable, + targetAuthoredDrag: targetAuthoredDragUnavailable, + viewport: gestureViewportUnavailable, + }), + ...scrollRuntimeOperationFacts({ scroll: linuxDesktopFact(device, scrollKindUnavailable) }), ...typeTextRuntimeOperationFacts({ type: linuxDesktopFact(device, typeKindUnavailable) }), ...linuxTouchFacts(device), // The Linux read is value-first (AXValue/title/description) where the captured tree is diff --git a/packages/platform-vega/src/runtime.test.ts b/packages/platform-vega/src/runtime.test.ts index c1c9060f51..d967c7bf0b 100644 --- a/packages/platform-vega/src/runtime.test.ts +++ b/packages/platform-vega/src/runtime.test.ts @@ -208,3 +208,41 @@ function expectLifecycleFacts( } } } + +// R52/R53: `gesture`, `scroll` and `swipe` never carried a vega capability bucket, so no cell of +// the gesture family was ever admitted on this owner. +test('declares every Vega gesture and scroll cell unavailable', async () => { + const facts = await createVegaPlatformRuntime(lifecycleHost()).inspectFacts({ + platform: 'vega', + id: 'vega-vvd', + name: 'Vega VVD', + kind: 'emulator', + target: 'tv', + booted: true, + }); + for (const operation of [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'gestureViewport', + ] as const) { + expect(facts.operations[operation]).toMatchObject({ + available: false, + hint: expect.stringContaining('remote navigation only'), + }); + } + // Two tiers keep the retired closures' own wording instead of this owner's: two-contact + // synthesis refused with no hint at all on a non-Android, non-Apple platform, and + // target-authored drag refused by naming the phases an adapter must preserve. + expect(facts.operations.performMultiTouchGesturePlan).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + }); + expect(facts.operations.performTargetAuthoredDrag).toMatchObject({ + available: false, + hint: expect.stringContaining('source hold, timed movement, and destination hold'), + }); + expect(facts.operations.scrollDirection).toMatchObject({ + available: false, + hint: expect.stringContaining('remote navigation only'), + }); +}); diff --git a/packages/platform-vega/src/runtime.ts b/packages/platform-vega/src/runtime.ts index ac43670a8f..84ecbee93e 100644 --- a/packages/platform-vega/src/runtime.ts +++ b/packages/platform-vega/src/runtime.ts @@ -17,6 +17,8 @@ import { homeRuntimeOperationFacts } from '@agent-device/contracts/home-runtime' import { bindAdmittedLocalInteractorOperations } from '@agent-device/contracts/interactor-operation-catalog'; import { localRuntimeOwner, sameRuntimeOwner } from '@agent-device/contracts/platform-runtime'; import { createUnavailablePlatformRuntimeFacts } from '@agent-device/contracts/platform-runtime-unavailable'; +import { TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT } from '@agent-device/contracts/gesture-admission'; +import { gestureRuntimeOperationFacts } from '@agent-device/contracts/gesture-runtime'; import { tvRemoteRuntimeOperationFacts } from '@agent-device/contracts/tv-remote-runtime'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -99,6 +101,25 @@ const typeUnavailable = vegaUnavailable( 'unsupported-platform-leaf', 'type is not supported on Vega OS: the Vega runtime exposes remote navigation only.', ); + +const gestureUnavailable = vegaUnavailable( + 'unsupported-platform-leaf', + 'Gestures are not supported on Vega OS: the Vega runtime exposes remote navigation only.', +); +const scrollUnavailable = vegaUnavailable( + 'unsupported-platform-leaf', + 'scroll is not supported on Vega OS: the Vega runtime exposes remote navigation only.', +); +/** + * The two tiers the retired admission refused BY NAME on a non-Android, non-Apple platform, in its + * own wording: two-contact synthesis with no hint at all, and target-authored drag by naming the + * phases an adapter has to preserve. + */ +const multiTouchUnavailable = vegaUnavailable('unsupported-platform-leaf'); +const targetAuthoredDragUnavailable = vegaUnavailable( + 'unsupported-platform-leaf', + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +); // `orientation` and every keyboard action never carried a Vega capability bucket at all; `back`, // `home`, and `tv-remote` did (the retired `vegaPlugin` closure), gated by the same VVD cell // their lifecycle open/close already require. @@ -135,6 +156,10 @@ function vegaFacts(device: DeviceInfo): RuntimeFacts viewport: unsupportedPlatformLeaf, // Vega exposes remote navigation only; it never carried a `focus` capability bucket. focus: focusUnavailable, + // Vega exposes remote navigation only; `gesture`, `scroll` and `swipe` never carried a vega + // capability bucket, so no gesture-family cell was ever admitted here. + gesture: gestureUnavailable, + scroll: scrollUnavailable, typeText: typeUnavailable, touch: unsupportedPlatformLeaf, elementText: unsupportedPlatformLeaf, @@ -169,6 +194,16 @@ function vegaFacts(device: DeviceInfo): RuntimeFacts ...tvRemoteRuntimeOperationFacts({ tvRemote: supported ? lifecycleAvailable : tvRemoteUnavailable, }), + // Two gesture tiers keep the retired closures' own wording instead of this owner's generic + // one; the rest had no retired closure and now refuse at admission rather than inside the + // Vega interactor. + ...gestureRuntimeOperationFacts({ + plan: gestureUnavailable, + directionalFling: gestureUnavailable, + multiTouch: multiTouchUnavailable, + targetAuthoredDrag: targetAuthoredDragUnavailable, + viewport: gestureUnavailable, + }), }, }); } diff --git a/packages/platform-web/src/runtime.test.ts b/packages/platform-web/src/runtime.test.ts index 6727949b2c..d7f9e523e7 100644 --- a/packages/platform-web/src/runtime.test.ts +++ b/packages/platform-web/src/runtime.test.ts @@ -418,3 +418,45 @@ function host( }, } as unknown as PlatformRuntimeHost; } + +// R52/R53: `scroll` is the one gesture-family command the web overlay ever admitted +// (`WEB_INTERACTION_COMMANDS`); `gesture` and `swipe` carried no web bucket at all, and the +// retired admission refused `platform === 'web'` outright. +test('admits web scrolling and refuses every gesture tier', async () => { + const binding = await createWebPlatformRuntime(host({ mode: 'local' })).bind({ + device, + intent: { kind: 'ordinary' }, + scope: scope(), + }); + expect(binding.facts.operations.scrollDirection).toEqual({ available: true }); + expect(binding.operations.scrollDirection).toBeTypeOf('function'); + for (const operation of [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'performMultiTouchGesturePlan', + 'gestureViewport', + ] as const) { + expect(binding.facts.operations[operation]).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + }); + expect(binding.operations[operation]).toBeUndefined(); + } + // The retired admission checked drag FIRST, before its `platform === 'web'` branch, so this one + // tier keeps the target-authored-drag wording rather than the bare platform refusal. + expect(binding.facts.operations.performTargetAuthoredDrag).toMatchObject({ + available: false, + hint: expect.stringContaining('source hold, timed movement, and destination hold'), + }); + expect(binding.operations.performTargetAuthoredDrag).toBeUndefined(); +}); + +test('refuses web scrolling on a non-browser web cell', async () => { + const binding = await createWebPlatformRuntime(host({ mode: 'local' })).bind({ + device: { ...device, kind: 'emulator' }, + intent: { kind: 'ordinary' }, + scope: scope(), + }); + expect(binding.facts.operations.scrollDirection.available).toBe(false); + expect(binding.operations.scrollDirection).toBeUndefined(); +}); diff --git a/packages/platform-web/src/runtime.ts b/packages/platform-web/src/runtime.ts index 65eef482e6..a26c02bf0b 100644 --- a/packages/platform-web/src/runtime.ts +++ b/packages/platform-web/src/runtime.ts @@ -15,6 +15,12 @@ import { bindLocalFocusInteractor, focusRuntimeOperationFacts, } from '@agent-device/contracts/focus-runtime'; +import { TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT } from '@agent-device/contracts/gesture-admission'; +import { gestureRuntimeOperationFacts } from '@agent-device/contracts/gesture-runtime'; +import { + bindLocalScrollInteractor, + scrollRuntimeOperationFacts, +} from '@agent-device/contracts/scroll-runtime'; import { localRuntimeOwner, sameRuntimeOwner, @@ -108,6 +114,16 @@ function webAvailableFact(condition: boolean, unavailable: RuntimeOperationFact) return condition ? available : unavailable; } +/** `gesture` and `swipe` never carried a web capability bucket; the browser drives no synthesis. */ +const gestureUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', +} as const); +const targetAuthoredDragUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} as const); const openTargetKindUnavailable = Object.freeze({ available: false, reason: 'unsupported-device-kind', @@ -258,6 +274,13 @@ function bindWebRuntime( pause: async (milliseconds) => await host.clock.sleep(milliseconds, signal), }), ), + ...(facts.operations.scrollDirection.available + ? bindLocalScrollInteractor({ + device, + signal, + resolveInteractor: host.localInteractors.resolve, + }) + : {}), ...(facts.operations.setViewport.available ? { setViewport: async (input) => { @@ -347,6 +370,18 @@ function webRuntimeFacts( fillRef: webOptionalOperationFact(interactor?.fillRef, browserDevice), tapElementSelector: readinessUnavailable, }), + // `scroll` is the one gesture-family command the web overlay admitted + // (`WEB_INTERACTION_COMMANDS`), so it shares focus's `{ device: true }` cell. `gesture` and + // `swipe` never carried a web bucket, and the retired admission refused `platform === 'web'` + // outright. Drag is the exception it checked FIRST, by naming the phases an adapter needs. + ...scrollRuntimeOperationFacts({ scroll: browserDevice }), + ...gestureRuntimeOperationFacts({ + plan: gestureUnavailable, + directionalFling: gestureUnavailable, + multiTouch: gestureUnavailable, + targetAuthoredDrag: targetAuthoredDragUnavailable, + viewport: gestureUnavailable, + }), ...viewportRuntimeOperationFacts({ setViewport: browserDevice }), // The web backend has no point-addressed read: `get` answers from the captured DOM tree, // which is what the legacy dispatch already did once its Apple-runner attempt failed. diff --git a/packages/provider-limrun/src/app-log-runtime.test.ts b/packages/provider-limrun/src/app-log-runtime.test.ts index 894487f474..794eb1d7fc 100644 --- a/packages/provider-limrun/src/app-log-runtime.test.ts +++ b/packages/provider-limrun/src/app-log-runtime.test.ts @@ -406,3 +406,78 @@ function expectLimrunNavigationAndKeyboardFacts( } } } + +// R52/R53: Limrun's two session kinds run different interactors. The Android emulator session +// runs the ordinary Android interactor (every tier); the iOS direct session's own +// `performGesture` refuses, so stating that as a fact refuses at admission instead of +// mid-execution — with the interactor's wording preserved. +const limrunAndroid = { + platform: 'android' as const, + id: 'limrun:android:lease-a', + name: 'Limrun Android', + kind: 'emulator' as const, + target: 'mobile' as const, + booted: true, +}; + +test('admits every gesture tier on a Limrun Android session', async () => { + const owner = createLimrunPlatformRuntimeOwner( + limrunOwnerOptions({ getInteractor: () => ({}) as never }), + ); + const binding = await owner.bind({ + device: limrunAndroid, + intent: { kind: 'ordinary' }, + scope, + }); + for (const operation of [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'performMultiTouchGesturePlan', + 'performTargetAuthoredDrag', + 'gestureViewport', + 'scrollDirection', + ] as const) { + expect(binding.facts.operations[operation]).toEqual({ available: true }); + expect(binding.operations[operation]).toBeTypeOf('function'); + } +}); + +test('refuses Limrun iOS gestures at admission while keeping its scroll', async () => { + const owner = createLimrunPlatformRuntimeOwner( + limrunOwnerOptions({ getInteractor: () => ({}) as never }), + ); + const binding = await owner.bind({ device, intent: { kind: 'ordinary' }, scope }); + for (const operation of [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'performMultiTouchGesturePlan', + 'performTargetAuthoredDrag', + 'gestureViewport', + ] as const) { + expect(binding.facts.operations[operation]).toEqual({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'Limrun iOS direct sessions do not expose portable gesture execution yet.', + }); + expect(binding.operations[operation]).toBeUndefined(); + } + // The iOS direct session exposes scrolling directly — no gesture synthesis involved. + expect(binding.facts.operations.scrollDirection).toEqual({ available: true }); + expect(binding.operations.scrollDirection).toBeTypeOf('function'); +}); + +test('closes every Limrun gesture and scroll cell without a live session', async () => { + const owner = createLimrunPlatformRuntimeOwner( + limrunOwnerOptions({ getInteractor: () => ({}) as never, hasLiveSession: () => false }), + ); + const facts = await owner.inspectFacts(limrunAndroid); + for (const operation of [ + 'performGesturePlan', + 'performMultiTouchGesturePlan', + 'performTargetAuthoredDrag', + 'gestureViewport', + 'scrollDirection', + ] as const) { + expect(facts.operations[operation].available).toBe(false); + } +}); diff --git a/packages/provider-limrun/src/app-log-runtime.ts b/packages/provider-limrun/src/app-log-runtime.ts index 955d855e62..b6506f4109 100644 --- a/packages/provider-limrun/src/app-log-runtime.ts +++ b/packages/provider-limrun/src/app-log-runtime.ts @@ -95,6 +95,8 @@ export function createLimrunPlatformRuntimeOwner( screenshot: liveSessionUnavailable, viewport: liveSessionUnavailable, focus: liveSessionUnavailable, + gesture: liveSessionUnavailable, + scroll: liveSessionUnavailable, typeText: liveSessionUnavailable, touch: liveSessionUnavailable, elementText: liveSessionUnavailable, diff --git a/packages/provider-limrun/src/interaction-operations.ts b/packages/provider-limrun/src/interaction-operations.ts index 66a729400d..3a104a843c 100644 --- a/packages/provider-limrun/src/interaction-operations.ts +++ b/packages/provider-limrun/src/interaction-operations.ts @@ -3,6 +3,19 @@ import { bindProviderFocusInteractor, focusRuntimeOperationFacts, } from '@agent-device/contracts/focus-runtime'; +import { + ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT, + TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} from '@agent-device/contracts/gesture-admission'; +import { + bindProviderGestureInteractor, + gestureRuntimeOperationFacts, + type GestureRuntimeOperationFacts, +} from '@agent-device/contracts/gesture-runtime'; +import { + bindProviderScrollInteractor, + scrollRuntimeOperationFacts, +} from '@agent-device/contracts/scroll-runtime'; import { homeRuntimeOperationFacts } from '@agent-device/contracts/home-runtime'; import { keyboardRuntimeOperationFacts } from '@agent-device/contracts/keyboard-runtime'; import { orientationRuntimeOperationFacts } from '@agent-device/contracts/orientation-runtime'; @@ -23,6 +36,63 @@ import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; import { setTimeout as sleep } from 'node:timers/promises'; const available = Object.freeze({ available: true } as const); +/** + * Limrun's iOS direct session drives text and touch but exposes no portable gesture execution — + * its interactor's own `performGesture` refuses with this wording. Stating it as a fact refuses at + * admission instead of mid-execution (ADR 0019 §6), keeping the agent-facing hint identical. + */ +const iosGestureUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'Limrun iOS direct sessions do not expose portable gesture execution yet.', +} as const); +const androidTvMultiTouchUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT, +} as const); +const androidTvDragUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT, +} as const); + +/** + * Gesture cells split by the interactor behind the session: an Android emulator session runs the + * ordinary Android interactor (every tier, minus the TV gates it always carried), while the iOS + * direct session has no gesture execution at all. + */ +function limrunGestureFacts( + device: DeviceInfo, + cell: RuntimeOperationUnavailability | typeof available, +): GestureRuntimeOperationFacts { + if (cell !== available) { + return gestureRuntimeOperationFacts({ + plan: cell, + directionalFling: cell, + multiTouch: cell, + targetAuthoredDrag: cell, + viewport: cell, + }); + } + if (device.platform !== 'android') { + return gestureRuntimeOperationFacts({ + plan: iosGestureUnavailable, + directionalFling: iosGestureUnavailable, + multiTouch: iosGestureUnavailable, + targetAuthoredDrag: iosGestureUnavailable, + viewport: iosGestureUnavailable, + }); + } + const tv = device.target === 'tv'; + return gestureRuntimeOperationFacts({ + plan: available, + directionalFling: available, + multiTouch: tv ? androidTvMultiTouchUnavailable : available, + targetAuthoredDrag: tv ? androidTvDragUnavailable : available, + viewport: available, + }); +} const homeUnavailableIos = Object.freeze({ available: false, reason: 'unsupported-provider-mode', @@ -83,10 +153,17 @@ export function limrunInteractionOperationFacts( fillRef: unsupportedTouch, tapElementSelector: liveSessionUnavailable ?? (isIosFamily(device) ? cell : unsupportedTouch), }), + ...limrunGestureFacts(device, cell), + // `scroll` needs no gesture synthesis: both session kinds expose it directly. + ...scrollRuntimeOperationFacts({ scroll: cell }), }); } -/** Binds the interactor-backed operations (snapshot, screenshot, focus, type) for one session. */ +/** + * Binds the interactor-backed operations (snapshot, screenshot, focus, type, gestures, scroll) for + * one session. Only a live session reaches here, so the gesture tiers are gated by the same device + * split their facts use rather than by liveness. + */ export function bindLimrunInteractionOperations( params: Readonly<{ device: DeviceInfo; @@ -108,6 +185,13 @@ export function bindLimrunInteractionOperations( pause: async (milliseconds) => await sleep(milliseconds, undefined, { signal }), }), ...bindProviderScreenshotInteractor({ device, signal, resolveInteractor }), + ...bindProviderGestureInteractor({ + device, + signal, + facts: limrunGestureFacts(device, available), + resolveInteractor, + }), + ...bindProviderScrollInteractor({ device, signal, resolveInteractor }), }); } diff --git a/packages/provider-webdriver/src/platform-runtime.test.ts b/packages/provider-webdriver/src/platform-runtime.test.ts index 2062b5a4d3..4abf1453c2 100644 --- a/packages/provider-webdriver/src/platform-runtime.test.ts +++ b/packages/provider-webdriver/src/platform-runtime.test.ts @@ -453,3 +453,53 @@ function host(run: PlatformRuntimeHost['commands']['run']): PlatformRuntimeHost }, } as unknown as PlatformRuntimeHost; } + +// R52/R53: gestures and scrolling ride the same reachability gate the captures do. The one extra +// gate is the retired multi-touch policy — this provider owns physical devices only, and +// two-finger synthesis on a physical iOS device was refused before this migration too. +test.each([ + ['Android physical', device, true], + ['iOS physical', { ...device, platform: 'apple' as const, appleOs: 'ios' as const }, false], +])('declares the WebDriver %s gesture and scroll cells', async (_name, owned, multiTouch) => { + const owner = createWebDriverPlatformRuntimeOwner({ + host: host(async () => ({ stdout: '', stderr: '', exitCode: 0 })), + owner: providerRuntimeOwner('browserstack', 'android'), + ownsDevice: () => true, + getInteractor: () => ({}) as unknown as Interactor, + }); + const facts = await owner.inspectFacts(owned); + expect(facts.operations.performGesturePlan).toEqual({ available: true }); + expect(facts.operations.performDirectionalFlingPlan).toEqual({ available: true }); + expect(facts.operations.performTargetAuthoredDrag).toEqual({ available: true }); + expect(facts.operations.gestureViewport).toEqual({ available: true }); + expect(facts.operations.scrollDirection).toEqual({ available: true }); + expect(facts.operations.performMultiTouchGesturePlan.available).toBe(multiTouch); + if (!multiTouch) { + expect(facts.operations.performMultiTouchGesturePlan).toMatchObject({ + hint: 'Two-finger gesture synthesis is iOS-simulator only — not available on physical iOS devices.', + }); + } +}); + +test('closes every WebDriver gesture and scroll cell when the interactor is unreachable', async () => { + const owner = createWebDriverPlatformRuntimeOwner({ + host: host(async () => ({ stdout: '', stderr: '', exitCode: 0 })), + owner: providerRuntimeOwner('browserstack', 'android'), + ownsDevice: () => true, + getInteractor: undefined, + }); + const facts = await owner.inspectFacts(device); + for (const operation of [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'performMultiTouchGesturePlan', + 'performTargetAuthoredDrag', + 'gestureViewport', + 'scrollDirection', + ] as const) { + expect(facts.operations[operation]).toMatchObject({ + available: false, + reason: 'unsupported-provider-mode', + }); + } +}); diff --git a/packages/provider-webdriver/src/platform-runtime.ts b/packages/provider-webdriver/src/platform-runtime.ts index 404bea3fdc..44f7723610 100644 --- a/packages/provider-webdriver/src/platform-runtime.ts +++ b/packages/provider-webdriver/src/platform-runtime.ts @@ -13,6 +13,15 @@ import { bindProviderFocusInteractor, focusRuntimeOperationFacts, } from '@agent-device/contracts/focus-runtime'; +import { PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT } from '@agent-device/contracts/gesture-admission'; +import { + bindProviderGestureInteractor, + gestureRuntimeOperationFacts, +} from '@agent-device/contracts/gesture-runtime'; +import { + bindProviderScrollInteractor, + scrollRuntimeOperationFacts, +} from '@agent-device/contracts/scroll-runtime'; import { homeRuntimeOperationFacts } from '@agent-device/contracts/home-runtime'; import { bindAdmittedProviderInteractorOperations } from '@agent-device/contracts/interactor-operation-catalog'; import { keyboardRuntimeOperationFacts } from '@agent-device/contracts/keyboard-runtime'; @@ -134,6 +143,21 @@ const typeUnavailable = Object.freeze({ reason: 'unsupported-provider-mode', hint: 'This WebDriver provider runtime does not expose text entry for this device.', } as const); +const gestureUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'This WebDriver provider runtime does not expose gestures for this device.', +} as const); +const scrollUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-provider-mode', + hint: 'This WebDriver provider runtime does not expose scrolling for this device.', +} as const); +const physicalIosMultiTouchUnavailable = Object.freeze({ + available: false, + reason: 'unsupported-device-kind', + hint: PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT, +} as const); const elementTextUnavailable = Object.freeze({ available: false, reason: 'unsupported-provider-mode', @@ -288,6 +312,8 @@ function webDriverInteractionOperations( : {}), ...(facts.operations.focusPoint.available ? bindProviderFocusInteractor(resolver) : {}), ...(facts.operations.typeText.available ? bindProviderTypeTextInteractor(resolver) : {}), + ...bindProviderGestureInteractor({ ...resolver, facts: facts.operations }), + ...(facts.operations.scrollDirection.available ? bindProviderScrollInteractor(resolver) : {}), ...bindAdmittedProviderInteractorOperations({ ...resolver, facts: facts.operations, @@ -383,6 +409,8 @@ function webDriverFacts( screenshot: inactiveSession, viewport: inactiveSession, focus: inactiveSession, + gesture: inactiveSession, + scroll: inactiveSession, typeText: inactiveSession, touch: inactiveSession, elementText: inactiveSession, @@ -415,6 +443,8 @@ function webDriverFacts( screenshot: screenshotUnavailable, viewport: viewportUnavailable, focus: focusUnavailable, + gesture: gestureUnavailable, + scroll: scrollUnavailable, typeText: typeUnavailable, touch: typeUnavailable, elementText: elementTextUnavailable, @@ -464,6 +494,18 @@ function webDriverFacts( fillRef: typeUnavailable, tapElementSelector: focusUnavailable, }), + // Gestures and scrolling ride the same provider interactor the captures do, so they need the + // same reachability. The one extra gate is the retired multi-touch policy: this provider only + // ever owns physical devices, and two-finger synthesis on a physical iOS device was refused + // before this migration exactly as it is refused here. + ...gestureRuntimeOperationFacts({ + plan: interactorCell(reachable, gestureUnavailable), + directionalFling: interactorCell(reachable, gestureUnavailable), + multiTouch: webDriverMultiTouchCell(device, reachable), + targetAuthoredDrag: interactorCell(reachable, gestureUnavailable), + viewport: interactorCell(reachable, gestureUnavailable), + }), + ...scrollRuntimeOperationFacts({ scroll: interactorCell(reachable, scrollUnavailable) }), // `back`/`home`/`orientation` ride the same reachable interactor; `tvRemote` always throws // unsupported in this interactor regardless of reachability (no capability declares it). ...backRuntimeOperationFacts({ back: interactorCell(reachable, backUnavailable) }), @@ -491,6 +533,12 @@ function webDriverFacts( }); } +/** Two-finger synthesis is iOS-simulator only, and this provider owns no simulators. */ +function webDriverMultiTouchCell(device: DeviceInfo, reachable: boolean): RuntimeOperationFact { + if (!reachable) return gestureUnavailable; + return device.platform === 'apple' ? physicalIosMultiTouchUnavailable : available; +} + /** The device shapes this provider can reach at all through its own WebDriver interactor. */ function webDriverInteractorDevice(device: DeviceInfo): boolean { return ( diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index de0eb569c0..df94a36088 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -78,10 +78,12 @@ const CONTRACT_EXPORTS = [ '@agent-device/contracts/durable-resource-envelope', '@agent-device/contracts/element-text-runtime', '@agent-device/contracts/focus-runtime', + '@agent-device/contracts/gesture-admission', '@agent-device/contracts/gesture-input', '@agent-device/contracts/gesture-normalization', '@agent-device/contracts/gesture-plan', '@agent-device/contracts/gesture-plan-types', + '@agent-device/contracts/gesture-runtime', '@agent-device/contracts/home-runtime', '@agent-device/contracts/interaction', '@agent-device/contracts/interaction-error', @@ -111,6 +113,7 @@ const CONTRACT_EXPORTS = [ '@agent-device/contracts/screenshot-runtime', '@agent-device/contracts/scroll-command', '@agent-device/contracts/scroll-gesture', + '@agent-device/contracts/scroll-runtime', '@agent-device/contracts/selector-observation-runtime', '@agent-device/contracts/session', '@agent-device/contracts/settings', diff --git a/scripts/layering/runtime-command-cutover-table.ts b/scripts/layering/runtime-command-cutover-table.ts index cfd16c248b..05dfeac696 100644 --- a/scripts/layering/runtime-command-cutover-table.ts +++ b/scripts/layering/runtime-command-cutover-table.ts @@ -29,6 +29,7 @@ import { retiredDispatchProjectionViolations } from './runtime-command-cutover-d * focus at R40, and type at R41 — the Wave 4 observation family is complete. Wave 5's generic * leaves follow: back at R42, home at R43, orientation at R44, tv-remote at R45, and the * action-selected keyboard at R46. + * The gesture cluster follows the touch leaves: gesture at R52, scroll at R53, swipe at R54. */ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ { @@ -972,6 +973,108 @@ export const MIGRATED_COMMAND_CUTOVERS: readonly MigratedCommandCutover[] = [ }, }, }, + { + rule: 'R52 gesture-runtime-cutover', + command: 'gesture', + subject: 'gesture execution', + tier: 'request-scoped', + execution: 'device-runtime', + legacyRetirement: { + // The whole admission and the plan dispatcher. `requireGestureSupported` was the last + // daemon-owned platform table in this family: its intent-dependent tiers are now owner + // facts, so the function and its five private helpers go with it. + routeNames: ['requireGestureSupported', 'dispatchGesturePlan'], + // `gesture` also leaves the hand-maintained overlay that granted it a capability bucket on + // a family the descriptor never listed. `dispatchGestureViewport` is deliberately NOT + // claimed here: the Maestro replay port still consumes it, and ADR 0019 §6 keeps a shared + // mechanic in place until its last consumer can move. + staticCommandSets: ['HARMONYOS_SUPPORTED_COMMANDS'], + }, + runtimeTypeNames: ['GestureRuntimeOperations', 'SnapshotRuntimeOperations'], + operations: { + names: [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'performMultiTouchGesturePlan', + 'performTargetAuthoredDrag', + 'captureSnapshot', + 'gestureViewport', + ], + }, + singularExecution: { + routes: ['handleInteractionCommands'], + operations: [ + 'performGesturePlan', + 'performDirectionalFlingPlan', + 'performMultiTouchGesturePlan', + 'performTargetAuthoredDrag', + 'captureSnapshot', + 'gestureViewport', + ], + // One lexical owner per tier, all four inside the single binder `swipe` shares (R54). + // Four keys rather than one because their CELLS differ, not their mechanics — the tier a + // gesture input selects is what admission proves and what that branch then calls, on a + // binding narrow enough that the operation needs no cast or non-null repair. + operationOwners: { + performGesturePlan: ['bindGestureTier'], + performDirectionalFlingPlan: ['bindGestureTier'], + performMultiTouchGesturePlan: ['bindGestureTier'], + performTargetAuthoredDrag: ['bindGestureTier'], + captureSnapshot: ['selectGestureFrame'], + gestureViewport: ['selectGestureFrame'], + }, + }, + }, + { + rule: 'R53 scroll-runtime-cutover', + command: 'scroll', + subject: 'directional scrolling', + tier: 'request-scoped', + execution: 'device-runtime', + legacyRetirement: { + // The interactor leaf and its dispatch-table arm. Deleting the descriptor's `dispatch` + // leaf drops `'scroll'` from `DescriptorDispatchCommandName`, which makes a surviving + // `DISPATCH_HANDLERS.scroll` a COMPILE error rather than something this row has to police. + routeNames: ['handleScrollCommand'], + staticCommandSets: ['HARMONYOS_SUPPORTED_COMMANDS'], + }, + runtimeTypeNames: ['ScrollRuntimeOperations'], + operations: { names: ['scrollDirection'] }, + singularExecution: { + routes: ['dispatchGenericCommand'], + operations: ['scrollDirection'], + // The edge verification consumes the SHARED `captureSnapshot` the selector family owns, so + // this row claims only the scroll pass itself. + operationOwners: { scrollDirection: ['scrollOnce'] }, + }, + }, + { + rule: 'R54 swipe-runtime-cutover', + command: 'swipe', + subject: 'coordinate swipe', + tier: 'request-scoped', + execution: 'device-runtime', + legacyRetirement: { + // `swipe` owned no adapter of its own: it normalized to a coordinate fling and executed + // through the same plan dispatcher `gesture` did (retired by R42). Its whole retirement is + // admission data — the capability bucket the row's automatic columns reject, plus the + // overlay membership below. + staticCommandSets: ['HARMONYOS_SUPPORTED_COMMANDS'], + }, + runtimeTypeNames: ['GestureRuntimeOperations', 'SnapshotRuntimeOperations'], + operations: { names: ['performGesturePlan', 'captureSnapshot', 'gestureViewport'] }, + singularExecution: { + routes: ['handleInteractionCommands'], + operations: ['performGesturePlan', 'captureSnapshot', 'gestureViewport'], + // Shared with `gesture` the way `find` shares the selector family's owners: a swipe series + // binds once and re-enters the same bound executor per repetition. + operationOwners: { + performGesturePlan: ['bindGestureTier'], + captureSnapshot: ['selectGestureFrame'], + gestureViewport: ['selectGestureFrame'], + }, + }, + }, ]; function snapshotRetiredDispatchProjectionProof( diff --git a/src/__tests__/eager-closure-budgets.ts b/src/__tests__/eager-closure-budgets.ts index 026a020010..686f303f11 100644 --- a/src/__tests__/eager-closure-budgets.ts +++ b/src/__tests__/eager-closure-budgets.ts @@ -122,7 +122,7 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ 'packages/contracts/src/app-inventory-runtime.ts': 1, 'packages/contracts/src/app-log-runtime.ts': 1, 'packages/contracts/src/app-state-runtime.ts': 1, - 'packages/contracts/src/apple-multitouch-support.ts': 5, + 'packages/contracts/src/apple-multitouch-support.ts': 6, 'packages/contracts/src/application-lifecycle-interaction.ts': 7, 'packages/contracts/src/application-lifecycle-runtime-plan.ts': 3, 'packages/contracts/src/application-lifecycle-runtime.ts': 1, @@ -144,7 +144,7 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ 'packages/contracts/src/facades/divergence.ts': 3, 'packages/contracts/src/facades/interaction.ts': 25, 'packages/contracts/src/facades/observability.ts': 7, - 'packages/contracts/src/facades/platform.ts': 48, + 'packages/contracts/src/facades/platform.ts': 51, 'packages/contracts/src/facades/progress.ts': 1, 'packages/contracts/src/facades/recording.ts': 3, 'packages/contracts/src/facades/remote.ts': 2, @@ -155,6 +155,8 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ 'packages/contracts/src/gesture-input.ts': 13, 'packages/contracts/src/gesture-normalization.ts': 14, 'packages/contracts/src/gesture-plan-types.ts': 1, + 'packages/contracts/src/gesture-admission.ts': 6, + 'packages/contracts/src/gesture-runtime.ts': 5, 'packages/contracts/src/gesture-plan.ts': 12, 'packages/contracts/src/interaction-error.ts': 1, 'packages/contracts/src/interaction-guarantees.ts': 1, @@ -166,7 +168,7 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ 'packages/contracts/src/platform-module.ts': 5, 'packages/contracts/src/platform-runtime-host.ts': 1, 'packages/contracts/src/platform-runtime-operations.ts': 2, - 'packages/contracts/src/platform-runtime-unavailable.ts': 21, + 'packages/contracts/src/platform-runtime-unavailable.ts': 23, 'packages/contracts/src/platform-runtime.ts': 6, 'packages/contracts/src/record-runtime-cutover.ts': 7, 'packages/contracts/src/screen-recording-runtime-plan.ts': 5, @@ -174,6 +176,7 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ 'packages/contracts/src/screenshot-runtime.ts': 4, 'packages/contracts/src/scroll-command.ts': 3, 'packages/contracts/src/scroll-gesture.ts': 10, + 'packages/contracts/src/scroll-runtime.ts': 4, 'packages/contracts/src/selector-observation-runtime.ts': 1, 'packages/contracts/src/settings.ts': 3, 'packages/contracts/src/snapshot-runtime.ts': 3, @@ -261,9 +264,9 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ */ export const HUB_BUDGETS: Readonly> = Object.freeze({ 'src/cli.ts': 362, - 'src/platform-runtime.ts': 37, - 'src/core/dispatch.ts': 88, - 'src/core/capabilities.ts': 76, + 'src/platform-runtime.ts': 39, + 'src/core/dispatch.ts': 83, + 'src/core/capabilities.ts': 75, 'src/core/command-descriptor/registry.ts': 66, 'src/core/command-descriptor/platform-execution-entry.ts': 3, 'src/core/interactors/register-builtins.ts': 73, diff --git a/src/__tests__/test-utils/runtime-operation-facts.ts b/src/__tests__/test-utils/runtime-operation-facts.ts index 71f2eaf7ca..2af8e143b2 100644 --- a/src/__tests__/test-utils/runtime-operation-facts.ts +++ b/src/__tests__/test-utils/runtime-operation-facts.ts @@ -1,7 +1,9 @@ import { applicationLifecycleOperationFacts } from '@agent-device/contracts/application-lifecycle-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { gestureRuntimeOperationFacts } from '@agent-device/contracts/gesture-runtime'; import type { RuntimeOperationFact } from '@agent-device/contracts/platform-runtime'; import { screenshotRuntimeOperationFacts } from '@agent-device/contracts/screenshot-runtime'; +import { scrollRuntimeOperationFacts } from '@agent-device/contracts/scroll-runtime'; import { snapshotRuntimeOperationFacts } from '@agent-device/contracts/snapshot-runtime'; import { touchRuntimeOperationFacts } from '@agent-device/contracts/touch-runtime'; @@ -44,6 +46,14 @@ export const unavailableDeploymentSnapshotAndShutdownOperationFacts = Object.fre fill: unavailable, tapElementSelector: unavailable, }), + ...gestureRuntimeOperationFacts({ + plan: unavailable, + directionalFling: unavailable, + multiTouch: unavailable, + targetAuthoredDrag: unavailable, + viewport: unavailable, + }), + ...scrollRuntimeOperationFacts({ scroll: unavailable }), ...elementTextRuntimeOperationFacts({ readTextAtPoint: unavailable }), back: unavailable, home: unavailable, diff --git a/src/core/__tests__/capabilities.test.ts b/src/core/__tests__/capabilities.test.ts index 0fdc64bc00..07f4eb4f48 100644 --- a/src/core/__tests__/capabilities.test.ts +++ b/src/core/__tests__/capabilities.test.ts @@ -293,6 +293,12 @@ test('web supports only the initial browser interaction slice', () => { 'fill', 'focus', 'find', + // `gesture` and `swipe` (R42/R44) join the migrated commands here for the same reason + // `focus`, `find`, `screenshot`, `scroll`, `snapshot`, `type` and `wait` already do: a + // command whose admission comes from exact owner facts carries no capability-matrix row, + // and a command with no row is not decided by this matrix at all. The web owner refuses + // every gesture tier — `platform-web/src/runtime.test.ts` is where that cell is pinned. + 'gesture', 'get', 'hover', 'press', @@ -300,22 +306,14 @@ test('web supports only the initial browser interaction slice', () => { 'screenshot', 'scroll', 'snapshot', + 'swipe', 'type', 'wait', ], [{ device: webDevice, expected: true, label: 'on web' }], ); assertCommandSupport( - [ - 'alert', - 'app-switcher', - 'clipboard', - 'gesture', - 'perf', - 'settings', - 'swipe', - 'trigger-app-event', - ], + ['alert', 'app-switcher', 'clipboard', 'perf', 'settings', 'trigger-app-event'], [{ device: webDevice, expected: false, label: 'on web' }], ); assertCommandSupport( diff --git a/src/core/__tests__/capability-plugin-routing-parity.test.ts b/src/core/__tests__/capability-plugin-routing-parity.test.ts index cb6fa82d90..107d229087 100644 --- a/src/core/__tests__/capability-plugin-routing-parity.test.ts +++ b/src/core/__tests__/capability-plugin-routing-parity.test.ts @@ -238,14 +238,7 @@ test('HarmonyOS static capabilities omit runtime-backed command admissions', () // Runtime-backed navigation, keyboard, and touch commands dropped out of the matrix entirely: // capability buckets), so they are absent here — not because HarmonyOS admission changed, but // because there is no bucket left for `isCommandSupportedOnDevice` to consult at all. - assert.deepEqual(availableCommands, [ - 'app-switcher', - 'gesture', - 'perf', - 'scroll', - 'settings', - 'swipe', - ]); + assert.deepEqual(availableCommands, ['app-switcher', 'perf', 'settings']); }); test('(b.2) unsupportedHint closures are verbatim across the full device matrix', () => { diff --git a/src/core/__tests__/dispatch-scroll.test.ts b/src/core/__tests__/dispatch-scroll.test.ts deleted file mode 100644 index 67189b0ab0..0000000000 --- a/src/core/__tests__/dispatch-scroll.test.ts +++ /dev/null @@ -1,219 +0,0 @@ -import { test } from 'vitest'; -import assert from 'node:assert/strict'; -import { dispatchCommand } from '../dispatch.ts'; -import { handleScrollCommand } from '../dispatch-scroll.ts'; -import { AppError } from '@agent-device/kernel/errors'; -import type { Interactor } from '@agent-device/contracts/interaction'; -import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; - -test('dispatch scroll rejects mixing amount and --pixels', async () => { - await assert.rejects( - () => dispatchCommand(IOS_SIMULATOR, 'scroll', ['down', '0.4'], undefined, { pixels: 240 }), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /either a relative amount or --pixels/i.test(error.message), - ); -}); - -test('dispatch scroll forwards pixels and duration without reporting ignored duration', async () => { - const calls: Array<{ direction: string; options: unknown }> = []; - const interactor = { - scroll: async (direction: any, options: unknown) => { - calls.push({ direction, options }); - return { ok: true }; - }, - } as unknown as Interactor; - - const result = await handleScrollCommand(interactor, ['down'], { - pixels: 200, - durationMs: 50, - }); - - assert.deepEqual(calls, [ - { - direction: 'down', - options: { - amount: undefined, - pixels: 200, - durationMs: 50, - releaseBehavior: 'controlled', - }, - }, - ]); - assert.equal(result.pixels, 200); - assert.equal(result.durationMs, undefined); -}); - -test('dispatch scroll reports duration when the interactor honored it', async () => { - const interactor = { - scroll: async () => ({ pixels: 200, durationMs: 50 }), - } as unknown as Interactor; - - const result = await handleScrollCommand(interactor, ['down'], { - pixels: 200, - durationMs: 50, - }); - - assert.equal(result.pixels, 200); - assert.equal(result.durationMs, 50); -}); - -test('dispatch scroll rejects duration above the shared cap', async () => { - const interactor = { - scroll: async () => { - throw new Error('scroll should be rejected before backend call'); - }, - } as unknown as Interactor; - - await assert.rejects( - () => handleScrollCommand(interactor, ['down'], { pixels: 200, durationMs: 10_001 }), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /durationMs.*at most 10000/i.test(error.message), - ); -}); - -test('dispatch scroll bottom rejects blind scrolling without snapshot support', async () => { - const calls: Array<{ direction: string; options: unknown }> = []; - const interactor = { - scroll: async (direction: any, options: unknown) => { - calls.push({ direction, options }); - return { lastPass: calls.length }; - }, - } as unknown as Interactor; - - await assert.rejects( - () => handleScrollCommand(interactor, ['bottom'], undefined), - (error: unknown) => - error instanceof AppError && - error.code === 'UNSUPPORTED_OPERATION' && - /requires snapshot support/i.test(error.message), - ); - - assert.equal(calls.length, 0); -}); - -test('dispatch scroll bottom does not scroll when no hidden content is below', async () => { - const calls: Array<{ direction: string; options: unknown }> = []; - const interactor = { - scroll: async (direction: any, options: unknown) => { - calls.push({ direction, options }); - return { lastPass: calls.length }; - }, - snapshot: async () => makeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), - } as unknown as Interactor; - - const result = await handleScrollCommand(interactor, ['bottom'], undefined); - - assert.equal(calls.length, 0); - assert.equal(result.direction, 'down'); - assert.equal(result.edge, 'bottom'); - assert.equal(result.passes, 0); - assert.match(String(result.message), /Already at bottom/); -}); - -test('dispatch scroll bottom scrolls only while scoped snapshot confirms hidden content', async () => { - const calls: Array<{ direction: string; options: unknown }> = []; - const snapshotScopes: unknown[] = []; - const snapshots = [ - makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), - makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), - makeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), - ]; - const interactor = { - scroll: async (direction: any, options: unknown) => { - calls.push({ direction, options }); - return { lastPass: calls.length }; - }, - snapshot: async (options: any) => { - snapshotScopes.push(options.scope); - return snapshots[Math.min(snapshotScopes.length - 1, snapshots.length - 1)]; - }, - } as unknown as Interactor; - - const result = await handleScrollCommand(interactor, ['bottom'], undefined); - - assert.equal(calls.length, 1); - assert.deepEqual(calls[0], { - direction: 'down', - options: { - amount: undefined, - pixels: undefined, - durationMs: undefined, - releaseBehavior: 'inertial', - }, - }); - assert.equal(result.passes, 1); - assert.equal(result.lastPass, 1); - assert.deepEqual(snapshotScopes, [undefined, 'Messages', 'Messages']); -}); - -test('dispatch scroll bottom tolerates unchanged signatures while hidden content advances', async () => { - const calls: Array<{ direction: string; options: unknown }> = []; - const snapshots = [ - makeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - makeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - makeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), - makeScrollSnapshot({ hiddenBelow: false, message: 'Repeated row' }), - ]; - let snapshotIndex = 0; - const interactor = { - scroll: async (direction: any, options: unknown) => { - calls.push({ direction, options }); - return { lastPass: calls.length }; - }, - snapshot: async () => snapshots[Math.min(snapshotIndex++, snapshots.length - 1)], - } as unknown as Interactor; - - const result = await handleScrollCommand(interactor, ['bottom'], undefined); - - assert.equal(calls.length, 2); - assert.equal(result.passes, 2); -}); - -test('dispatch scroll bottom keeps scoped snapshot failures scoped', async () => { - let snapshotCount = 0; - const interactor = { - scroll: async () => ({}), - snapshot: async (options: any) => { - snapshotCount += 1; - if (options.scope) throw new Error('scoped snapshot failed'); - return makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }); - }, - } as unknown as Interactor; - - await assert.rejects( - () => handleScrollCommand(interactor, ['bottom'], undefined), - (error: unknown) => - error instanceof AppError && - error.code === 'COMMAND_FAILED' && - /scoped container/i.test(error.message) && - error.details?.scope === 'Messages', - ); - assert.equal(snapshotCount, 2); -}); - -function makeScrollSnapshot(options: { hiddenBelow: boolean; message: string }) { - return { - backend: 'xctest' as const, - nodes: [ - { - index: 1, - type: 'ScrollView', - label: 'Messages', - hiddenContentBelow: options.hiddenBelow ? true : undefined, - rect: { x: 0, y: 100, width: 400, height: 600 }, - }, - { - index: 2, - parentIndex: 1, - type: 'Button', - label: options.message, - rect: { x: 0, y: 640, width: 400, height: 56 }, - }, - ], - truncated: false, - }; -} diff --git a/src/core/__tests__/gesture-capabilities.test.ts b/src/core/__tests__/gesture-capabilities.test.ts deleted file mode 100644 index ab9962c883..0000000000 --- a/src/core/__tests__/gesture-capabilities.test.ts +++ /dev/null @@ -1,145 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import type { - GestureCommandInput, - GestureSemanticInput, -} from '@agent-device/contracts/interaction'; -import { - normalizePublicGesture, - normalizePublicSwipeMotion, -} from '@agent-device/contracts/gesture-normalization'; -import { requireGestureSupported } from '../capabilities.ts'; -import { AppError } from '@agent-device/kernel/errors'; -import type { DeviceInfo } from '@agent-device/kernel/device'; - -const oneFingerPan: GestureSemanticInput = { - intent: 'pan', - origin: { x: 100, y: 200 }, - delta: { x: 40, y: -20 }, -}; -const twoFingerPan: GestureSemanticInput = { ...oneFingerPan, pointerCount: 2 }; -const pinch: GestureSemanticInput = { intent: 'pinch', scale: 1.2 }; -const fling: GestureSemanticInput = { - intent: 'fling', - direction: 'left', - origin: { x: 100, y: 200 }, -}; -const drag: GestureCommandInput = { - intent: 'drag', - source: 'id="source"', - destination: 'id="destination"', -}; - -const device = (fields: Partial): DeviceInfo => ({ - platform: 'android', - id: 'test-device', - name: 'Test device', - kind: 'emulator', - ...fields, -}); - -function assertSupported(input: GestureCommandInput, target: DeviceInfo): void { - assert.doesNotThrow(() => requireGestureSupported(input, target)); -} - -function assertUnsupported(input: GestureCommandInput, target: DeviceInfo, expected: RegExp): void { - assert.throws( - () => requireGestureSupported(input, target), - (error: unknown) => - error instanceof AppError && - error.code === 'UNSUPPORTED_OPERATION' && - expected.test(error.message), - ); -} - -test('Android phones and emulators support single- and multi-touch gesture plans', () => { - for (const kind of ['device', 'emulator'] as const) { - const target = device({ kind }); - assertSupported(oneFingerPan, target); - assertSupported(twoFingerPan, target); - assertSupported(pinch, target); - } -}); - -test('target-authored drag is admitted only where adapters preserve every authored phase', () => { - for (const kind of ['device', 'emulator'] as const) { - assertSupported(drag, device({ kind, target: 'mobile' })); - } - for (const appleOs of ['ios', 'ipados'] as const) { - for (const kind of ['device', 'simulator'] as const) { - assertSupported(drag, device({ platform: 'apple', appleOs, kind, target: 'mobile' })); - } - } - assertSupported(drag, device({ platform: 'apple', kind: 'simulator', target: 'mobile' })); - - const inexactBackends = [ - device({ target: 'tv' }), - device({ platform: 'apple', appleOs: 'tvos', kind: 'simulator', target: 'tv' }), - device({ platform: 'apple', appleOs: 'macos', kind: 'device', target: 'desktop' }), - device({ platform: 'apple', appleOs: 'visionos', kind: 'simulator' }), - device({ platform: 'apple', appleOs: 'watchos', kind: 'simulator' }), - device({ platform: 'linux', kind: 'device', target: 'desktop' }), - device({ platform: 'vega', kind: 'device', target: 'tv' }), - device({ platform: 'web', kind: 'device', target: 'desktop' }), - ]; - for (const target of inexactBackends) { - assert.throws( - () => requireGestureSupported(drag, target), - (error: unknown) => - error instanceof AppError && - error.code === 'UNSUPPORTED_OPERATION' && - error.details?.gesture === 'drag' && - /source hold, timed movement, and destination hold/.test(String(error.details?.hint)), - ); - } -}); - -test('iOS and iPadOS simulators support multi-touch while physical devices do not', () => { - for (const appleOs of ['ios', 'ipados'] as const) { - const simulator = device({ platform: 'apple', appleOs, kind: 'simulator' }); - const physical = device({ platform: 'apple', appleOs, kind: 'device' }); - assertSupported(oneFingerPan, simulator); - assertSupported(twoFingerPan, simulator); - assertSupported(pinch, simulator); - assertSupported(oneFingerPan, physical); - assertUnsupported(twoFingerPan, physical, /physical iOS devices/); - assert.throws( - () => requireGestureSupported(pinch, physical), - (error: unknown) => - error instanceof AppError && /iOS-simulator only/.test(String(error.details?.hint)), - ); - } -}); - -test('TV, spatial, watch, desktop, Linux, and web gesture policy stays explicit', () => { - const androidTv = device({ target: 'tv' }); - const tvOs = device({ platform: 'apple', appleOs: 'tvos', kind: 'simulator', target: 'tv' }); - const visionOs = device({ platform: 'apple', appleOs: 'visionos', kind: 'simulator' }); - const watchOs = device({ platform: 'apple', appleOs: 'watchos', kind: 'simulator' }); - const macOs = device({ platform: 'apple', appleOs: 'macos', kind: 'device', target: 'desktop' }); - const linux = device({ platform: 'linux', kind: 'device', target: 'desktop' }); - const web = device({ platform: 'web', kind: 'device', target: 'desktop' }); - - assertSupported(oneFingerPan, androidTv); - assertUnsupported(twoFingerPan, androidTv, /Android TV/); - assert.throws( - () => requireGestureSupported(twoFingerPan, androidTv), - (error: unknown) => - error instanceof AppError && - /Android TV has no touch input/.test(String(error.details?.hint)), - ); - assertUnsupported(twoFingerPan, tvOs, /tvOS/); - assertUnsupported(twoFingerPan, visionOs, /visionOS/i); - assertUnsupported(oneFingerPan, watchOs, /watchos/); - assertSupported(oneFingerPan, macOs); - assertUnsupported(twoFingerPan, macOs, /macOS/); - assertSupported(oneFingerPan, linux); - assertSupported( - normalizePublicSwipeMotion({ from: { x: 10, y: 20 }, to: { x: 110, y: 20 } }).gesture, - linux, - ); - assertSupported(normalizePublicGesture({ kind: 'swipe', preset: 'left' }).gesture, linux); - assertUnsupported(fling, linux, /Linux/); - assertUnsupported(twoFingerPan, linux, /linux/i); - assertUnsupported(oneFingerPan, web, /web/); -}); diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index 8d85e823b0..189e64e2c2 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -3,12 +3,6 @@ import { commandDescriptors } from './command-descriptor/registry.ts'; import { tryGetPlugin } from './platform-plugin-registry.ts'; import { registerBuiltinPlatformPlugins } from './interactors/register-builtins.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { AppError } from '@agent-device/kernel/errors'; -import type { - GestureCommandInput, - GestureSemanticInput, -} from '@agent-device/contracts/interaction'; -import { assertAppleMultiTouchSupported } from '@agent-device/contracts/apple-multitouch-support'; // Populate the PlatformPlugin registry once at module load (idempotent; registers // only lazy closures, so no leaf code is imported and CLI cold-start is unaffected @@ -36,16 +30,9 @@ export type CommandCapability = { const WEB_DEVICE: KindMatrix = { device: true }; const HARMONYOS_ALL: KindMatrix = { emulator: true, device: true }; -const HARMONYOS_SUPPORTED_COMMANDS = new Set([ - 'perf', - 'app-switcher', - 'gesture', - 'scroll', - 'settings', - 'swipe', -]); +const HARMONYOS_SUPPORTED_COMMANDS = new Set(['perf', 'app-switcher', 'settings']); const WEB_QUERY_COMMANDS = ['audio'] as const; -const WEB_SUPPORTED_COMMANDS = new Set([...WEB_QUERY_COMMANDS, 'scroll']); +const WEB_SUPPORTED_COMMANDS = new Set(WEB_QUERY_COMMANDS); // Built from the additive command-descriptor registry (ADR-0008, Phase 1 step 3). // The hand-authored literal was deleted after #906 proved deriveCapabilityMatrix is // byte-equal to it (platform/kind buckets). The per-command `supports()` / @@ -160,77 +147,3 @@ export function supportedPlatformsForCommand(command: string): string[] { } return supported; } - -export function requireGestureSupported(input: GestureCommandInput, device: DeviceInfo): void { - if (input.intent === 'drag') { - requireTargetAuthoredDragSupported(input, device); - return; - } - if (device.platform === 'web' || device.appleOs === 'watchos') { - throw unsupportedGesture(input, gesturePlatformMessage(input, device)); - } - if (isMultiTouchGesture(input)) { - requireMultiTouchGestureSupported(input, device); - return; - } - if (device.appleOs === 'visionos') { - throw unsupportedGesture(input, gesturePlatformMessage(input, device)); - } - // Linux can preserve public coordinate/preset swipe through its drag primitive, but cannot - // honor the speed semantics authored by `gesture fling`. - if (input.intent === 'fling' && 'direction' in input && device.platform === 'linux') { - throw unsupportedGesture(input, 'gesture fling is not supported on Linux'); - } -} - -function requireTargetAuthoredDragSupported( - input: Extract, - device: DeviceInfo, -): void { - if (supportsTargetAuthoredDrag(device)) return; - throw unsupportedGesture( - input, - gesturePlatformMessage(input, device), - 'Target-authored drag requires an adapter that preserves source hold, timed movement, and destination hold; it is supported on Android touch devices and iOS/iPadOS.', - ); -} - -function supportsTargetAuthoredDrag(device: DeviceInfo): boolean { - if (device.platform === 'android') { - return device.target !== 'tv'; - } - if (device.platform !== 'apple') return false; - if (device.appleOs === undefined) return device.target !== 'desktop' && device.target !== 'tv'; - return device.appleOs === 'ios' || device.appleOs === 'ipados'; -} - -function isMultiTouchGesture(input: GestureSemanticInput): boolean { - if (input.intent === 'pan') return ('pointerCount' in input ? input.pointerCount : 1) === 2; - return input.intent === 'pinch' || input.intent === 'rotate' || input.intent === 'transform'; -} - -function requireMultiTouchGestureSupported(input: GestureSemanticInput, device: DeviceInfo): void { - if (device.platform === 'android') { - if (device.target !== 'tv') return; - throw unsupportedGesture( - input, - `gesture ${input.intent} is not supported on Android TV`, - 'Android TV has no touch input — this gesture is supported on Android phones, tablets, and the iOS simulator only.', - ); - } - if (device.platform !== 'apple') { - throw unsupportedGesture(input, gesturePlatformMessage(input, device)); - } - assertAppleMultiTouchSupported(device, input.intent); -} - -function gesturePlatformMessage(input: GestureCommandInput, device: DeviceInfo): string { - return `gesture ${input.intent} is not supported on ${device.appleOs ?? device.platform}`; -} - -function unsupportedGesture(input: GestureCommandInput, message: string, hint?: string): AppError { - return new AppError('UNSUPPORTED_OPERATION', message, { - gesture: input.intent, - ...(hint ? { hint } : {}), - }); -} diff --git a/src/core/command-descriptor/__tests__/gesture-runtime-execution.test.ts b/src/core/command-descriptor/__tests__/gesture-runtime-execution.test.ts new file mode 100644 index 0000000000..22c82af0a6 --- /dev/null +++ b/src/core/command-descriptor/__tests__/gesture-runtime-execution.test.ts @@ -0,0 +1,23 @@ +import { + gestureRuntimePlanUses, + swipeRuntimePlanUses, +} from '@agent-device/contracts/platform-runtime-operations'; +import { expect, test } from 'vitest'; +import { commandDescriptors } from '../registry.ts'; + +test('gesture and swipe descriptors declare only the runtime uses they can select', () => { + const gesture = commandDescriptors.find(({ name }) => name === 'gesture'); + const swipe = commandDescriptors.find(({ name }) => name === 'swipe'); + + expect(gesture?.platformExecution).toEqual({ + kind: 'device-runtime', + uses: gestureRuntimePlanUses, + }); + expect(swipe?.platformExecution).toEqual({ + kind: 'device-runtime', + uses: swipeRuntimePlanUses, + }); + expect(swipeRuntimePlanUses.map(({ required }) => required)).toEqual([ + ['performGesturePlan', 'captureSnapshot'], + ]); +}); diff --git a/src/core/command-descriptor/__tests__/parity.test.ts b/src/core/command-descriptor/__tests__/parity.test.ts index a853430ee8..7ef469a2a6 100644 --- a/src/core/command-descriptor/__tests__/parity.test.ts +++ b/src/core/command-descriptor/__tests__/parity.test.ts @@ -60,6 +60,7 @@ const NO_CAPABILITY_PUBLIC_COMMANDS = new Set([ PUBLIC_COMMANDS.events, PUBLIC_COMMANDS.find, PUBLIC_COMMANDS.focus, + PUBLIC_COMMANDS.gesture, PUBLIC_COMMANDS.get, PUBLIC_COMMANDS.home, PUBLIC_COMMANDS.install, @@ -75,9 +76,11 @@ const NO_CAPABILITY_PUBLIC_COMMANDS = new Set([ PUBLIC_COMMANDS.record, PUBLIC_COMMANDS.reinstall, PUBLIC_COMMANDS.replay, + PUBLIC_COMMANDS.scroll, PUBLIC_COMMANDS.shutdown, PUBLIC_COMMANDS.screenshot, PUBLIC_COMMANDS.snapshot, + PUBLIC_COMMANDS.swipe, PUBLIC_COMMANDS.test, PUBLIC_COMMANDS.trace, PUBLIC_COMMANDS.tvRemote, @@ -220,6 +223,9 @@ test('generic route commands that reach platform dispatch declare the dispatch f PUBLIC_COMMANDS.gesture, PUBLIC_COMMANDS.focus, PUBLIC_COMMANDS.screenshot, + // R43 retired scroll's dispatch leaf with its capability bucket: the bound + // `scrollDirection` operation is its only execution. + PUBLIC_COMMANDS.scroll, PUBLIC_COMMANDS.viewport, PUBLIC_COMMANDS.back, PUBLIC_COMMANDS.home, @@ -341,7 +347,16 @@ test('capability-checked command list is built from descriptor capabilities', () false, 'snapshot admission comes from exact device-runtime facts', ); - assert.ok(expectedNames.has(PUBLIC_COMMANDS.gesture), 'gesture remains capability-checked'); + assert.equal( + expectedNames.has(PUBLIC_COMMANDS.gesture), + false, + 'gesture admission comes from exact device-runtime facts', + ); + assert.equal( + expectedNames.has(PUBLIC_COMMANDS.click), + false, + 'click admission comes from exact device-runtime facts', + ); assert.equal( expectedNames.has(PUBLIC_COMMANDS.capabilities), false, diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index 77885d753c..093feb946a 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -1,4 +1,3 @@ -import type { CommandCapability } from '../capabilities.ts'; // The typed-flags request from contracts/, not the daemon's server-side refinement: these // descriptors read `command`, `positionals` and `flags` and never touch `internal`. import type { DispatchedCommand } from '@agent-device/contracts/command'; @@ -36,6 +35,7 @@ import { fillRuntimeUses, findRuntimePlanUses, focusRuntimeUse, + gestureRuntimePlanUses, homeRuntimeUse, hoverRuntimeUses, keyboardRuntimePlanUses, @@ -43,6 +43,8 @@ import { orientationRuntimeUse, pressRuntimeUses, screenshotRuntimePlanUses, + scrollRuntimePlanUses, + swipeRuntimePlanUses, selectorCaptureRuntimePlanUses, selectorTextCaptureRuntimePlanUses, shutdownTargetUse, @@ -241,11 +243,6 @@ const ANDROID_ALL = { emulator: true, device: true, unknown: true }; const LINUX_DEVICE = { device: true }; const LINUX_NONE = {}; -const ALL_DEVICE_COMMAND_CAPABILITY = { - apple: APPLE_SIM_AND_DEVICE, - android: ANDROID_ALL, - linux: LINUX_DEVICE, -} satisfies CommandCapability; // --------------------------------------------------------------------------- // ADR 0019 §6 platform-execution modes. Every descriptor declares one; there is // no registry-entry default (see `readDeclaredPlatformExecution`). @@ -261,10 +258,9 @@ const NO_PLATFORM_EXECUTION = { kind: 'none' } as const; const LEGACY_PLATFORM_EXECUTION = { kind: 'legacy' } as const; /** - * The daemon/recording traits every generic-route mutating command shares, migrated or not. - * Split from the legacy execution pair (`dispatch`/`capability`, see - * {@link LEGACY_LINUX_DEVICE_EXECUTION}) so a migrated descriptor spreads this alone instead of - * hand-expanding it minus the two fields migration strips. + * The daemon/recording traits every generic-route mutating command shares. The legacy execution + * pair it was split from (`dispatch`/`capability`) is gone: `scroll` was its last consumer, and + * R53 migrated it, so every generic-route mutating command now spreads this alone. */ const GENERIC_MUTATING_COMMAND_TRAITS = { recordsSessionAction: true, @@ -287,19 +283,6 @@ const GENERIC_MUTATING_COMMAND_TRAITS = { | 'batchable' >; -/** - * The legacy `dispatch`/`capability` pair a still-unmigrated generic-route mutating command - * carries alongside {@link GENERIC_MUTATING_COMMAND_TRAITS}; migration strips both together (one - * owner fact replaces the capability bucket, one bound operation replaces the dispatch leaf). - */ -const LEGACY_LINUX_DEVICE_EXECUTION = { - dispatch: {}, - capability: { apple: APPLE_SIM_AND_DEVICE, android: ANDROID_ALL, linux: LINUX_DEVICE }, -} as const satisfies Pick< - Extract, - 'dispatch' | 'capability' ->; - // click/fill/press/longpress differ only in their timeout budget and response // shaping: same owner file, same pre-dispatch target identity, same interaction // route and dialog guard, same device buckets, and the same session-bound claim @@ -1268,10 +1251,12 @@ export const RAW_COMMAND_DESCRIPTORS = [ refFrameEffect: 'may-invalidate', androidBlockingDialogGuard: true, }, - capability: ALL_DEVICE_COMMAND_CAPABILITY, + // R52 retires this command's capability bucket: admission is the owner's gesture-tier facts, + // which the retired `requireGestureSupported` used to decide inside the daemon. The declared + // uses are the four tiers one gesture input can select between (ADR 0019 §9). timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, - platformExecution: LEGACY_PLATFORM_EXECUTION, + platformExecution: { kind: 'device-runtime', uses: gestureRuntimePlanUses }, }, { name: 'home', @@ -1305,14 +1290,16 @@ export const RAW_COMMAND_DESCRIPTORS = [ }, { name: 'scroll', - ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/interaction/index.ts'] as const } : {}), + ...(ownerFilesEnabled ? { ownerFiles: ['src/daemon/scroll-runtime.ts'] as const } : {}), catalog: { group: 'public' }, frameworkTier: 'core', + // R53 retires this command's capability bucket and its `dispatch` leaf together: admission is + // the owner's `scrollDirection` fact, and the only execution is the bound operation. `scroll` + // was the last holder of the legacy `dispatch`/`capability` pair, which retires with it. ...GENERIC_MUTATING_COMMAND_TRAITS, - ...LEGACY_LINUX_DEVICE_EXECUTION, timeoutPolicy: postActionObservationTimeoutPolicy('scroll', DEFAULT_TIMEOUT_POLICY), postActionObservation: postActionObservation('scroll'), - platformExecution: LEGACY_PLATFORM_EXECUTION, + platformExecution: { kind: 'device-runtime', uses: scrollRuntimePlanUses }, }, { name: 'swipe', @@ -1327,10 +1314,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ refFrameEffect: 'may-invalidate', androidBlockingDialogGuard: true, }, - capability: { apple: APPLE_SIM_AND_DEVICE, android: ANDROID_ALL, linux: LINUX_DEVICE }, + // R54 retires this command's capability bucket. A swipe always normalizes to a coordinate + // fling, so it declares only the one-contact plan it can select. timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, - platformExecution: LEGACY_PLATFORM_EXECUTION, + platformExecution: { kind: 'device-runtime', uses: swipeRuntimePlanUses }, }, { name: 'focus', diff --git a/src/core/dispatch-scroll.ts b/src/core/dispatch-scroll.ts deleted file mode 100644 index 1d7f43a624..0000000000 --- a/src/core/dispatch-scroll.ts +++ /dev/null @@ -1,131 +0,0 @@ -import type { Interactor } from '@agent-device/contracts/interactor-types'; -import { - type ResolvedScrollExecutionOptions, - type ScrollCommandOptions, - assertExclusiveScrollDistanceInputs, - honoredScrollDurationMs, - normalizeScrollDurationMs, - resolveScrollExecutionOptions, -} from '@agent-device/contracts/scroll-command'; -import { type ScrollDirection, parseScrollDirection } from '@agent-device/contracts/scroll-gesture'; -import { AppError } from '@agent-device/kernel/errors'; -import { - captureScrollEdgeState, - formatScrollEdgeMessage, - runScrollEdgePasses, - type ScrollEdge, - type ScrollEdgeState, -} from '../utils/scroll-edge-state.ts'; -import { withSuccessText } from '../utils/success-text.ts'; -import type { DispatchContext } from './dispatch-context.ts'; - -type ScrollTarget = { direction: ScrollDirection; edge?: ScrollEdge }; - -export async function handleScrollCommand( - interactor: Interactor, - positionals: string[], - context: DispatchContext | undefined, -): Promise> { - const directionInput = positionals[0]; - const amount = positionals[1] ? Number(positionals[1]) : undefined; - const pixels = context?.pixels; - const durationMs = context?.durationMs; - if (!directionInput) throw new AppError('INVALID_ARGS', 'scroll requires direction'); - assertScrollCommandInputs(amount, pixels, durationMs); - - const target = parseScrollTarget(directionInput); - const options = resolveScrollExecutionOptions({ amount, pixels, durationMs }, target.edge); - const { interactionResult, completedPasses } = await runDispatchedScroll( - interactor, - context, - target, - options, - ); - const result = buildDispatchedScrollResult(target, options, completedPasses, interactionResult); - return withSuccessText( - result, - formatScrollEdgeMessage(target.direction, target.edge, completedPasses, amount, pixels), - ); -} - -function assertScrollCommandInputs( - amount: number | undefined, - pixels: number | undefined, - durationMs: number | undefined, -): void { - if (amount !== undefined && !Number.isFinite(amount)) { - throw new AppError('INVALID_ARGS', 'scroll amount must be a number'); - } - normalizeScrollDurationMs(durationMs); - assertExclusiveScrollDistanceInputs({ amount, pixels }); -} - -async function runDispatchedScroll( - interactor: Interactor, - context: DispatchContext | undefined, - target: ScrollTarget, - options: ResolvedScrollExecutionOptions, -): Promise<{ interactionResult: Record; completedPasses: number }> { - if (target.edge) { - const edgeResult = await runScrollEdgePasses({ - edge: target.edge, - captureState: async (scope) => - await captureVerifiedScrollEdgeState(interactor, context, target.edge!, scope), - scroll: async () => await interactor.scroll(target.direction, options), - }); - return { interactionResult: edgeResult.result ?? {}, completedPasses: edgeResult.passes }; - } - return { - interactionResult: (await interactor.scroll(target.direction, options)) ?? {}, - completedPasses: 1, - }; -} - -function buildDispatchedScrollResult( - target: ScrollTarget, - options: ScrollCommandOptions, - completedPasses: number, - interactionResult: Record, -): Record { - const durationMs = honoredScrollDurationMs(interactionResult); - return { - direction: target.direction, - ...(target.edge ? { edge: target.edge, passes: completedPasses } : {}), - ...(options.amount !== undefined ? { amount: options.amount } : {}), - ...(options.pixels !== undefined ? { pixels: options.pixels } : {}), - ...(durationMs !== undefined ? { durationMs } : {}), - ...interactionResult, - }; -} - -async function captureVerifiedScrollEdgeState( - interactor: Interactor, - context: DispatchContext | undefined, - edge: ScrollEdge, - scope?: string, -): Promise { - if (typeof interactor.snapshot !== 'function') { - throw new AppError( - 'UNSUPPORTED_OPERATION', - `scroll ${edge} requires snapshot support to verify hidden content before scrolling`, - ); - } - const snapshot = interactor.snapshot; - return await captureScrollEdgeState({ - edge, - scope, - captureNodes: async (snapshotScope) => - ( - await snapshot({ - appBundleId: context?.appBundleId, - scope: snapshotScope, - }) - ).nodes ?? [], - }); -} - -function parseScrollTarget(input: string): ScrollTarget { - if (input === 'bottom') return { direction: 'down', edge: 'bottom' }; - if (input === 'top') return { direction: 'up', edge: 'top' }; - return { direction: parseScrollDirection(input) }; -} diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index eea84f813e..37d91a4a7d 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -1,4 +1,4 @@ -import type { GesturePlan, Interactor, RunnerContext } from '@agent-device/contracts/interaction'; +import type { Interactor, RunnerContext } from '@agent-device/contracts/interaction'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import type { Rect } from '@agent-device/kernel/snapshot'; @@ -8,7 +8,6 @@ import { successText, withSuccessText } from '../utils/success-text.ts'; import { parseTriggerAppEventArgs, resolveAppEventUrl } from './app-events.ts'; import type { DescriptorDispatchCommandName } from './command-descriptor/registry.ts'; import type { DispatchContext } from './dispatch-context.ts'; -import { handleScrollCommand } from './dispatch-scroll.ts'; import { getInteractor } from './interactors.ts'; export type { DispatchContext } from './dispatch-context.ts'; @@ -72,18 +71,6 @@ async function dispatchWithInteractor( ); } -export async function dispatchGesturePlan( - device: DeviceInfo, - plan: GesturePlan, - context?: DispatchContext, -): Promise | void> { - const interactor = await getInteractor(device, runnerContextFromDispatchContext(context)); - if (!interactor.performGesture) { - throw new AppError('UNSUPPORTED_OPERATION', 'Gesture execution is unavailable'); - } - return await interactor.performGesture(plan); -} - export async function dispatchGestureViewport( device: DeviceInfo, context?: DispatchContext, @@ -129,8 +116,6 @@ type DispatchHandler = (args: DispatchHandlerArgs) => Promise = { - scroll: ({ interactor, positionals, context }) => - handleScrollCommand(interactor, positionals, context), 'trigger-app-event': ({ device, interactor, positionals, context }) => handleTriggerAppEventCommand(device, interactor, positionals, context), 'app-switcher': async ({ interactor }) => { diff --git a/src/daemon/__tests__/gesture-admission-parity.test.ts b/src/daemon/__tests__/gesture-admission-parity.test.ts new file mode 100644 index 0000000000..fff9c0cadf --- /dev/null +++ b/src/daemon/__tests__/gesture-admission-parity.test.ts @@ -0,0 +1,178 @@ +import { expect, test } from 'vitest'; +import type { + GestureCommandInput, + GestureSemanticInput, +} from '@agent-device/contracts/gesture-plan-types'; +import { + normalizePublicGesture, + normalizePublicSwipeMotion, +} from '@agent-device/contracts/gesture-normalization'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { createPlatformRuntimeGateway } from '../../platform-runtime.ts'; +import { createRequestRuntimeBindings } from '../request-runtime-binding.ts'; +import { resolveBoundGestureRuntime } from '../gesture-runtime.ts'; + +/** + * The parity artifact for R42/R44 (ADR 0019 §6). + * + * This is the retired `requireGestureSupported` suite's device matrix, re-pointed at what + * replaced it: the REAL composed runtime gateway's owner facts, admitted through the real daemon + * gesture admission. Every assertion below — admitted or refused, message and hint — is the + * behavior `main` produced before the cutover, so a fact cell that drifts from the admission it + * restates fails here rather than on a device. + */ +const gateway = createPlatformRuntimeGateway({ + resolveSessionArtifacts: () => ({ + outputPath: '/sessions/parity/app.log', + pidPath: '/sessions/parity/app-log.pid', + }), + sessionsDir: '/sessions', +}); + +const oneFingerPan: GestureSemanticInput = { + intent: 'pan', + origin: { x: 100, y: 200 }, + delta: { x: 40, y: -20 }, +}; +const twoFingerPan: GestureSemanticInput = { ...oneFingerPan, pointerCount: 2 }; +const pinch: GestureSemanticInput = { intent: 'pinch', scale: 1.2 }; +const fling: GestureSemanticInput = { + intent: 'fling', + direction: 'left', + origin: { x: 100, y: 200 }, +}; +const drag: GestureCommandInput = { + intent: 'drag', + source: 'id="source"', + destination: 'id="destination"', +}; + +const device = (fields: Partial): DeviceInfo => ({ + platform: 'android', + id: 'test-device', + name: 'Test device', + kind: 'emulator', + ...fields, +}); + +/** The production binding seam, so admission runs against the real inspect-then-bind path. */ +async function admit(input: GestureCommandInput, target: DeviceInfo) { + const bindings = createRequestRuntimeBindings({ + gateway, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + admitDeviceClaim: async () => {}, + }); + try { + return await resolveBoundGestureRuntime({ + device: target, + input, + inspectFacts: bindings.inspectFacts, + bindDevice: bindings.bindDevice, + }); + } finally { + await bindings[Symbol.asyncDispose](); + } +} + +async function expectAdmitted(input: GestureCommandInput, target: DeviceInfo): Promise { + const resolved = await admit(input, target); + expect(resolved.ok, `${input.intent} should be admitted on ${target.platform}`).toBe(true); +} + +async function expectRefused( + input: GestureCommandInput, + target: DeviceInfo, + message: RegExp, + hint?: RegExp, +): Promise { + const resolved = await admit(input, target); + expect(resolved.ok).toBe(false); + if (resolved.ok) return; + expect(resolved.response.error.code).toBe('UNSUPPORTED_OPERATION'); + expect(resolved.response.error.message).toMatch(message); + if (hint) expect(String(resolved.response.error.hint)).toMatch(hint); +} + +test('Android phones and emulators admit single- and multi-touch gesture plans', async () => { + for (const kind of ['device', 'emulator'] as const) { + const target = device({ kind }); + await expectAdmitted(oneFingerPan, target); + await expectAdmitted(twoFingerPan, target); + await expectAdmitted(pinch, target); + } +}); + +test('target-authored drag is admitted only where adapters preserve every authored phase', async () => { + for (const kind of ['device', 'emulator'] as const) { + await expectAdmitted(drag, device({ kind, target: 'mobile' })); + } + for (const appleOs of ['ios', 'ipados'] as const) { + for (const kind of ['device', 'simulator'] as const) { + await expectAdmitted(drag, device({ platform: 'apple', appleOs, kind, target: 'mobile' })); + } + } + await expectAdmitted(drag, device({ platform: 'apple', kind: 'simulator', target: 'mobile' })); + + const inexactBackends = [ + device({ target: 'tv' }), + device({ platform: 'apple', appleOs: 'tvos', kind: 'simulator', target: 'tv' }), + device({ platform: 'apple', appleOs: 'macos', kind: 'device', target: 'desktop' }), + device({ platform: 'apple', appleOs: 'visionos', kind: 'simulator' }), + device({ platform: 'apple', appleOs: 'watchos', kind: 'simulator' }), + device({ platform: 'linux', kind: 'device', target: 'desktop' }), + device({ platform: 'vega', kind: 'device', target: 'tv' }), + device({ platform: 'web', kind: 'device', target: 'desktop' }), + ]; + for (const target of inexactBackends) { + await expectRefused( + drag, + target, + /^gesture drag is not supported on /, + /source hold, timed movement, and destination hold/, + ); + } +}); + +test('iOS and iPadOS simulators admit multi-touch while physical devices do not', async () => { + for (const appleOs of ['ios', 'ipados'] as const) { + const simulator = device({ platform: 'apple', appleOs, kind: 'simulator' }); + const physical = device({ platform: 'apple', appleOs, kind: 'device' }); + await expectAdmitted(oneFingerPan, simulator); + await expectAdmitted(twoFingerPan, simulator); + await expectAdmitted(pinch, simulator); + await expectAdmitted(oneFingerPan, physical); + await expectRefused(twoFingerPan, physical, /physical iOS devices/); + await expectRefused(pinch, physical, /physical iOS devices/, /iOS-simulator only/); + } +}); + +test('TV, spatial, watch, desktop, Linux, and web gesture policy stays explicit', async () => { + const androidTv = device({ target: 'tv' }); + const tvOs = device({ platform: 'apple', appleOs: 'tvos', kind: 'simulator', target: 'tv' }); + const visionOs = device({ platform: 'apple', appleOs: 'visionos', kind: 'simulator' }); + const watchOs = device({ platform: 'apple', appleOs: 'watchos', kind: 'simulator' }); + const macOs = device({ platform: 'apple', appleOs: 'macos', kind: 'device', target: 'desktop' }); + const linux = device({ platform: 'linux', kind: 'device', target: 'desktop' }); + const web = device({ platform: 'web', kind: 'device', target: 'desktop' }); + + await expectAdmitted(oneFingerPan, androidTv); + await expectRefused(twoFingerPan, androidTv, /Android TV/, /Android TV has no touch input/); + await expectRefused(twoFingerPan, tvOs, /tvOS/); + await expectRefused(twoFingerPan, visionOs, /visionOS/); + await expectRefused(oneFingerPan, watchOs, /watchos/); + await expectAdmitted(oneFingerPan, macOs); + await expectRefused(twoFingerPan, macOs, /macOS/); + await expectAdmitted(oneFingerPan, linux); + await expectAdmitted( + normalizePublicSwipeMotion({ from: { x: 10, y: 20 }, to: { x: 110, y: 20 } }).gesture, + linux, + ); + await expectAdmitted(normalizePublicGesture({ kind: 'swipe', preset: 'left' }).gesture, linux); + await expectRefused(fling, linux, /gesture fling is not supported on Linux/); + await expectRefused(twoFingerPan, linux, /linux/); + await expectRefused(oneFingerPan, web, /web/); +}); diff --git a/src/daemon/__tests__/request-handler-chain.test.ts b/src/daemon/__tests__/request-handler-chain.test.ts index 4d52a7c659..e93cd63f28 100644 --- a/src/daemon/__tests__/request-handler-chain.test.ts +++ b/src/daemon/__tests__/request-handler-chain.test.ts @@ -11,6 +11,8 @@ import { makeIosSession, makeSession } from '../../__tests__/test-utils/session- import { makeSnapshotState } from '../../__tests__/test-utils/snapshot-builders.ts'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; import { dispatchSwipeViaRuntime } from '../handlers/interaction-gesture.ts'; +import { createPlatformRuntimeGateway } from '../../platform-runtime.ts'; +import { createRequestRuntimeBindings } from '../request-runtime-binding.ts'; import { createLocalLinuxToolProvider, withLinuxToolProvider, @@ -160,7 +162,26 @@ test('duration-less public coordinate swipe retains Linux drag behavior', async const sessionStore = makeSessionStore('agent-device-linux-swipe-'); sessionStore.set('linux-swipe', makeSession('linux-swipe', { device: LINUX_DEVICE })); const drags: number[][] = []; + let captureCount = 0; const provider = createLocalLinuxToolProvider({ + accessibility: { + captureTree: async () => { + captureCount += 1; + return { + nodes: [ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 200, height: 200 }, + visibleToUser: true, + }, + ], + truncated: false, + surface: 'desktop', + }; + }, + }, input: { click: async () => {}, doubleClick: async () => {}, @@ -174,10 +195,38 @@ test('duration-less public coordinate swipe retains Linux drag behavior', async }, }); + // R44: swipe binds its gesture tier before executing, so this drives the REAL Linux owner + // through the composed gateway. Linux advertises no `gestureViewport`, so the coordinate frame + // still comes from the capture below — the preferred-operation fallback, unchanged. + const gateway = createPlatformRuntimeGateway({ + resolveSessionArtifacts: () => ({ + outputPath: '/sessions/linux-swipe/app.log', + pidPath: '/sessions/linux-swipe/app-log.pid', + }), + sessionsDir: '/sessions', + }); + let bindCount = 0; + const bindings = createRequestRuntimeBindings({ + gateway: { + ...gateway, + bind: async (request) => { + bindCount += 1; + return await gateway.bind(request); + }, + }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + admitDeviceClaim: async () => {}, + }); const response = await withLinuxToolProvider( provider, async () => await dispatchSwipeViaRuntime({ + inspectFacts: bindings.inspectFacts, + bindDevice: bindings.bindDevice, req: { ...makeRequest('swipe'), session: 'linux-swipe', @@ -186,14 +235,11 @@ test('duration-less public coordinate swipe retains Linux drag behavior', async sessionName: 'linux-swipe', sessionStore, contextFromFlags: () => ({}), - captureSnapshotForSession: async () => - makeSnapshotState([ - { - index: 0, - rect: { x: 0, y: 0, width: 200, height: 200 }, - visibleToUser: true, - }, - ]), + captureSnapshotForSession: async (_session, _flags, _store, _context, options) => { + assert.ok(options.boundCapture); + const captured = await options.boundCapture({ options: { surface: 'desktop' } }); + return makeSnapshotState(captured.nodes ?? []); + }, }), ); @@ -202,6 +248,8 @@ test('duration-less public coordinate swipe retains Linux drag behavior', async assert.ok(response.data); assert.equal(response.data.kind, 'fling'); assert.equal(response.data.durationMs, 100); + assert.equal(bindCount, 1); + assert.equal(captureCount, 1); assert.deepEqual(drags, [[10, 20, 110, 20, 100]]); }); diff --git a/src/daemon/__tests__/request-router-android-modal.test.ts b/src/daemon/__tests__/request-router-android-modal.test.ts index dfdc599708..731a180e10 100644 --- a/src/daemon/__tests__/request-router-android-modal.test.ts +++ b/src/daemon/__tests__/request-router-android-modal.test.ts @@ -20,7 +20,11 @@ vi.mock('../../core/dispatch.ts', async (importOriginal) => { }; }); -import { createRequestHandler } from './test-device-runtime-gateway.ts'; +import { + createRequestHandler, + gestureDeviceRuntimeGateway, + gestureRuntimeSpies, +} from './test-device-runtime-gateway.ts'; import type { SessionState } from '../types.ts'; import { LeaseRegistry } from '../lease-registry.ts'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; @@ -117,6 +121,7 @@ test('generic Android gesture commands dismiss blocking system dialogs during re dispatchResult = {}; execCalls.length = 0; dispatchCalls.length = 0; + gestureRuntimeSpies.scrollDirection.mockClear(); const sessionStore = makeSessionStore('agent-device-router-android-modal-'); sessionStore.set('default', makeAndroidSession('default')); @@ -130,6 +135,7 @@ test('generic Android gesture commands dismiss blocking system dialogs during re leaseRegistry: new LeaseRegistry(), deviceInventoryGateways: createTestDeviceInventoryGateways(), trackDownloadableArtifact: () => 'artifact-id', + deviceRuntimeGateway: gestureDeviceRuntimeGateway, }); const response = await handler({ @@ -141,7 +147,13 @@ test('generic Android gesture commands dismiss blocking system dialogs during re }); expect(response.ok).toBe(true); - expect(dispatchCalls).toEqual([['scroll', 'down', '0.55']]); + // R43: `scroll` reaches the device through its bound operation, so the dispatcher sees nothing. + expect(dispatchCalls).toEqual([]); + expect(gestureRuntimeSpies.scrollDirection).toHaveBeenCalledTimes(1); + expect(gestureRuntimeSpies.scrollDirection.mock.calls[0]?.[0]).toMatchObject({ + direction: 'down', + options: { amount: 0.55 }, + }); expect(execCalls).toEqual([['-s', 'emulator-5554', 'shell', 'input', 'tap', '210', '640']]); expect(openAndroidApp).toHaveBeenCalledWith( expect.objectContaining({ id: 'emulator-5554' }), @@ -153,9 +165,14 @@ test('generic Android gesture commands dismiss blocking system dialogs during re test('generic Android gesture commands continue when recording dialog inspection fails', async () => { snapshotCalls = 0; snapshotMode = 'throws'; - dispatchResult = { warning: 'The platform response already carried a warning.' }; execCalls.length = 0; dispatchCalls.length = 0; + gestureRuntimeSpies.scrollDirection.mockClear(); + // The owner's own result is what the readiness warning has to merge with, so the bound + // operation carries it now that the dispatcher no longer executes this command. + gestureRuntimeSpies.scrollDirection.mockResolvedValueOnce({ + warning: 'The platform response already carried a warning.', + }); const sessionStore = makeSessionStore('agent-device-router-android-modal-'); sessionStore.set('default', makeAndroidSession('default')); @@ -170,6 +187,7 @@ test('generic Android gesture commands continue when recording dialog inspection leaseRegistry: new LeaseRegistry(), deviceInventoryGateways: createTestDeviceInventoryGateways(), trackDownloadableArtifact: () => 'artifact-id', + deviceRuntimeGateway: gestureDeviceRuntimeGateway, }); const response = await handler({ @@ -181,7 +199,13 @@ test('generic Android gesture commands continue when recording dialog inspection }); expect(response.ok).toBe(true); - expect(dispatchCalls).toEqual([['scroll', 'down', '0.55']]); + // R43: `scroll` reaches the device through its bound operation, so the dispatcher sees nothing. + expect(dispatchCalls).toEqual([]); + expect(gestureRuntimeSpies.scrollDirection).toHaveBeenCalledTimes(1); + expect(gestureRuntimeSpies.scrollDirection.mock.calls[0]?.[0]).toMatchObject({ + direction: 'down', + options: { amount: 0.55 }, + }); expect(execCalls).toEqual([]); expect(openAndroidApp).not.toHaveBeenCalled(); expect(snapshotCalls).toBe(1); @@ -200,6 +224,7 @@ test('generic Android gesture commands skip local dialog recovery for provider d dispatchResult = {}; execCalls.length = 0; dispatchCalls.length = 0; + gestureRuntimeSpies.scrollDirection.mockClear(); const sessionStore = makeSessionStore('agent-device-router-android-modal-provider-'); const session = makeAndroidSession('default'); @@ -223,6 +248,7 @@ test('generic Android gesture commands skip local dialog recovery for provider d deviceInventoryGateways: createTestDeviceInventoryGateways(), providerDeviceRuntimeScope: providers.providerDeviceRuntimeScope, trackDownloadableArtifact: () => 'artifact-id', + deviceRuntimeGateway: gestureDeviceRuntimeGateway, }); const response = await handler({ @@ -234,7 +260,13 @@ test('generic Android gesture commands skip local dialog recovery for provider d }); expect(response.ok).toBe(true); - expect(dispatchCalls).toEqual([['scroll', 'down', '0.55']]); + // R43: `scroll` reaches the device through its bound operation, so the dispatcher sees nothing. + expect(dispatchCalls).toEqual([]); + expect(gestureRuntimeSpies.scrollDirection).toHaveBeenCalledTimes(1); + expect(gestureRuntimeSpies.scrollDirection.mock.calls[0]?.[0]).toMatchObject({ + direction: 'down', + options: { amount: 0.55 }, + }); expect(execCalls).toEqual([]); expect(snapshotCalls).toBe(0); }); diff --git a/src/daemon/__tests__/request-router-recording-health.test.ts b/src/daemon/__tests__/request-router-recording-health.test.ts index 1cc3fd1861..5ed3421d36 100644 --- a/src/daemon/__tests__/request-router-recording-health.test.ts +++ b/src/daemon/__tests__/request-router-recording-health.test.ts @@ -5,42 +5,32 @@ import path from 'node:path'; vi.mock('../../core/dispatch.ts', async (importOriginal) => { const actual = await importOriginal(); - return { - ...actual, - dispatchCommand: vi.fn(async () => ({})), - dispatchGesturePlan: vi.fn(async () => ({})), - dispatchGestureViewport: vi.fn(async () => ({ x: 0, y: 0, width: 390, height: 844 })), - }; + return { ...actual, dispatchCommand: vi.fn(async () => ({})) }; }); vi.mock('../../platforms/apple/core/runner/runner-client.ts', () => ({ getRunnerSessionSnapshot: vi.fn(), })); -import { - dispatchCommand, - dispatchGesturePlan, - dispatchGestureViewport, -} from '../../core/dispatch.ts'; +import { dispatchCommand } from '../../core/dispatch.ts'; import { getRunnerSessionSnapshot } from '../../platforms/apple/core/runner/runner-client.ts'; -import { createRequestHandler } from './test-device-runtime-gateway.ts'; +import { + createRequestHandler, + gestureDeviceRuntimeGateway, + gestureRuntimeSpies, +} from './test-device-runtime-gateway.ts'; import type { SessionState } from '../types.ts'; import { LeaseRegistry } from '../lease-registry.ts'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; import { makeTestScreenRecordingResource } from '../../__tests__/test-utils/screen-recording-live-handle.ts'; const mockDispatch = vi.mocked(dispatchCommand); -const mockDispatchGesturePlan = vi.mocked(dispatchGesturePlan); -const mockDispatchGestureViewport = vi.mocked(dispatchGestureViewport); const mockGetRunnerSessionSnapshot = vi.mocked(getRunnerSessionSnapshot); beforeEach(() => { mockDispatch.mockReset(); mockDispatch.mockResolvedValue({}); - mockDispatchGesturePlan.mockReset(); - mockDispatchGesturePlan.mockResolvedValue({}); - mockDispatchGestureViewport.mockReset(); - mockDispatchGestureViewport.mockResolvedValue({ x: 0, y: 0, width: 390, height: 844 }); + for (const spy of Object.values(gestureRuntimeSpies)) spy.mockClear(); mockGetRunnerSessionSnapshot.mockReset(); }); @@ -130,6 +120,7 @@ test('router allows canonical iOS simulator gestures during overlay recording af leaseRegistry: new LeaseRegistry(), deviceInventoryGateways: createTestDeviceInventoryGateways(), trackDownloadableArtifact: () => 'artifact-id', + deviceRuntimeGateway: gestureDeviceRuntimeGateway, }); const response = await handler({ @@ -143,8 +134,8 @@ test('router allows canonical iOS simulator gestures during overlay recording af expect(response.ok).toBe(true); expect(mockGetRunnerSessionSnapshot).not.toHaveBeenCalled(); - expect(mockDispatchGestureViewport).toHaveBeenCalledOnce(); - expect(mockDispatchGesturePlan).toHaveBeenCalledOnce(); + expect(gestureRuntimeSpies.gestureViewport).toHaveBeenCalledOnce(); + expect(gestureRuntimeSpies.performMultiTouchGesturePlan).toHaveBeenCalledOnce(); const recording = sessionStore.get('default')?.screenRecording?.handle.inspect(); expect(recording?.invalidatedReason).toBeUndefined(); expect(recording?.gestureEvents).toHaveLength(1); diff --git a/src/daemon/__tests__/request-router-replay-scope.test.ts b/src/daemon/__tests__/request-router-replay-scope.test.ts index e965159d6d..e329461674 100644 --- a/src/daemon/__tests__/request-router-replay-scope.test.ts +++ b/src/daemon/__tests__/request-router-replay-scope.test.ts @@ -35,6 +35,7 @@ import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; import { LeaseRegistry } from '../lease-registry.ts'; import { createRequestHandler, + gestureRuntimeSpies, lifecycleDeviceRuntimeGateway, } from './test-device-runtime-gateway.ts'; import { ensureDeviceReady } from '../device-ready.ts'; @@ -47,6 +48,7 @@ const mockEnsureDeviceReady = vi.mocked(ensureDeviceReady); const mockAwaitFixtureReadiness = vi.mocked(awaitFixtureReadiness); beforeEach(() => { + gestureRuntimeSpies.scrollDirection.mockClear(); mockDispatch.mockReset(); mockDispatch.mockResolvedValue({}); mockResolveTargetDevice.mockReset(); @@ -85,7 +87,10 @@ test('replay runs active-session actions inside the parent request provider scop }); expect(response).toMatchObject({ ok: true }); - expect(mockDispatch).toHaveBeenCalledTimes(2); + // `app-switcher` is still a legacy dispatch leaf; `scroll down` reaches its bound operation + // instead (R53), so the flow's two actions land on two different execution paths. + expect(mockDispatch).toHaveBeenCalledTimes(1); + expect(gestureRuntimeSpies.scrollDirection).toHaveBeenCalledTimes(1); expect(appleRunnerProvider).toHaveBeenCalledTimes(1); }); diff --git a/src/daemon/__tests__/request-router-response-level.test.ts b/src/daemon/__tests__/request-router-response-level.test.ts index 8d99c58c22..f63b542393 100644 --- a/src/daemon/__tests__/request-router-response-level.test.ts +++ b/src/daemon/__tests__/request-router-response-level.test.ts @@ -35,7 +35,11 @@ vi.mock('../response-views.ts', async (importOriginal) => { }); import { dispatchCommand } from '../../core/dispatch.ts'; -import { createRequestHandler } from './test-device-runtime-gateway.ts'; +import { + createRequestHandler, + gestureDeviceRuntimeGateway, + gestureRuntimeSpies, +} from './test-device-runtime-gateway.ts'; import type { DaemonRequest, SessionState } from '../types.ts'; import { LeaseRegistry } from '../lease-registry.ts'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; @@ -74,6 +78,9 @@ function makeHandler() { leaseRegistry: new LeaseRegistry(), deviceInventoryGateways: createTestDeviceInventoryGateways(), trackDownloadableArtifact: () => 'artifact-id', + // `scroll` is the view-less subject of case (e), and R53 moved it onto a bound runtime, so + // the handler needs an owner that admits `scrollDirection`. + deviceRuntimeGateway: gestureDeviceRuntimeGateway, }), }; } @@ -92,6 +99,8 @@ function request(command: string, overrides: Partial = {}): Daemo beforeEach(() => { mockDispatch.mockReset(); mockDispatch.mockImplementation(async () => ({ ...REPRESENTATIVE_PAYLOAD })); + gestureRuntimeSpies.scrollDirection.mockReset(); + gestureRuntimeSpies.scrollDirection.mockResolvedValue({}); }); test('(a) default identity: responseLevel absent === default === no meta, byte-identical', async () => { @@ -136,10 +145,20 @@ test('(d) digest composes with --cost: viewed data plus an additive cost block', test('(e) digest on a command with no registered view is byte-identical to default', async () => { const { handler } = makeHandler(); - const digest = await handler(request('scroll', { meta: { responseLevel: 'digest' } })); - const def = await handler(request('scroll', { meta: {} })); + // `scroll` has no registered view. It no longer reaches the mocked `dispatchCommand` (R53 put + // it on a bound runtime), so the representative payload comes from the bound operation instead. + gestureRuntimeSpies.scrollDirection.mockResolvedValue({ ...REPRESENTATIVE_PAYLOAD }); + const scrollRequest: Partial = { positionals: ['down'], meta: {} }; + const digest = await handler( + request('scroll', { ...scrollRequest, meta: { responseLevel: 'digest' } }), + ); + const def = await handler(request('scroll', scrollRequest)); expect(JSON.stringify(digest)).toBe(JSON.stringify(def)); - if (digest.ok) expect(digest.data).toEqual(REPRESENTATIVE_PAYLOAD); + // The owner's payload passes through the view-less path; scroll's own result fields sit beside + // it, and its success text owns `message`. + if (digest.ok) { + expect(digest.data).toMatchObject({ items: REPRESENTATIVE_PAYLOAD.items, direction: 'down' }); + } }); test('(f) boundary survival: meta.responseLevel survives commandRpcParamsSchema parsing', () => { diff --git a/src/daemon/__tests__/request-router-screenshot.test.ts b/src/daemon/__tests__/request-router-screenshot.test.ts index 97555cfcf6..5511271e81 100644 --- a/src/daemon/__tests__/request-router-screenshot.test.ts +++ b/src/daemon/__tests__/request-router-screenshot.test.ts @@ -233,10 +233,9 @@ test('router serializes concurrent commands for the same device across sessions' writeSolidPng(input.outPath); await gate('screenshot'); }, - }); - mockDispatch.mockImplementation(async (_device, command) => { - await gate(command); - return {}; + onScroll: async () => { + await gate('scroll'); + }, }); const handler = createRequestHandler({ diff --git a/src/daemon/__tests__/screenshot-runtime-fixture.ts b/src/daemon/__tests__/screenshot-runtime-fixture.ts index e06d723205..6a50cdc1e2 100644 --- a/src/daemon/__tests__/screenshot-runtime-fixture.ts +++ b/src/daemon/__tests__/screenshot-runtime-fixture.ts @@ -40,6 +40,8 @@ export type ScreenshotRuntimeFixtureOptions = Readonly<{ /** Replaces the default "write a solid PNG at the requested path" capture behavior. */ onCapture?: (input: CaptureScreenshotInput) => Promise | void; snapshotResult?: (input: CaptureSnapshotInput) => SnapshotResult; + /** Gate for the neighbouring bound `scroll`, used by the device-lock serialization tests. */ + onScroll?: () => Promise | void; }>; export type ScreenshotRuntimeFixture = Readonly<{ @@ -74,6 +76,12 @@ export function screenshotRuntimeFixture( options.snapshotResult?.(input) ?? { nodes: [], backend: 'android' }, ); const tapPoint = vi.fn(async (_input: TapPointInput) => ({})); + // R43: `scroll` is the neighbouring command the device-lock tests use to prove serialization, + // and it now reaches the platform through a bound operation like the screenshot beside it. + const scrollDirection = vi.fn(async () => { + await options.onScroll?.(); + return {}; + }); // The unavailable gateway is the exhaustive fact catalog; only the capture cells are overridden. const facts = async (device: DeviceInfo): Promise> => { @@ -83,6 +91,7 @@ export function screenshotRuntimeFixture( operations: { ...base.operations, ...screenshotRuntimeOperationFacts({ capture: options.capture ?? available }), + scrollDirection: available, ...snapshotRuntimeOperationFacts({ capture: options.snapshot ?? available, customActions: options.snapshot ?? available, @@ -111,6 +120,7 @@ export function screenshotRuntimeFixture( captureSnapshotWithCustomActions: captureSnapshot, captureSnapshotWithoutActiveApp: captureSnapshot, tapPoint, + scrollDirection, }, [Symbol.asyncDispose]: async () => {}, }; diff --git a/src/daemon/__tests__/scroll-runtime.test.ts b/src/daemon/__tests__/scroll-runtime.test.ts new file mode 100644 index 0000000000..83878a50ba --- /dev/null +++ b/src/daemon/__tests__/scroll-runtime.test.ts @@ -0,0 +1,343 @@ +import { expect, expectTypeOf, test } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import type { + BoundDeviceRuntime, + PlatformRuntimeOperations, + RuntimeFacts, +} from '@agent-device/contracts/platform'; +import type { DaemonCommandContext } from '../context.ts'; +import { + resolveScrollRuntimePlan, + type ScrollRuntimePlan, +} from '@agent-device/contracts/platform-runtime-operations'; +import { resolveBoundScrollRuntime } from '../scroll-runtime.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; +import { unavailableDeploymentSnapshotAndShutdownOperationFacts } from '../../__tests__/test-utils/runtime-operation-facts.ts'; +import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; + +/** + * The retired `handleScrollCommand` suite, re-pointed at the bound runtime (R43). Every + * assertion is the behavior `main` produced: the same parse rejections, the same execution + * options handed to the owner, the same edge-pass loop, and the same scoped-capture failure. + * + * What moved is WHERE the edge refusal happens — the retired leaf discovered a missing snapshot + * mid-command, and admission now proves the capture before any pass runs (ADR 0019 §6). + */ +type ScrollCall = { direction: string; options: unknown }; + +function bindings(options: { + scroll: (direction: string, scrollOptions: unknown) => Promise | void>; + captureSnapshot?: (input: { options?: { scope?: string } }) => Promise; +}): { inspectFacts: InspectDeviceRuntimeFacts; bindDevice: BindDeviceRuntime } { + const available = { available: true } as const; + const facts = { + device: { family: 'apple', kind: 'simulator', providerMode: 'local' }, + operations: { + ...unavailableDeploymentSnapshotAndShutdownOperationFacts, + scrollDirection: available, + ...(options.captureSnapshot ? { captureSnapshot: available } : {}), + }, + } as unknown as RuntimeFacts; + return { + inspectFacts: async () => facts, + bindDevice: (async () => + ({ + facts, + operations: { + scrollDirection: async (input: { direction: string; options: unknown }) => + await options.scroll(input.direction, input.options), + ...(options.captureSnapshot ? { captureSnapshot: options.captureSnapshot } : {}), + }, + }) as unknown as BoundDeviceRuntime) as unknown as BindDeviceRuntime, + }; +} + +async function runScroll( + positionals: string[], + context: Partial, + options: Parameters[0], +): Promise> { + const resolved = await resolveBoundScrollRuntime({ + device: IOS_SIMULATOR, + positionals, + context: context as DaemonCommandContext, + ...bindings(options), + }); + if (!resolved.ok) throw new AppError('UNSUPPORTED_OPERATION', 'admission refused the scroll'); + const data = await resolved.execute({ + dispatchContext: context as DaemonCommandContext, + } as Parameters[0]); + return (data ?? {}) as Record; +} + +test('bound scroll rejects mixing amount and --pixels', async () => { + await assert.rejects( + () => + runScroll( + ['down', '0.4'], + { pixels: 240 }, + { + scroll: async () => { + throw new Error('scroll should be rejected before the owner is reached'); + }, + }, + ), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /either a relative amount or --pixels/i.test(error.message), + ); +}); + +test('bound scroll forwards pixels and duration without reporting ignored duration', async () => { + const calls: ScrollCall[] = []; + const result = await runScroll( + ['down'], + { pixels: 200, durationMs: 50 }, + { + scroll: async (direction, options) => { + calls.push({ direction, options }); + return { ok: true }; + }, + }, + ); + + assert.deepEqual(calls, [ + { + direction: 'down', + options: { + amount: undefined, + pixels: 200, + durationMs: 50, + releaseBehavior: 'controlled', + }, + }, + ]); + assert.equal(result.pixels, 200); + assert.equal(result.durationMs, undefined); +}); + +test('bound scroll reports duration when the owner honored it', async () => { + const result = await runScroll( + ['down'], + { pixels: 200, durationMs: 50 }, + { + scroll: async () => ({ pixels: 200, durationMs: 50 }), + }, + ); + assert.equal(result.pixels, 200); + assert.equal(result.durationMs, 50); +}); + +test('bound scroll rejects duration above the shared cap', async () => { + await assert.rejects( + () => + runScroll( + ['down'], + { pixels: 200, durationMs: 10_001 }, + { + scroll: async () => { + throw new Error('scroll should be rejected before the owner is reached'); + }, + }, + ), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /durationMs.*at most 10000/i.test(error.message), + ); +}); + +test('bound scroll bottom refuses at admission when the owner declares no capture', async () => { + const calls: ScrollCall[] = []; + const resolved = await resolveBoundScrollRuntime({ + device: IOS_SIMULATOR, + positionals: ['bottom'], + context: {} as DaemonCommandContext, + ...bindings({ + scroll: async (direction, options) => { + calls.push({ direction, options }); + return { lastPass: calls.length }; + }, + }), + }); + + assert.equal(resolved.ok, false); + if (resolved.ok || resolved.response.ok) return; + assert.equal(resolved.response.error.code, 'UNSUPPORTED_OPERATION'); + assert.match(String(resolved.response.error.message), /requires snapshot support/i); + // The refusal is now proof-before-execution: no pass ran, and none could have. + assert.equal(calls.length, 0); +}); + +test('bound scroll bottom does not scroll when no hidden content is below', async () => { + const calls: ScrollCall[] = []; + const result = await runScroll( + ['bottom'], + {}, + { + scroll: async (direction, options) => { + calls.push({ direction, options }); + return { lastPass: calls.length }; + }, + captureSnapshot: async () => + makeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), + }, + ); + + assert.equal(calls.length, 0); + assert.equal(result.direction, 'down'); + assert.equal(result.edge, 'bottom'); + assert.equal(result.passes, 0); + assert.match(String(result.message), /Already at bottom/); +}); + +test('bound scroll bottom scrolls only while a scoped capture confirms hidden content', async () => { + const calls: ScrollCall[] = []; + const snapshotScopes: unknown[] = []; + const snapshots = [ + makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), + makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }), + makeScrollSnapshot({ hiddenBelow: false, message: 'Latest message' }), + ]; + const result = await runScroll( + ['bottom'], + {}, + { + scroll: async (direction, options) => { + calls.push({ direction, options }); + return { lastPass: calls.length }; + }, + captureSnapshot: async (input) => { + snapshotScopes.push(input.options?.scope); + return snapshots[Math.min(snapshotScopes.length - 1, snapshots.length - 1)]; + }, + }, + ); + + assert.equal(calls.length, 1); + assert.deepEqual(calls[0], { + direction: 'down', + options: { + amount: undefined, + pixels: undefined, + durationMs: undefined, + releaseBehavior: 'inertial', + }, + }); + assert.equal(result.passes, 1); + assert.equal(result.lastPass, 1); + assert.deepEqual(snapshotScopes, [undefined, 'Messages', 'Messages']); +}); + +test('bound scroll bottom tolerates unchanged signatures while hidden content advances', async () => { + const calls: ScrollCall[] = []; + const snapshots = [ + makeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), + makeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), + makeScrollSnapshot({ hiddenBelow: true, message: 'Repeated row' }), + makeScrollSnapshot({ hiddenBelow: false, message: 'Repeated row' }), + ]; + let snapshotIndex = 0; + const result = await runScroll( + ['bottom'], + {}, + { + scroll: async (direction, options) => { + calls.push({ direction, options }); + return { lastPass: calls.length }; + }, + captureSnapshot: async () => snapshots[Math.min(snapshotIndex++, snapshots.length - 1)], + }, + ); + + assert.equal(calls.length, 2); + assert.equal(result.passes, 2); +}); + +test('bound scroll bottom keeps scoped capture failures scoped', async () => { + let snapshotCount = 0; + await assert.rejects( + () => + runScroll( + ['bottom'], + {}, + { + scroll: async () => ({}), + captureSnapshot: async (input) => { + snapshotCount += 1; + if (input.options?.scope) throw new Error('scoped snapshot failed'); + return makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }); + }, + }, + ), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + /scoped container/i.test(error.message) && + error.details?.scope === 'Messages', + ); + assert.equal(snapshotCount, 2); +}); + +function makeScrollSnapshot(options: { hiddenBelow: boolean; message: string }) { + return { + backend: 'xctest' as const, + nodes: [ + { + index: 1, + type: 'ScrollView', + label: 'Messages', + hiddenContentBelow: options.hiddenBelow ? true : undefined, + rect: { x: 0, y: 100, width: 400, height: 600 }, + }, + { + index: 2, + parentIndex: 1, + type: 'Button', + label: options.message, + rect: { x: 0, y: 640, width: 400, height: 56 }, + }, + ], + truncated: false, + }; +} + +/** + * R53 type-level regression. The two scroll plans must project DIFFERENT bindings: an edge scroll + * proves `captureSnapshot` statically, and an ordinary scroll must not be able to name it at all. + * + * This is the property a runtime `if (!captureSnapshot) throw` guard silently gave up — the guard + * type-checks against a widened binding, so the compiler stops enforcing what admission proved. + */ +test('the edge plan proves its capture statically and the direction plan cannot expose one', () => { + const direction = resolveScrollRuntimePlan({}); + const edge = resolveScrollRuntimePlan({ edge: 'bottom' }); + + // The discriminant carries the edge, so a caller that narrows to `edge` also holds it. + expect(direction.kind).toBe('direction'); + expect(edge).toMatchObject({ kind: 'edge', edge: 'bottom' }); + + // Structural: the required sets differ, and only the edge use names the capture. + expect([...direction.use.required]).toEqual(['scrollDirection']); + expect([...edge.use.required]).toEqual(['scrollDirection', 'captureSnapshot']); + + type DirectionOperations = BoundDeviceRuntime< + Extract['use'] + >['operations']; + type EdgeOperations = BoundDeviceRuntime< + Extract['use'] + >['operations']; + /** Keys the binding guarantees — an optional key drops out, which is the whole point here. */ + type RequiredKeys = { [K in keyof T]-?: object extends Pick ? never : K }[keyof T]; + + // The edge binding GUARANTEES the capture: demote it to `preferred` and this fails, because + // `captureSnapshot` leaves the required set. + expectTypeOf>().toEqualTypeOf< + 'scrollDirection' | 'captureSnapshot' + >(); + // The ordinary binding cannot even name a capture — absent, not merely optional. + expectTypeOf().toEqualTypeOf<'scrollDirection'>(); + expectTypeOf>().toEqualTypeOf<'scrollDirection'>(); +}); diff --git a/src/daemon/__tests__/test-device-runtime-gateway.ts b/src/daemon/__tests__/test-device-runtime-gateway.ts index d1826d520a..6661f41c4d 100644 --- a/src/daemon/__tests__/test-device-runtime-gateway.ts +++ b/src/daemon/__tests__/test-device-runtime-gateway.ts @@ -1,3 +1,4 @@ +import { vi } from 'vitest'; import { type ApplicationLifecycleOperationFacts, applicationLifecycleOperationFacts, @@ -8,7 +9,9 @@ import { localRuntimeOwner, narrowDeviceBinding, } from '@agent-device/contracts/platform-runtime'; +import type { GesturePlanInput } from '@agent-device/contracts/gesture-runtime'; import type { PlatformRuntimeOperations } from '@agent-device/contracts/platform-runtime-operations'; +import type { ScrollDirectionInput } from '@agent-device/contracts/scroll-runtime'; import { createRequestHandler as createProductionRequestHandler, type RequestRouterDeps, @@ -73,12 +76,18 @@ async function lifecycleBindingForTest(device: DeviceInfo) { bootTargetHeadless: unavailable, listApps: unavailable, ...lifecycleFacts, + ...admittedGestureFamilyFacts, }, }, - operations: availableApplicationLifecycleOperations( - await applicationLifecycleRuntimeFixture(device), - lifecycleFacts, - ), + operations: { + ...availableApplicationLifecycleOperations( + await applicationLifecycleRuntimeFixture(device), + lifecycleFacts, + ), + // The gesture family rides along: replay flows drive `scroll`/`swipe` through this gateway, + // and R52/R53 put them on bound operations rather than the mocked dispatcher. + ...gestureRuntimeSpies, + }, [Symbol.asyncDispose]: async () => {}, }; } @@ -143,6 +152,55 @@ export const unavailableDeviceRuntimeGateway: DeviceRuntimeGateway {}, }); +/** + * Spies for the gesture surface a router-level test drives. They replace the retired + * `dispatchGesturePlan` / `dispatchGestureViewport` module mocks: gestures now reach the platform + * through a bound operation, so the observation point is the operation, not the dispatcher. + */ +/** The gesture-family cells a gateway admits when it stands in for a working owner. */ +const admittedGestureFamilyFacts = Object.freeze({ + captureSnapshot: available, + performGesturePlan: available, + performDirectionalFlingPlan: available, + performMultiTouchGesturePlan: available, + performTargetAuthoredDrag: available, + gestureViewport: available, + scrollDirection: available, +}); + +export const gestureRuntimeSpies = { + captureSnapshot: vi.fn(async () => ({ backend: 'xctest' as const, nodes: [] })), + performGesturePlan: vi.fn(async (_input: GesturePlanInput) => ({})), + performDirectionalFlingPlan: vi.fn(async (_input: GesturePlanInput) => ({})), + performMultiTouchGesturePlan: vi.fn(async (_input: GesturePlanInput) => ({})), + performTargetAuthoredDrag: vi.fn(async (_input: GesturePlanInput) => ({})), + gestureViewport: vi.fn(async () => ({ x: 0, y: 0, width: 390, height: 844 })), + scrollDirection: vi.fn(async (_input: ScrollDirectionInput) => ({})), +}; + +/** The unavailable gateway plus an admitted gesture/scroll surface. */ +export const gestureDeviceRuntimeGateway: DeviceRuntimeGateway = + Object.freeze({ + inspectFacts: async (device) => (await gestureBinding(device)).facts, + bind: async (request) => await gestureBinding(request.device), + shutdown: async () => {}, + }); + +async function gestureBinding(device: DeviceInfo) { + const base = await unavailableBinding(device); + return { + ...base, + facts: { + ...base.facts, + operations: { + ...base.facts.operations, + ...admittedGestureFamilyFacts, + }, + }, + operations: { ...base.operations, ...gestureRuntimeSpies }, + } as unknown as Awaited['bind']>>; +} + async function unavailableBinding(device: DeviceInfo) { return await unavailableDeviceRuntimeGateway.bind({ device, diff --git a/src/daemon/generic-runtime-execution.ts b/src/daemon/generic-runtime-execution.ts index 15c5c0d404..9409a45ccf 100644 --- a/src/daemon/generic-runtime-execution.ts +++ b/src/daemon/generic-runtime-execution.ts @@ -1,7 +1,9 @@ import type { ResolvedGenericExecution } from './request-generic-dispatch.ts'; import { resolveBoundFocusRuntime } from './focus-runtime.ts'; import { resolveScreenshotGenericExecution } from './screenshot-runtime.ts'; +import { resolveBoundScrollRuntime } from './scroll-runtime.ts'; import type { ScreenshotRuntimeBindings } from './screenshot-runtime-binding.ts'; +import type { DaemonCommandContext } from './context.ts'; import type { DaemonRequest, SessionState } from './types.ts'; import { resolveBoundViewportRuntime } from './viewport-runtime.ts'; import { resolveBoundBackRuntime } from './back-runtime.ts'; @@ -15,7 +17,12 @@ import { resolveBoundTvRemoteRuntime } from './tv-remote-runtime.ts'; * name. `undefined` means the leaf still executes through legacy platform dispatch. */ export async function resolveGenericRuntimeExecution( - params: Readonly<{ req: DaemonRequest; session: SessionState }> & ScreenshotRuntimeBindings, + params: Readonly<{ + req: DaemonRequest; + session: SessionState; + context: DaemonCommandContext; + }> & + ScreenshotRuntimeBindings, ): Promise { switch (params.req.command) { case 'screenshot': @@ -27,6 +34,14 @@ export async function resolveGenericRuntimeExecution( inspectFacts: params.inspectFacts, bindDevice: params.bindDevice, }); + case 'scroll': + return await resolveBoundScrollRuntime({ + device: params.session.device, + positionals: params.req.positionals ?? [], + context: params.context, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); case 'viewport': return await resolveBoundViewportRuntime({ device: params.session.device, diff --git a/src/daemon/gesture-runtime.ts b/src/daemon/gesture-runtime.ts new file mode 100644 index 0000000000..68dce7b251 --- /dev/null +++ b/src/daemon/gesture-runtime.ts @@ -0,0 +1,184 @@ +import { gestureRefusalMessage } from '@agent-device/contracts/gesture-admission'; +import type { GestureCommandInput, GesturePlan } from '@agent-device/contracts/gesture-plan-types'; +import type { + GesturePlanInput, + GestureRuntimeOperations, +} from '@agent-device/contracts/gesture-runtime'; +import type { + CaptureSnapshotInput, + SnapshotResult, +} from '@agent-device/contracts/snapshot-runtime'; +import { + resolveGestureRuntimePlan, + type GestureRuntimePlan, +} from '@agent-device/contracts/platform-runtime-operations'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError, normalizeError } from '@agent-device/kernel/errors'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import type { DaemonCommandContext } from './context.ts'; +import type { DaemonFailureResponse } from './handlers/response.ts'; +import { admitRuntimeOperations, type RuntimeAdmissionBindings } from './runtime-admission.ts'; +import { runtimeExecutionFromContext } from './snapshot-runtime-capture-input.ts'; + +/** + * One request's bound gesture authority. `gesture` and `swipe` both build their plans inside the + * shared client-side orchestration, which can execute several plans per request (`swipe --count`, + * a drag's resolved endpoints), so the executor is a closure over the single binding rather than + * a value frozen at bind time. + */ +export type BoundGestureExecutor = Readonly<{ + captureSnapshot: (input: CaptureSnapshotInput) => Promise; + performPlan: ( + plan: GesturePlan, + context: DaemonCommandContext, + ) => Promise | void>; + /** + * Present only when the admitted owner advertised its own frame read. Absence is not a failure: + * the caller derives the frame from a capture instead, exactly as it does today for an owner + * without one (Linux). + */ + gestureViewport?: (context: DaemonCommandContext) => Promise; +}>; + +export type ResolvedGestureRuntime = + | Readonly<{ ok: false; response: DaemonFailureResponse }> + | Readonly<{ ok: true; gestures: BoundGestureExecutor }>; + +/** + * The one place `gesture` and `swipe` reach a device (ADR 0019). The gesture input selects ONE + * execution tier, admission inspects that tier's fact on the exact owner, and the handler binds + * once — before any plan is built, so a device that cannot synthesize this gesture is refused + * where the retired `requireGestureSupported` refused it rather than mid-series. + * + * The refusal message is composed from the tier and the device so every string the retired + * admission produced survives verbatim; the hint comes from the owner's own fact. + */ +export async function resolveBoundGestureRuntime( + params: { + device: DeviceInfo; + /** The normalized gesture — for `swipe`, the fling its motion normalizes to. */ + input: GestureCommandInput; + } & RuntimeAdmissionBindings, +): Promise { + const plan = resolveGestureRuntimePlan(params.input); + const admitted = await admitRuntimeOperations({ + command: 'gesture', + device: params.device, + required: plan.use.required, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + // The retired admission THREW an `AppError`, which the gesture handler's catch normalized — + // so the refusal is built the same way here. Going through `errorResponse` instead would + // drop the code's default hint and its `retriable` classification from the wire shape. + unavailableResponse: (unavailable) => ({ + ok: false, + error: normalizeError( + new AppError( + 'UNSUPPORTED_OPERATION', + gestureRefusalMessage(params.device, plan.tier, params.input.intent), + { + gesture: params.input.intent, + ...(unavailable.hint === undefined ? {} : { hint: unavailable.hint }), + }, + ), + ), + }), + }); + if (admitted.type === 'response') return { ok: false, response: admitted.response }; + // One bind, with the exactly-typed use the tier selected (ADR 0019 §9). + return { ok: true, gestures: await bindGestureTier(admitted.bind, params.device, plan) }; +} + +/** + * The ONE place each bound gesture tier executes (R52/R54, shared by `gesture` and `swipe`). + * + * Each branch binds its own exactly-typed use, so the operation it calls is non-optional by + * construction — no cast, no non-null repair, and exactly one lexical owner per tier, which is + * what lets the cutover gate prove no parallel route exists. The branches read alike because + * the tiers share their mechanics; what differs is the cell each one proves. + */ +async function bindGestureTier( + bind: Extract>, { type: 'admitted' }>['bind'], + device: DeviceInfo, + plan: GestureRuntimePlan, +): Promise { + switch (plan.tier) { + case 'plan': { + const runtime = await bind(device, plan.use); + return { + performPlan: async (gesturePlan, context) => + await runtime.operations.performGesturePlan(gesturePlanInput(gesturePlan, context)), + ...selectGestureFrame(runtime), + }; + } + case 'directional-fling': { + const runtime = await bind(device, plan.use); + return { + performPlan: async (gesturePlan, context) => + await runtime.operations.performDirectionalFlingPlan( + gesturePlanInput(gesturePlan, context), + ), + ...selectGestureFrame(runtime), + }; + } + case 'multi-touch': { + const runtime = await bind(device, plan.use); + return { + performPlan: async (gesturePlan, context) => + await runtime.operations.performMultiTouchGesturePlan( + gesturePlanInput(gesturePlan, context), + ), + ...selectGestureFrame(runtime), + }; + } + case 'target-authored-drag': { + const runtime = await bind(device, plan.use); + return { + performPlan: async (gesturePlan, context) => + await runtime.operations.performTargetAuthoredDrag( + gesturePlanInput(gesturePlan, context), + ), + ...selectGestureFrame(runtime), + }; + } + } +} + +/** Keeps both viewport paths inside the request's admitted gesture binding. */ +function selectGestureFrame( + runtime: Readonly<{ + operations: Readonly<{ + captureSnapshot: (input: CaptureSnapshotInput) => Promise; + gestureViewport?: GestureRuntimeOperations['gestureViewport']; + }>; + }>, +): Pick { + const { gestureViewport } = runtime.operations; + const selected = gestureViewport ? { operations: { gestureViewport } } : undefined; + return Object.freeze({ + captureSnapshot: async (input: CaptureSnapshotInput) => + await runtime.operations.captureSnapshot(input), + ...(selected + ? { + gestureViewport: async (context: DaemonCommandContext) => + await selected.operations.gestureViewport(gestureViewportInput(context)), + } + : {}), + }); +} + +/** The neutral intent one gesture carries, projected from a resolved command context. */ +function gesturePlanInput(plan: GesturePlan, context: DaemonCommandContext): GesturePlanInput { + return { + plan, + ...(context.appBundleId === undefined ? {} : { options: { appBundleId: context.appBundleId } }), + execution: runtimeExecutionFromContext(context), + }; +} + +function gestureViewportInput(context: DaemonCommandContext) { + return { + ...(context.appBundleId === undefined ? {} : { options: { appBundleId: context.appBundleId } }), + execution: runtimeExecutionFromContext(context), + }; +} diff --git a/src/daemon/handlers/__tests__/gesture-runtime-bindings.fixtures.ts b/src/daemon/handlers/__tests__/gesture-runtime-bindings.fixtures.ts new file mode 100644 index 0000000000..30c4457be6 --- /dev/null +++ b/src/daemon/handlers/__tests__/gesture-runtime-bindings.fixtures.ts @@ -0,0 +1,65 @@ +import { vi } from 'vitest'; +import type { + BoundDeviceRuntime, + PlatformRuntimeOperations, + RuntimeFacts, +} from '@agent-device/contracts/platform'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import type { + BindDeviceRuntime, + InspectDeviceRuntimeFacts, +} from '../../request-runtime-binding.ts'; +import { unavailableDeploymentSnapshotAndShutdownOperationFacts } from '../../../__tests__/test-utils/runtime-operation-facts.ts'; + +const available = Object.freeze({ available: true } as const); + +/** + * A runtime that admits every gesture tier, with one spy per operation. + * + * The counters on `inspectFacts` / `bindDevice` are the ADR 0019 §9 regression surface: a handler + * that binds per repetition, or re-inspects after resolving a target, shows up here as a count + * greater than one (the defect #1944's P1 fixed). + */ +export function gestureRuntimeBindingsFixture( + options: Readonly<{ viewport?: Rect; unavailable?: readonly string[] }> = {}, +) { + const unavailable = new Set(options.unavailable ?? []); + const cell = (operation: string) => + unavailable.has(operation) + ? ({ available: false, reason: 'unsupported-platform-leaf' } as const) + : available; + const operationSpies = { + captureSnapshot: vi.fn(async () => ({ backend: 'xctest' as const, nodes: [] })), + performGesturePlan: vi.fn(async () => ({})), + performDirectionalFlingPlan: vi.fn(async () => ({})), + performMultiTouchGesturePlan: vi.fn(async () => ({})), + performTargetAuthoredDrag: vi.fn(async () => ({})), + gestureViewport: vi.fn( + async () => options.viewport ?? ({ x: 0, y: 0, width: 400, height: 800 } as Rect), + ), + }; + const facts = { + device: { family: 'apple', kind: 'simulator', providerMode: 'local' }, + operations: { + ...unavailableDeploymentSnapshotAndShutdownOperationFacts, + captureSnapshot: cell('captureSnapshot'), + performGesturePlan: cell('performGesturePlan'), + performDirectionalFlingPlan: cell('performDirectionalFlingPlan'), + performMultiTouchGesturePlan: cell('performMultiTouchGesturePlan'), + performTargetAuthoredDrag: cell('performTargetAuthoredDrag'), + gestureViewport: cell('gestureViewport'), + }, + } as unknown as RuntimeFacts; + const inspectFacts = vi.fn(async () => facts) as unknown as InspectDeviceRuntimeFacts & + ReturnType; + const bindDevice = vi.fn( + async () => + ({ + facts, + operations: Object.fromEntries( + Object.entries(operationSpies).filter(([name]) => !unavailable.has(name)), + ), + }) as unknown as BoundDeviceRuntime, + ) as unknown as BindDeviceRuntime & ReturnType; + return { ...operationSpies, facts, inspectFacts, bindDevice }; +} diff --git a/src/daemon/handlers/__tests__/install-source.test.ts b/src/daemon/handlers/__tests__/install-source.test.ts index c168b0bb51..ba148bfab0 100644 --- a/src/daemon/handlers/__tests__/install-source.test.ts +++ b/src/daemon/handlers/__tests__/install-source.test.ts @@ -12,8 +12,10 @@ import { narrowDeviceBinding, providerRuntimeOwner, } from '@agent-device/contracts/platform-runtime'; +import { gestureRuntimeOperationFacts } from '@agent-device/contracts/gesture-runtime'; import type { PlatformRuntimeOperations } from '@agent-device/contracts/platform-runtime-operations'; import { screenshotRuntimeOperationFacts } from '@agent-device/contracts/screenshot-runtime'; +import { scrollRuntimeOperationFacts } from '@agent-device/contracts/scroll-runtime'; import { snapshotRuntimeOperationFacts } from '@agent-device/contracts/snapshot-runtime'; import { touchRuntimeOperationFacts } from '@agent-device/contracts/touch-runtime'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -365,6 +367,14 @@ function sourceRuntimeFacts( fill: unavailable, tapElementSelector: unavailable, }), + ...gestureRuntimeOperationFacts({ + plan: unavailable, + directionalFling: unavailable, + multiTouch: unavailable, + targetAuthoredDrag: unavailable, + viewport: unavailable, + }), + ...scrollRuntimeOperationFacts({ scroll: unavailable }), readTextAtPoint: unavailable, back: unavailable, home: unavailable, diff --git a/src/daemon/handlers/__tests__/interaction-gesture-drag.test.ts b/src/daemon/handlers/__tests__/interaction-gesture-drag.test.ts index 7de89e2048..26602f1dd6 100644 --- a/src/daemon/handlers/__tests__/interaction-gesture-drag.test.ts +++ b/src/daemon/handlers/__tests__/interaction-gesture-drag.test.ts @@ -7,15 +7,6 @@ import { import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; import { activateCompleteRefFrame, refFrameState } from '../../ref-frame.ts'; -vi.mock('../../../core/dispatch.ts', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - dispatchGestureViewport: vi.fn(async () => ({ x: 0, y: 0, width: 400, height: 800 })), - dispatchGesturePlan: vi.fn(async () => ({})), - }; -}); - vi.mock('../interaction-snapshot.ts', async (importOriginal) => { const actual = await importOriginal(); return { @@ -26,15 +17,14 @@ vi.mock('../interaction-snapshot.ts', async (importOriginal) => { }; }); -import { dispatchGesturePlan } from '../../../core/dispatch.ts'; import { handleInteractionCommands } from '../interaction.ts'; +import { gestureRuntimeBindingsFixture } from './gesture-runtime-bindings.fixtures.ts'; -const mockDispatchGesturePlan = vi.mocked(dispatchGesturePlan); const contextFromFlags = () => ({}); +let gestures = gestureRuntimeBindingsFixture(); beforeEach(() => { - mockDispatchGesturePlan.mockClear(); - mockDispatchGesturePlan.mockResolvedValue({}); + gestures = gestureRuntimeBindingsFixture(); }); function makeDragSession(sessionName: string) { @@ -96,6 +86,8 @@ async function runDrag(sessionStore: ReturnType, sessio sessionName, sessionStore, contextFromFlags, + inspectFacts: gestures.inspectFacts, + bindDevice: gestures.bindDevice, }); } @@ -129,7 +121,11 @@ test('recorded ref drag dispatches once and stores portable selectors with both expect(response.data).not.toHaveProperty('selectorChain'); expect(response.data).not.toHaveProperty('targetEvidence'); } - expect(mockDispatchGesturePlan).toHaveBeenCalledTimes(1); + // ADR 0019 §9 regression guard: a drag resolves two targets and still takes exactly one + // inspection and one bind — the #1944 P1 shape. + expect(gestures.performTargetAuthoredDrag).toHaveBeenCalledTimes(1); + expect(gestures.inspectFacts).toHaveBeenCalledTimes(1); + expect(gestures.bindDevice).toHaveBeenCalledTimes(1); expect(refFrameState(session)).toBe('expired'); const recorded = session.actions[0]; @@ -163,5 +159,5 @@ test('a second ref drag is rejected before dispatch after the first drag expires currentGeneration: 42, }); } - expect(mockDispatchGesturePlan).toHaveBeenCalledTimes(1); + expect(gestures.performTargetAuthoredDrag).toHaveBeenCalledTimes(1); }); diff --git a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts index d03abc4003..b742438900 100644 --- a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts +++ b/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts @@ -158,6 +158,8 @@ function elementReadFacts(device: DeviceInfo): RuntimeFacts { screenshot: { available: false, reason: 'owner-capability-missing' }, viewport: { available: false, reason: 'owner-capability-missing' }, focus: { available: false, reason: 'owner-capability-missing' }, + gesture: { available: false, reason: 'owner-capability-missing' }, + scroll: { available: false, reason: 'owner-capability-missing' }, typeText: { available: false, reason: 'owner-capability-missing' }, touch: { available: false, reason: 'owner-capability-missing' }, elementText: { available: false, reason: 'owner-capability-missing' }, @@ -145,6 +147,8 @@ test('appstate rejects web before Android app-state backend dispatch', async () screenshot: { available: false, reason: 'unsupported-platform-leaf' }, viewport: { available: false, reason: 'unsupported-platform-leaf' }, focus: { available: false, reason: 'unsupported-platform-leaf' }, + gesture: { available: false, reason: 'unsupported-platform-leaf' }, + scroll: { available: false, reason: 'unsupported-platform-leaf' }, typeText: { available: false, reason: 'unsupported-platform-leaf' }, touch: { available: false, reason: 'unsupported-platform-leaf' }, elementText: { available: false, reason: 'unsupported-platform-leaf' }, diff --git a/src/daemon/handlers/interaction-gesture.ts b/src/daemon/handlers/interaction-gesture.ts index b572f418e9..dbe54a8c98 100644 --- a/src/daemon/handlers/interaction-gesture.ts +++ b/src/daemon/handlers/interaction-gesture.ts @@ -23,7 +23,7 @@ import { splitRefGenerationSuffix, type Point, } from '@agent-device/kernel/snapshot'; -import { requireGestureSupported } from '../../core/capabilities.ts'; +import { resolveBoundGestureRuntime, type BoundGestureExecutor } from '../gesture-runtime.ts'; import { isActiveProviderDevice } from '../../provider-device-runtime.ts'; import { sleep } from '../../utils/timeouts.ts'; import { ensureAndroidBlockingSystemDialogReady } from '../android-system-dialog.ts'; @@ -52,6 +52,19 @@ type GestureInteractionOutcome = { recordedTargets?: { source: RecordedTargetCapture; destination: RecordedTargetCapture }; }; +/** + * A refused gesture short-circuits with the admission's own response. The retired + * `requireGestureSupported` threw, so the refusal skipped the after-command dialog check and + * returned the normalized error; returning it here preserves that control flow exactly. + */ +type GestureInteractionResult = GestureInteractionOutcome | Readonly<{ refused: DaemonResponse }>; + +function isRefusal( + result: GestureInteractionResult, +): result is Readonly<{ refused: DaemonResponse }> { + return 'refused' in result; +} + export async function dispatchGestureViaRuntime( params: GestureHandlerParams, ): Promise { @@ -63,14 +76,22 @@ export async function dispatchGestureViaRuntime( async function runGestureInteraction( params: GestureHandlerParams, session: SessionState, -): Promise { +): Promise { const input = readGesturePayload(params.req.input); const gesture = prepareGestureCommandInput(input, session); if (gesture.intent === 'pan' && params.req.internal?.gestureExecutionProfile) { gesture.executionProfile = params.req.internal.gestureExecutionProfile; } - requireGestureSupported(gesture, session.device); - const runtime = createGestureRuntime(params); + // ADR 0019 §9: the gesture input selects one execution tier and this is the request's ONE bind, + // taken before any plan is built — a drag binds here, not after its targets resolve. + const bound = await resolveBoundGestureRuntime({ + device: session.device, + input: gesture, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); + if (!bound.ok) return { refused: bound.response }; + const runtime = createGestureRuntime(params, bound.gestures); const context = { session: params.sessionName, requestId: params.req.meta?.requestId }; const result = await runPreparedGesture(runtime, context, gesture, params.req.internal); return buildGestureOutcome(input, gesture, result, params.req.flags); @@ -134,11 +155,19 @@ export async function dispatchSwipeViaRuntime( ): Promise { return await dispatchGestureInteraction(params, 'swipe', async (session) => { const input = readSwipeInput(params.req.input); - requireGestureSupported(normalizePublicSwipeMotion(input).gesture, session.device); + // One bind for the whole series: `--count N` executes the bound operation N times under a + // single binding, never one bind per repetition (ADR 0019 §9). + const bound = await resolveBoundGestureRuntime({ + device: session.device, + input: normalizePublicSwipeMotion(input).gesture, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); + if (!bound.ok) return { refused: bound.response }; const count = input.count ?? 1; const pauseMs = input.pauseMs ?? 0; const pattern = input.pattern ?? 'one-way'; - const runtime = createGestureRuntime(params); + const runtime = createGestureRuntime(params, bound.gestures); const result = await runSwipeRepetitions(runtime, params, input, count, pauseMs, pattern); return { positionals: swipeReplayPositionals(input), @@ -161,9 +190,10 @@ export async function dispatchSwipeViaRuntime( }); } -function createGestureRuntime(params: GestureHandlerParams) { +function createGestureRuntime(params: GestureHandlerParams, gestures: BoundGestureExecutor) { return createInteractionRuntime({ ...params, + gestures, pairedGestureViewport: params.req.internal?.gestureViewport, }); } @@ -171,7 +201,7 @@ function createGestureRuntime(params: GestureHandlerParams) { async function dispatchGestureInteraction( params: GestureHandlerParams, command: 'gesture' | 'swipe', - run: (session: SessionState) => Promise, + run: (session: SessionState) => Promise, ): Promise { const session = params.sessionStore.get(params.sessionName); if (!session) return noActiveSessionError(); @@ -186,6 +216,7 @@ async function dispatchGestureInteraction( phase: 'before-command', }); const outcome = await run(session); + if (isRefusal(outcome)) return outcome.refused; if (!providerDevice) { await ensureAndroidBlockingSystemDialogReady({ session, diff --git a/src/daemon/handlers/interaction-runtime.ts b/src/daemon/handlers/interaction-runtime.ts index 4d237bba62..6e9bbe4882 100644 --- a/src/daemon/handlers/interaction-runtime.ts +++ b/src/daemon/handlers/interaction-runtime.ts @@ -1,4 +1,3 @@ -import { dispatchGesturePlan, dispatchGestureViewport } from '../../core/dispatch.ts'; import { publicPlatformString } from '@agent-device/kernel/device'; import type { AgentDeviceBackend, @@ -21,11 +20,19 @@ import { buildAppleRunnerRequestOptions } from '../apple-runner-options.ts'; import { isLocalIosRunnerSession } from '../direct-ios-selector.ts'; import { confirmIosOffscreenTargetVisible } from '../offscreen-target-probe.ts'; import type { BoundTouchExecutor } from '../touch-runtime.ts'; +import type { BoundGestureExecutor } from '../gesture-runtime.ts'; +import type { DaemonCommandContext } from '../context.ts'; type InteractionRuntimeParams = InteractionHandlerParams & { captureSnapshotForSession: CaptureSnapshotForSession; pairedGestureViewport?: Rect; touchExecutor?: BoundTouchExecutor; + /** + * The request's single gesture binding (ADR 0019), supplied only by the `gesture`/`swipe` + * handler. Every other interaction command leaves it out, and the backend then exposes no + * gesture members at all — the touch leaves that share this backend execute no gestures. + */ + gestures?: BoundGestureExecutor; }; export function createInteractionRuntime(params: InteractionRuntimeParams) { @@ -58,6 +65,8 @@ function createInteractionBackend( params: InteractionRuntimeParams & { session: SessionState }, ): AgentDeviceBackend { const { req, session } = params; + const gestureContext = () => + params.contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath); return { platform: publicPlatformString(session.device), captureSnapshot: async (context, options): Promise => ({ @@ -71,16 +80,11 @@ function createInteractionBackend( preferredBackend: options?.preferredBackend, includeRects: options?.includeRects === true, signal: context.signal, - boundCapture: params.touchExecutor?.captureSnapshot, + boundCapture: params.touchExecutor?.captureSnapshot ?? params.gestures?.captureSnapshot, }, ), }), - resolveGestureViewport: async () => - params.pairedGestureViewport ?? - (await dispatchGestureViewport( - session.device, - params.contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), - )), + ...gestureBackendMembers(params, session, gestureContext), // #1542: iOS-only escape hatch for the off-screen refusal double-check. // Local (non-provider) iOS sessions get a direct, AX-tree-independent // probe (deliberately NOT skipped while postGestureStabilization is @@ -103,15 +107,38 @@ function createInteractionBackend( }) : undefined, ...touchBackendMembers(params.touchExecutor, session, req.flags), + }; +} + +/** + * The gesture members, present only for the `gesture`/`swipe` handler that bound them (R52/R54). + * Every other interaction command shares this backend and executes no gestures, so it gets + * neither member — the backend holds no gesture reach it cannot prove. + * + * A replay-supplied viewport still wins over the owner's own read, exactly as before; an owner + * with no frame read answers `undefined` and the caller derives the frame from a capture, which + * is how a Linux gesture resolves its coordinate frame today. + */ +function gestureBackendMembers( + params: InteractionRuntimeParams, + session: SessionState, + gestureContext: () => DaemonCommandContext, +): Pick { + const gestures = params.gestures; + const pairedGestureViewport = params.pairedGestureViewport; + if (!gestures) { + return pairedGestureViewport + ? { resolveGestureViewport: async (): Promise => pairedGestureViewport } + : {}; + } + return { + resolveGestureViewport: async (): Promise => + pairedGestureViewport ?? (await gestures.gestureViewport?.(gestureContext())), performGesture: async (_context, plan): Promise => { + // ADR 0014 side-effect seam: the plan is built; expire the ref frame synchronously before + // executing so a later step cannot reuse it. expireRefFrame(session); - return toBackendActionResult( - await dispatchGesturePlan( - session.device, - plan, - params.contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), - ), - ); + return toBackendActionResult(await gestures.performPlan(plan, gestureContext())); }, }; } diff --git a/src/daemon/request-router.ts b/src/daemon/request-router.ts index 01734b67fb..9462985187 100644 --- a/src/daemon/request-router.ts +++ b/src/daemon/request-router.ts @@ -429,6 +429,13 @@ async function dispatchGenericForLockedScope(params: { const runtimeExecution = await resolveGenericRuntimeExecution({ req: lockedScope.req, session, + // `scroll` parses its distance/timing flags during admission, so the resolved context is + // needed before the dispatcher builds its own. + context: lockedScope.contextFromFlags( + lockedScope.req.flags, + session.appBundleId, + session.trace?.outPath, + ), inspectFacts: lockedScope.inspectFacts, bindDevice: lockedScope.bindDevice, }); diff --git a/src/daemon/scroll-runtime.ts b/src/daemon/scroll-runtime.ts new file mode 100644 index 0000000000..ea83085513 --- /dev/null +++ b/src/daemon/scroll-runtime.ts @@ -0,0 +1,234 @@ +import { + assertExclusiveScrollDistanceInputs, + honoredScrollDurationMs, + normalizeScrollDurationMs, + resolveScrollExecutionOptions, + type ResolvedScrollExecutionOptions, + type ScrollCommandOptions, +} from '@agent-device/contracts/scroll-command'; +import { parseScrollDirection, type ScrollDirection } from '@agent-device/contracts/scroll-gesture'; +import { + resolveScrollRuntimePlan, + type ScrollRuntimePlan, +} from '@agent-device/contracts/platform-runtime-operations'; +import type { BoundDeviceRuntime } from '@agent-device/contracts/platform-runtime'; +import type { ScrollDirectionInput } from '@agent-device/contracts/scroll-runtime'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; +import { + captureScrollEdgeState, + formatScrollEdgeMessage, + runScrollEdgePasses, + type ScrollEdge, + type ScrollEdgeState, +} from '../utils/scroll-edge-state.ts'; +import { withSuccessText } from '../utils/success-text.ts'; +import type { DaemonCommandContext } from './context.ts'; +import { errorResponse } from './handlers/response.ts'; +import type { ResolvedGenericExecution } from './request-generic-dispatch.ts'; +import { resolveBoundGenericRuntime, type RuntimeAdmissionBindings } from './runtime-admission.ts'; +import { runtimeExecutionFromContext } from './snapshot-runtime-capture-input.ts'; + +type ScrollTarget = Readonly<{ + direction: ScrollDirection; + edge?: ScrollEdge; +}>; + +/** + * Both bindings come straight from the declared uses, so neither restates what a use already says: + * an ordinary scroll cannot name a capture, and an edge scroll's `captureSnapshot` is non-optional + * because `scrollEdgeUse` requires it. + */ +type BoundScrollDirection = BoundDeviceRuntime< + Extract['use'] +>; +type BoundScrollEdge = BoundDeviceRuntime['use']>; + +/** `scroll bottom` scrolls down to the edge; `scroll top` scrolls up to it. */ +function parseScrollTarget(input: string): ScrollTarget { + if (input === 'bottom') return { direction: 'down', edge: 'bottom' }; + if (input === 'top') return { direction: 'up', edge: 'top' }; + return { direction: parseScrollDirection(input) }; +} + +function assertScrollCommandInputs( + amount: number | undefined, + pixels: number | undefined, + durationMs: number | undefined, +): void { + if (amount !== undefined && !Number.isFinite(amount)) { + throw new AppError('INVALID_ARGS', 'scroll amount must be a number'); + } + normalizeScrollDurationMs(durationMs); + assertExclusiveScrollDistanceInputs({ amount, pixels }); +} + +/** + * The one place `scroll` reaches a device (ADR 0019). Admission inspects the exact owner's + * `scrollDirection` fact — plus `captureSnapshot` for an edge scroll, which cannot verify hidden + * content without one — and binds once, before the dispatcher runs. + * + * The whole positional/flag parse happens here rather than inside the executor so an invalid + * `scroll` is rejected exactly where the retired leaf rejected it: before any device work. + */ +export async function resolveBoundScrollRuntime( + params: { + device: DeviceInfo; + positionals: readonly string[]; + context: DaemonCommandContext; + } & RuntimeAdmissionBindings, +): Promise { + const directionInput = params.positionals[0]; + const amount = params.positionals[1] ? Number(params.positionals[1]) : undefined; + const pixels = params.context.pixels; + const durationMs = params.context.durationMs; + if (!directionInput) throw new AppError('INVALID_ARGS', 'scroll requires direction'); + assertScrollCommandInputs(amount, pixels, durationMs); + + const target = parseScrollTarget(directionInput); + const options = resolveScrollExecutionOptions({ amount, pixels, durationMs }, target.edge); + const plan = resolveScrollRuntimePlan({ + ...(target.edge === undefined ? {} : { edge: target.edge }), + }); + const admission = { + command: 'scroll', + device: params.device, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }; + switch (plan.kind) { + case 'direction': + return await resolveBoundGenericRuntime( + { ...admission, use: plan.use }, + async (runtime, dispatchContext) => + await executeDirectionScroll(runtime, target, options, dispatchContext), + ); + case 'edge': { + const edge = plan.edge; + return await resolveBoundGenericRuntime( + { + ...admission, + // The retired leaf refused an unsupported edge scroll by naming what the edge needs, so + // the capture requirement keeps saying so rather than collapsing into "not supported". + unavailableResponse: (unavailable) => scrollEdgeUnsupported(edge, unavailable.hint), + use: plan.use, + }, + async (runtime, dispatchContext) => + await executeEdgeScroll(runtime, edge, target, options, dispatchContext), + ); + } + } +} + +function scrollEdgeUnsupported(edge: ScrollEdge, hint: string | undefined) { + return errorResponse( + 'UNSUPPORTED_OPERATION', + `scroll ${edge} requires snapshot support to verify hidden content before scrolling`, + undefined, + hint === undefined ? undefined : { hint }, + ); +} + +/** One pass. This binding carries no capture, so an edge-style read will not type-check here. */ +async function executeDirectionScroll( + runtime: BoundScrollDirection, + target: ScrollTarget, + options: ResolvedScrollExecutionOptions, + context: DaemonCommandContext, +): Promise> { + return scrollResult( + target, + options, + 1, + (await scrollOnce(runtime, target, options, context)) ?? {}, + ); +} + +/** Repeats the pass while the verified state still moves; the capture needs no guard here. */ +async function executeEdgeScroll( + runtime: BoundScrollEdge, + edge: ScrollEdge, + target: ScrollTarget, + options: ResolvedScrollExecutionOptions, + context: DaemonCommandContext, +): Promise> { + const edgeResult = await runScrollEdgePasses({ + edge, + captureState: async (scope) => await captureEdgeState(runtime, edge, scope, context), + scroll: async () => await scrollOnce(runtime, target, options, context), + }); + return scrollResult(target, options, edgeResult.passes, edgeResult.result ?? {}); +} + +async function captureEdgeState( + runtime: BoundScrollEdge, + edge: ScrollEdge, + scope: string | undefined, + context: DaemonCommandContext, +): Promise { + return await captureScrollEdgeState({ + edge, + scope, + captureNodes: async (snapshotScope) => + ( + await runtime.operations.captureSnapshot({ + options: { + ...(context.appBundleId === undefined ? {} : { appBundleId: context.appBundleId }), + scope: snapshotScope, + }, + execution: runtimeExecutionFromContext(context), + }) + ).nodes ?? [], + }); +} + +/** The single lexical owner of the bound call (R53); the edge binding satisfies this shape too. */ +async function scrollOnce( + runtime: BoundScrollDirection, + target: ScrollTarget, + options: ResolvedScrollExecutionOptions, + context: DaemonCommandContext, +): Promise | void> { + return await runtime.operations.scrollDirection(scrollInput(target.direction, options, context)); +} + +/** The one response shape both executors report. Owner fields win, as the retired leaf had them. */ +function scrollResult( + target: ScrollTarget, + options: ScrollCommandOptions, + completedPasses: number, + interactionResult: Record, +): Record { + const durationMs = honoredScrollDurationMs(interactionResult); + return withSuccessText( + { + direction: target.direction, + ...(target.edge ? { edge: target.edge, passes: completedPasses } : {}), + ...(options.amount !== undefined ? { amount: options.amount } : {}), + ...(options.pixels !== undefined ? { pixels: options.pixels } : {}), + ...(durationMs !== undefined ? { durationMs } : {}), + ...interactionResult, + }, + formatScrollEdgeMessage( + target.direction, + target.edge, + completedPasses, + options.amount, + options.pixels, + ), + ); +} + +/** The neutral intent one scroll carries, projected from a resolved command context. */ +function scrollInput( + direction: ScrollDirection, + options: ResolvedScrollExecutionOptions, + context: DaemonCommandContext, +): ScrollDirectionInput { + return { + direction, + options, + ...(context.appBundleId === undefined ? {} : { target: { appBundleId: context.appBundleId } }), + execution: runtimeExecutionFromContext(context), + }; +} diff --git a/src/platform-runtime-gateway.test.ts b/src/platform-runtime-gateway.test.ts index a6cb76d971..662401f957 100644 --- a/src/platform-runtime-gateway.test.ts +++ b/src/platform-runtime-gateway.test.ts @@ -48,6 +48,8 @@ describe('composed platform runtime gateway', () => { elementText: unavailable, viewport: unavailable, focus: unavailable, + gesture: unavailable, + scroll: unavailable, typeText: unavailable, back: unavailable, home: unavailable, @@ -134,6 +136,8 @@ describe('composed platform runtime gateway', () => { screenshot: unavailable, viewport: unavailable, focus: unavailable, + gesture: unavailable, + scroll: unavailable, typeText: unavailable, touch: unavailable, elementText: unavailable, diff --git a/src/platform-runtime-gateway.ts b/src/platform-runtime-gateway.ts index d813c73480..1619a1bdc6 100644 --- a/src/platform-runtime-gateway.ts +++ b/src/platform-runtime-gateway.ts @@ -310,6 +310,8 @@ function unavailableProviderBinding( screenshot: unavailable, viewport: unavailable, focus: unavailable, + gesture: unavailable, + scroll: unavailable, typeText: unavailable, touch: unavailable, elementText: unavailable, @@ -339,6 +341,8 @@ function unavailableProviderFacts(runtime: ProviderDeviceRuntime, device: Device screenshot: unavailable, viewport: unavailable, focus: unavailable, + gesture: unavailable, + scroll: unavailable, typeText: unavailable, touch: unavailable, elementText: unavailable, diff --git a/src/platforms/apple/core/__tests__/interactions.test.ts b/src/platforms/apple/core/__tests__/interactions.test.ts index 4b669f16aa..0070798145 100644 --- a/src/platforms/apple/core/__tests__/interactions.test.ts +++ b/src/platforms/apple/core/__tests__/interactions.test.ts @@ -5,10 +5,13 @@ import path from 'node:path'; import { iosRunnerOverrides, performGestureApple } from '../../interactions.ts'; import { runAppleRunnerCommand } from '../runner/runner-client.ts'; import { AppError } from '@agent-device/kernel/errors'; +import { + gestureRefusalMessage, + PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT, +} from '@agent-device/contracts/gesture-admission'; import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types'; import type { RunnerCommand } from '../runner/runner-contract.ts'; import { TEXT_ENTRY_ROUTES } from '@agent-device/contracts/interactor-types'; -import { requireGestureSupported } from '../../../../core/capabilities.ts'; import { IOS_TEST_DEVICE, IOS_TEST_SIMULATOR, @@ -296,29 +299,18 @@ test('performGestureApple sends exact two-pointer pan samples through gesture', }); test('Apple admission and execution share the same multi-touch refusal', async () => { - let admissionError: AppError | undefined; - try { - requireGestureSupported( - { - intent: 'pan', - origin: { x: 100, y: 200 }, - delta: { x: 80, y: -40 }, - pointerCount: 2, - }, - IOS_TEST_DEVICE, - ); - } catch (error) { - if (error instanceof AppError) admissionError = error; - } - assert.ok(admissionError); + // Admission is now the Apple owner's `performMultiTouchGesturePlan` fact composed with the + // shared refusal wording (R42); execution keeps its defensive adapter check. The two must + // still say the same thing, which is what this pins. + const admissionMessage = gestureRefusalMessage(IOS_TEST_DEVICE, 'multi-touch', 'pan'); await assert.rejects( () => performGestureApple(IOS_TEST_DEVICE, {}, {}, twoFingerPanPlan()), (error: unknown) => error instanceof AppError && error.code === 'UNSUPPORTED_OPERATION' && - error.message === admissionError.message && - error.details?.hint === admissionError.details?.hint, + error.message === admissionMessage && + error.details?.hint === PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT, ); assert.equal(mockRunAppleRunnerCommand.mock.calls.length, 0); }); diff --git a/test/integration/smoke-tvos-platform-coverage.test.ts b/test/integration/smoke-tvos-platform-coverage.test.ts index 66e2e36278..ffbc43ec81 100644 --- a/test/integration/smoke-tvos-platform-coverage.test.ts +++ b/test/integration/smoke-tvos-platform-coverage.test.ts @@ -5,11 +5,9 @@ import test from 'node:test'; import { TVOS_SIMULATOR } from '../../src/__tests__/test-utils/device-fixtures.ts'; import { PUBLIC_COMMANDS } from '../../src/command-catalog.ts'; -import { - isCommandSupportedOnDevice, - requireGestureSupported, -} from '../../src/core/capabilities.ts'; -import { AppError } from '@agent-device/kernel/errors'; +import { isCommandSupportedOnDevice } from '../../src/core/capabilities.ts'; +import { createPlatformRuntimeGateway } from '../../src/platform-runtime.ts'; +import { gestureRefusalMessage } from '@agent-device/contracts/gesture-admission'; import { TVOS_COVERAGE_GAP_ISSUE, TVOS_PLATFORM_COVERAGE, @@ -103,23 +101,26 @@ test('tvOS capability denials match the mechanical capability matrix', () => { assert.equal('admission' in audio ? audio.admission : undefined, 'host-dependent'); }); -test('tvOS gesture contract preserves the typed multi-touch denial', () => { +test('tvOS gesture contract preserves the typed multi-touch denial', async () => { assert.equal(TVOS_PLATFORM_COVERAGE[PUBLIC_COMMANDS.gesture].level, 'command-contract'); - assert.throws( - () => - requireGestureSupported( - { - intent: 'pan', - origin: { x: 100, y: 200 }, - delta: { x: 40, y: -20 }, - pointerCount: 2, - }, - TVOS_SIMULATOR, - ), - (error: unknown) => - error instanceof AppError && - error.code === 'UNSUPPORTED_OPERATION' && - /tvOS has no touch input/.test(String(error.details?.hint)), + // R42: the denial is now the Apple owner's own fact for the tvOS leaf, and the refusal a + // gesture reports is that fact's hint under the shared wording. + const facts = await createPlatformRuntimeGateway({ + resolveSessionArtifacts: () => ({ + outputPath: '/sessions/tvos/app.log', + pidPath: '/sessions/tvos/app-log.pid', + }), + sessionsDir: '/sessions', + }).inspectFacts(TVOS_SIMULATOR); + const multiTouch = facts.operations.performMultiTouchGesturePlan; + assert.equal(multiTouch.available, false); + assert.match( + String(multiTouch.available === false ? multiTouch.hint : ''), + /tvOS has no touch input/, + ); + assert.equal( + gestureRefusalMessage(TVOS_SIMULATOR, 'multi-touch', 'pan'), + 'gesture pan is not supported on tvOS', ); }); diff --git a/test/integration/smoke-web-platform-coverage.test.ts b/test/integration/smoke-web-platform-coverage.test.ts index a760720011..b2bb3078f9 100644 --- a/test/integration/smoke-web-platform-coverage.test.ts +++ b/test/integration/smoke-web-platform-coverage.test.ts @@ -41,8 +41,8 @@ test('web coverage exhaustively classifies the public catalog', () => { test('web coverage report has the expected classification counts', () => { assert.deepEqual(WEB_PLATFORM_COVERAGE_CLASSIFICATION_SUMMARY, { - capabilityDenial: 9, - contract: 18, + capabilityDenial: 7, + contract: 20, gap: 15, live: 12, total: 54, diff --git a/test/integration/tvos-e2e/coverage-manifest.ts b/test/integration/tvos-e2e/coverage-manifest.ts index 41c08bc929..4b0a49a208 100644 --- a/test/integration/tvos-e2e/coverage-manifest.ts +++ b/test/integration/tvos-e2e/coverage-manifest.ts @@ -173,7 +173,7 @@ export const TVOS_PLATFORM_COVERAGE = { 'the existing provider scenario maps Back to the tvOS Menu remote press', ), [C.gesture]: contract( - 'src/core/__tests__/gesture-capabilities.test.ts', + 'src/daemon/__tests__/gesture-admission-parity.test.ts', 'TV, spatial, watch, desktop, Linux, and web gesture policy stays explicit', 'the typed Apple gesture policy refuses tvOS multi-touch while preserving the narrower gesture contract', ), diff --git a/test/integration/web-e2e/coverage-manifest.ts b/test/integration/web-e2e/coverage-manifest.ts index 17a958c5af..2a0f4344a5 100644 --- a/test/integration/web-e2e/coverage-manifest.ts +++ b/test/integration/web-e2e/coverage-manifest.ts @@ -157,7 +157,13 @@ export const WEB_PLATFORM_COVERAGE = { 'back/home/orientation/tv-remote/keyboard never carried a web capability bucket', 'the exact-owner runtime fact rejects native back navigation on the web target', ), - [C.gesture]: denial('Web capability model rejects touch gesture input'), + // R52/R54: the web refusal moved from the capability matrix to the web owner's own gesture + // facts, so the evidence is the cell test rather than a mechanical matrix denial. + [C.gesture]: contract( + 'packages/platform-web/src/runtime.test.ts', + 'admits web scrolling and refuses every gesture tier', + 'the web runtime owner declares every gesture tier unavailable', + ), [C.home]: contract( 'packages/platform-web/src/runtime.test.ts', 'back/home/orientation/tv-remote/keyboard never carried a web capability bucket', @@ -178,7 +184,11 @@ export const WEB_PLATFORM_COVERAGE = { 'scroll by pixels', 'web scroll moves the provider-backed page by the requested pixels', ), - [C.swipe]: denial('Web capability model rejects touch swipe input'), + [C.swipe]: contract( + 'packages/platform-web/src/runtime.test.ts', + 'admits web scrolling and refuses every gesture tier', + 'swipe shares the gesture tiers the web runtime owner declares unavailable', + ), [C.focus]: contract( 'src/core/__tests__/web-interactor.test.ts', 'web interactor delegates first-slice operations to the scoped provider',