diff --git a/packages/contracts/src/snapshot-tree.ts b/packages/contracts/src/snapshot-tree.ts index dfb6233ea..e98345976 100644 --- a/packages/contracts/src/snapshot-tree.ts +++ b/packages/contracts/src/snapshot-tree.ts @@ -31,9 +31,7 @@ export function findSnapshotAncestor( return null; } -/** - * Returns the nearest ancestor matching `predicate`; false means keep walking. - */ +/** Returns the nearest ancestor matching `predicate`; false means keep walking. */ export function findNearestAncestor( nodes: SnapshotNode[], node: SnapshotNode, diff --git a/packages/contracts/src/snapshot.test.ts b/packages/contracts/src/snapshot.test.ts index 485f5d350..672eb655f 100644 --- a/packages/contracts/src/snapshot.test.ts +++ b/packages/contracts/src/snapshot.test.ts @@ -74,10 +74,9 @@ test('findNearestAncestor adapts a predicate to the shared tree walk', () => { { ref: 'e20', index: 20, parentIndex: 10, type: 'Cell' }, ]; - assert.equal( - findNearestAncestor(nodes, nodes[1]!, (ancestor) => ancestor.type === 'Window')?.index, - 10, - ); + const isWindow = (ancestor: SnapshotNode) => ancestor.type === 'Window'; + + assert.equal(findNearestAncestor(nodes, nodes[1]!, isWindow)?.index, 10); }); test('snapshot tree and scroll semantics identify nodes through their stable indexes', () => { diff --git a/src/core/interaction-targeting.fixtures.ts b/src/core/interaction-targeting.fixtures.ts index 87ffc8e00..1f88de3d2 100644 --- a/src/core/interaction-targeting.fixtures.ts +++ b/src/core/interaction-targeting.fixtures.ts @@ -74,3 +74,110 @@ export const ELEMENT14_DISTINCT_SUBTREE_NODES: RawSnapshotNode[] = [ hittable: true, }, ]; + +/** + * One node per decision `resolveActionableTouchResolution` can reach, so an + * indexed pass and an unindexed one can be compared across the whole policy + * rather than on the branch a single example happens to hit: a same-rect + * actionable descendant (1 -> 2), semantic targets (3, 4), a nonhittable leaf + * under a hittable ancestor (5), both overly-broad shapes — a scrolling + * container (7) and the viewport-sized root (10) — a covered node (8), and a + * parentless, rectless node with no usable target at all (9). + */ +export const INDEXED_PARITY_POLICY_NODES: RawSnapshotNode[] = [ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeOther', + label: 'Save wrapper', + rect: { x: 20, y: 100, width: 120, height: 44 }, + hittable: false, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'XCUIElementTypeImage', + identifier: 'save-hit-area', + rect: { x: 20, y: 100, width: 120, height: 44 }, + hittable: true, + }, + { + index: 3, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Save', + rect: { x: 20, y: 200, width: 100, height: 40 }, + hittable: false, + }, + { + index: 4, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeCell', + label: 'Account row', + rect: { x: 10, y: 260, width: 370, height: 60 }, + hittable: true, + }, + { + index: 5, + depth: 2, + parentIndex: 4, + type: 'XCUIElementTypeStaticText', + label: 'Account', + rect: { x: 24, y: 272, width: 80, height: 20 }, + hittable: false, + }, + { + index: 6, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeScrollView', + rect: { x: 0, y: 340, width: 390, height: 300 }, + hittable: true, + }, + { + index: 7, + depth: 2, + parentIndex: 6, + type: 'XCUIElementTypeOther', + label: 'Feed item', + rect: { x: 20, y: 360, width: 200, height: 40 }, + hittable: false, + }, + { + index: 8, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Under overlay', + rect: { x: 20, y: 700, width: 100, height: 20 }, + hittable: false, + interactionBlocked: 'covered', + }, + { + index: 9, + depth: 0, + type: 'XCUIElementTypeOther', + label: 'Virtual item', + hittable: false, + }, + { + index: 10, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Status', + rect: { x: 20, y: 760, width: 60, height: 20 }, + hittable: false, + }, +]; diff --git a/src/core/interaction-targeting.test.ts b/src/core/interaction-targeting.test.ts index 250e1aa06..5e544e20f 100644 --- a/src/core/interaction-targeting.test.ts +++ b/src/core/interaction-targeting.test.ts @@ -9,11 +9,13 @@ import { import { makeSnapshotState } from '../__tests__/test-utils/snapshot-builders.ts'; import { classifyActionableTouchCandidates, + createActionableTouchResolver, resolveActionableTouchResolution, } from './interaction-targeting.ts'; import { ELEMENT14_DISTINCT_SUBTREE_NODES, EQUIVALENT_WRAPPER_CHAIN_NODES, + INDEXED_PARITY_POLICY_NODES, } from './interaction-targeting.fixtures.ts'; test('collapses one same-label wrapper chain to its shared actionable node', () => { @@ -201,3 +203,31 @@ test('falls back to the original node when no usable touch target exists', () => assert.equal(resolution.reason, 'original'); assert.equal(resolution.node.label, 'Virtual item'); }); + +test('the batch resolver preserves every actionability policy branch', () => { + const snapshot = makeSnapshotState(INDEXED_PARITY_POLICY_NODES); + const resolveTouch = createActionableTouchResolver(snapshot.nodes); + + const unindexed = snapshot.nodes.map((node) => + resolveActionableTouchResolution(snapshot.nodes, node), + ); + const indexed = snapshot.nodes.map(resolveTouch); + + assert.deepEqual(indexed, unindexed); + assert.deepEqual( + indexed.map((resolution) => [resolution.node.index, resolution.reason]), + [ + [0, 'hittable-ancestor'], + [2, 'same-rect-descendant'], + [2, 'hittable-ancestor'], + [3, 'semantic-target'], + [4, 'semantic-target'], + [4, 'hittable-ancestor'], + [6, 'hittable-ancestor'], + [7, 'overly-broad-ancestor'], + [8, 'covered'], + [9, 'original'], + [10, 'overly-broad-ancestor'], + ], + ); +}); diff --git a/src/core/interaction-targeting.ts b/src/core/interaction-targeting.ts index 364a17360..9004df070 100644 --- a/src/core/interaction-targeting.ts +++ b/src/core/interaction-targeting.ts @@ -3,6 +3,7 @@ import { centerOfRect } from '@agent-device/kernel/snapshot'; import { containsPoint, pickLargestRect } from '@agent-device/kernel/rect'; import { findNearestAncestor, + findSnapshotAncestor, normalizeType, isViewportRootNode, } from '@agent-device/contracts/snapshot'; @@ -13,20 +14,7 @@ import { resolveRectCenter, } from '../utils/rect-center.ts'; import { intersectArea } from '../utils/screenshot-geometry.ts'; - -const SEMANTIC_TOUCH_ROLE_FRAGMENTS = [ - 'button', - 'link', - 'menuitem', - 'tabitem', - 'textfield', - 'searchfield', - 'securetextfield', - 'checkbox', - 'radio', - 'switch', - 'cell', -]; +import { isSemanticTouchTarget } from './touch-semantics.ts'; type ActionableTouchResolutionReason = | 'same-rect-descendant' @@ -41,29 +29,32 @@ type ActionableTouchResolution = { reason: ActionableTouchResolutionReason; }; +type ActionableTouchIndex = { + nodesByIndex: ReadonlyMap; + childrenByParentIndex: ReadonlyMap; + viewportRootRects: readonly Rect[]; +}; + type ActionableTouchCandidateClassification = | { kind: 'equivalent'; node: SnapshotNode } | { kind: 'ambiguous'; candidates: SnapshotNode[] }; -/** - * Mutating selector matches may collapse only when their tree structure proves - * that they describe one action: every candidate is on one ancestor/descendant - * chain and every candidate resolves to the same actionable node. Geometry may - * help resolve a wrapper to its control, but never chooses between branches. - */ export function classifyActionableTouchCandidates( nodes: SnapshotNode[], candidates: SnapshotNode[], ): ActionableTouchCandidateClassification { const first = candidates[0]; if (!first) return { kind: 'ambiguous', candidates }; - const byIndex = new Map(nodes.map((node) => [node.index, node])); - if (!candidatesFormSingleAncestryChain(candidates, byIndex)) { + const index = buildActionableTouchIndex(nodes); + if (!candidatesFormSingleAncestryChain(candidates, index.nodesByIndex)) { return { kind: 'ambiguous', candidates }; } - const actionable = resolveActionableTouchResolution(nodes, first).node; + const actionable = resolveActionableTouchResolutionWithIndex(nodes, first, index).node; for (const candidate of candidates.slice(1)) { - if (resolveActionableTouchResolution(nodes, candidate).node.index !== actionable.index) { + if ( + resolveActionableTouchResolutionWithIndex(nodes, candidate, index).node.index !== + actionable.index + ) { return { kind: 'ambiguous', candidates }; } } @@ -101,13 +92,6 @@ function isAncestorOf( return false; } -/** - * The tree's viewport root as an interaction target: a viewport root node whose - * rect is exactly the tree root's. Promotion that lands here has retargeted to - * "the screen" rather than to the thing that matched, which is why the - * `hittable-ancestor-below-root` promotion stage declines it and `find` - * excludes it from candidacy and ranking. - */ export function isRootInteractionContainer( node: SnapshotNode, root: SnapshotNode | undefined, @@ -124,46 +108,84 @@ export function isRootInteractionContainer( ); } -/** @internal Exposed for focused policy tests. */ export function resolveActionableTouchResolution( nodes: SnapshotNode[], node: SnapshotNode, +): ActionableTouchResolution { + return resolveActionableTouchResolutionWithIndex(nodes, node); +} + +/** Resolves many candidates against one snapshot without rebuilding its indexes. */ +export function createActionableTouchResolver( + nodes: SnapshotNode[], +): (node: SnapshotNode) => ActionableTouchResolution { + const index = buildActionableTouchIndex(nodes); + return (node) => resolveActionableTouchResolutionWithIndex(nodes, node, index); +} + +function resolveActionableTouchResolutionWithIndex( + nodes: SnapshotNode[], + node: SnapshotNode, + index?: ActionableTouchIndex, ): ActionableTouchResolution { if (isSnapshotNodeInteractionBlocked(node)) { return { node, reason: 'covered' }; } - const descendant = findPreferredActionableDescendant(nodes, node); - if (descendant?.rect && resolveRectCenter(descendant.rect)) { - return { node: descendant, reason: 'same-rect-descendant' }; - } - if (isSemanticTouchTarget(node) && node.rect && resolveRectCenter(node.rect)) { - return { node, reason: 'semantic-target' }; - } - const ancestor = findNearestHittableAncestor(nodes, node); - if ( - ancestor?.rect && - !isSnapshotNodeInteractionBlocked(ancestor) && - resolveRectCenter(ancestor.rect) - ) { - if (isOverlyBroadAncestor(node, ancestor, nodes)) { - return { node, reason: 'overly-broad-ancestor' }; - } - return { node: ancestor, reason: 'hittable-ancestor' }; + return ( + resolvePreferredDescendant(nodes, node, index) ?? + resolveSemanticTarget(node) ?? + resolveHittableAncestor(nodes, node, index) ?? { node, reason: 'original' } + ); +} + +function resolvePreferredDescendant( + nodes: SnapshotNode[], + node: SnapshotNode, + index: ActionableTouchIndex | undefined, +): ActionableTouchResolution | null { + const descendant = findPreferredActionableDescendant(nodes, node, index); + return descendant?.rect && resolveRectCenter(descendant.rect) + ? { node: descendant, reason: 'same-rect-descendant' } + : null; +} + +function resolveSemanticTarget(node: SnapshotNode): ActionableTouchResolution | null { + return isSemanticTouchTarget(node) && node.rect && resolveRectCenter(node.rect) + ? { node, reason: 'semantic-target' } + : null; +} + +function resolveHittableAncestor( + nodes: SnapshotNode[], + node: SnapshotNode, + index: ActionableTouchIndex | undefined, +): ActionableTouchResolution | null { + const ancestor = findNearestHittableAncestor(nodes, node, index); + if (!ancestor?.rect || isSnapshotNodeInteractionBlocked(ancestor)) return null; + if (!resolveRectCenter(ancestor.rect)) return null; + if (isOverlyBroadAncestor(node, ancestor, nodes, index)) { + return { node, reason: 'overly-broad-ancestor' }; } - return { node, reason: 'original' }; + return { node: ancestor, reason: 'hittable-ancestor' }; } function findNearestHittableAncestor( nodes: SnapshotNode[], node: SnapshotNode, + index: ActionableTouchIndex | undefined, ): SnapshotNode | null { if (node.hittable) return node; - return findNearestAncestor(nodes, node, (parent) => parent.hittable === true); + const isHittable = (parent: SnapshotNode) => parent.hittable === true; + if (!index) return findNearestAncestor(nodes, node, isHittable); + return findSnapshotAncestor(nodes, node, index.nodesByIndex, (parent) => + isHittable(parent) ? parent : null, + ); } function findPreferredActionableDescendant( nodes: SnapshotNode[], node: SnapshotNode, + index: ActionableTouchIndex | undefined, ): SnapshotNode | null { const targetRect = normalizeRect(node.rect); if (!targetRect) return null; @@ -172,14 +194,11 @@ function findPreferredActionableDescendant( const visited = new Set(); while (!visited.has(current.ref)) { visited.add(current.ref); - const sameRectChildren = nodes.filter((candidate) => { - if ( - candidate.parentIndex !== current.index || - !candidate.hittable || - isSnapshotNodeInteractionBlocked(candidate) - ) { - return false; - } + const children = index + ? (index.childrenByParentIndex.get(current.index) ?? []) + : nodes.filter((candidate) => candidate.parentIndex === current.index); + const sameRectChildren = children.filter((candidate) => { + if (!candidate.hittable || isSnapshotNodeInteractionBlocked(candidate)) return false; const candidateRect = normalizeRect(candidate.rect); return candidateRect ? areRectsApproximatelyEqual(candidateRect, targetRect) : false; }); @@ -192,29 +211,11 @@ function findPreferredActionableDescendant( return current === node ? null : current; } -/** - * THE canonical interactive-role classification for touch: a node whose - * type/role/subrole names a control that independently receives taps. Shared - * by the hittable-ancestor promotion above and #1280's press-retarget - * competing-descendant guard (`press-retarget.ts`) — one list, never a - * parallel copy. - */ -export function isSemanticTouchTarget(node: SnapshotNode): boolean { - const roles = [node.type, node.role, node.subrole].map((value) => normalizeType(value ?? '')); - return roles.some(isSemanticTouchRole); -} - -function isSemanticTouchRole(role: string): boolean { - // Match Tab exactly so broad roles like Table/TabBar do not become touch targets. - return ( - role === 'tab' || SEMANTIC_TOUCH_ROLE_FRAGMENTS.some((fragment) => role.includes(fragment)) - ); -} - function isOverlyBroadAncestor( node: SnapshotNode, ancestor: SnapshotNode, nodes: SnapshotNode[], + index: ActionableTouchIndex | undefined, ): boolean { const nodeRect = normalizeRect(node.rect); const ancestorRect = normalizeRect(ancestor.rect); @@ -222,7 +223,7 @@ function isOverlyBroadAncestor( if (isScrollingContainer(ancestor) && !areRectsApproximatelyEqual(nodeRect, ancestorRect)) { return true; } - const rootViewportRect = resolveRootViewportRect(nodes, nodeRect); + const rootViewportRect = resolveRootViewportRect(nodes, nodeRect, index); if (!rootViewportRect) return false; if (!isRectViewportSized(ancestorRect, rootViewportRect)) return false; return !areRectsApproximatelyEqual(nodeRect, ancestorRect); @@ -242,12 +243,18 @@ function isScrollingContainer(node: SnapshotNode): boolean { ); } -function resolveRootViewportRect(nodes: SnapshotNode[], targetRect: Rect): Rect | null { +function resolveRootViewportRect( + nodes: SnapshotNode[], + targetRect: Rect, + index: ActionableTouchIndex | undefined, +): Rect | null { const targetCenter = centerOfRect(targetRect); - const viewportRects = nodes - .filter(isViewportRootNode) - .map((node) => normalizeRect(node.rect)) - .filter((rect): rect is Rect => rect !== null); + const viewportRects = + index?.viewportRootRects ?? + nodes + .filter(isViewportRootNode) + .map((node) => normalizeRect(node.rect)) + .filter((rect): rect is Rect => rect !== null); if (viewportRects.length === 0) return null; const containingRects = viewportRects.filter((rect) => @@ -256,6 +263,25 @@ function resolveRootViewportRect(nodes: SnapshotNode[], targetRect: Rect): Rect return pickLargestRect(containingRects.length > 0 ? containingRects : viewportRects); } +function buildActionableTouchIndex(nodes: readonly SnapshotNode[]): ActionableTouchIndex { + const nodesByIndex = new Map(); + const childrenByParentIndex = new Map(); + const viewportRootRects: Rect[] = []; + for (const node of nodes) { + nodesByIndex.set(node.index, node); + if (typeof node.parentIndex === 'number') { + const children = childrenByParentIndex.get(node.parentIndex); + if (children) children.push(node); + else childrenByParentIndex.set(node.parentIndex, [node]); + } + if (isViewportRootNode(node)) { + const rect = normalizeRect(node.rect); + if (rect) viewportRootRects.push(rect); + } + } + return { nodesByIndex, childrenByParentIndex, viewportRootRects }; +} + function isRectViewportSized(rect: Rect, viewportRect: Rect): boolean { const overlapArea = intersectArea(rect, viewportRect); const rectArea = rect.width * rect.height; diff --git a/src/core/interaction-touch-point.ts b/src/core/interaction-touch-point.ts index 20edf683d..f2751a4c3 100644 --- a/src/core/interaction-touch-point.ts +++ b/src/core/interaction-touch-point.ts @@ -5,7 +5,7 @@ import { normalizeRect, resolveRectCenter, } from '../utils/rect-center.ts'; -import { isSemanticTouchTarget } from './interaction-targeting.ts'; +import { isSemanticTouchTarget } from './touch-semantics.ts'; export type InteractionTouchPointResolution = | { kind: 'resolved'; point: Point; strategy: 'center' | 'parent-owned' } diff --git a/src/core/press-retarget.test.ts b/src/core/press-retarget.test.ts index 5eba30fbd..0ce8e9a35 100644 --- a/src/core/press-retarget.test.ts +++ b/src/core/press-retarget.test.ts @@ -397,7 +397,7 @@ test('resolvePressRecordingTarget P2a contrast: a container with a UNIQUE id kee // --------------------------------------------------------------------------- // P2b (#1280 re-review): the guard is built from the canonical interactive -// classification (`isSemanticTouchTarget`, core/interaction-targeting.ts), +// classification (`isSemanticTouchTarget`, core/touch-semantics.ts), // not a parallel list — roles the old private fragment list missed must // block. And a geometry condition: the selected descendant's rect center // must lie INSIDE the container's rect, else the replay tap point is not diff --git a/src/core/press-retarget.ts b/src/core/press-retarget.ts index c3e7565e9..b062fb77e 100644 --- a/src/core/press-retarget.ts +++ b/src/core/press-retarget.ts @@ -18,7 +18,7 @@ import { resolveRectCenter } from '../utils/rect-center.ts'; import { demoteNonUniqueLocalIdentity, readNodeLocalIdentity } from '@agent-device/ad-script'; import { buildIndexMap } from '../replay/target-evidence-tree.ts'; import { normalizeSelectorText } from '@agent-device/selectors'; -import { isSemanticTouchTarget } from './interaction-targeting.ts'; +import { isSemanticTouchTarget } from './touch-semantics.ts'; /** * Rule 3's fail-closed guard: a descendant that could independently receive diff --git a/src/core/touch-semantics.ts b/src/core/touch-semantics.ts new file mode 100644 index 000000000..32c3b5e9b --- /dev/null +++ b/src/core/touch-semantics.ts @@ -0,0 +1,23 @@ +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { normalizeType } from '@agent-device/contracts/snapshot'; + +const TOUCH_ROLE_FRAGMENTS = [ + 'button', + 'link', + 'menuitem', + 'tabitem', + 'textfield', + 'searchfield', + 'securetextfield', + 'checkbox', + 'radio', + 'switch', + 'cell', +]; + +export function isSemanticTouchTarget(node: SnapshotNode): boolean { + const roles = [node.type, node.role, node.subrole].map((value) => normalizeType(value ?? '')); + return roles.some( + (role) => role === 'tab' || TOUCH_ROLE_FRAGMENTS.some((fragment) => role.includes(fragment)), + ); +} diff --git a/src/daemon/handlers/__tests__/find-match-ranking.test.ts b/src/daemon/handlers/__tests__/find-match-ranking.test.ts new file mode 100644 index 000000000..72dc4cb55 --- /dev/null +++ b/src/daemon/handlers/__tests__/find-match-ranking.test.ts @@ -0,0 +1,183 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { RawSnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; +import { makeSnapshotState } from '../../../__tests__/test-utils/snapshot-builders.ts'; +import { preferOnscreenMatches } from '../find-match-ranking.ts'; + +const VIEWPORT = { x: 0, y: 0, width: 390, height: 844 }; +const DUPLICATE_MATCH_COUNT = 32; + +function observeWholeTreeScans(nodes: SnapshotState['nodes']): { + observed: SnapshotState['nodes']; + scans: { iterations: number; filter: number; map: number }; +} { + const scans = { iterations: 0, filter: 0, map: 0 }; + const observed = new Proxy(nodes, { + get(target, property) { + if (property === Symbol.iterator) scans.iterations += 1; + if (property === 'filter' || property === 'map') scans[property] += 1; + const value = Reflect.get(target, property) as unknown; + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + return { observed, scans }; +} + +/** + * Duplicate-heavy and deliberately worst-case: every match is nonsemantic, + * nonhittable and childless, so the old implementation had to run all three + * whole-tree lookups — same-rect descendants, the nearest hittable ancestor's + * index map, and the viewport roots an overly-broad ancestor is measured + * against — once per candidate. Widths shrink as the list goes on so the + * area tie-break has to reverse the input order. + */ +function duplicateHeavyCapture(): SnapshotState { + const raw: RawSnapshotNode[] = [ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeApplication', + rect: VIEWPORT, + hittable: true, + }, + ]; + for (let position = 0; position < DUPLICATE_MATCH_COUNT; position += 1) { + raw.push({ + index: position + 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeOther', + label: 'Item', + rect: { + x: 20, + y: 40 + position * 20, + width: 100 + (DUPLICATE_MATCH_COUNT - position), + height: 16, + }, + hittable: false, + }); + } + return makeSnapshotState(raw); +} + +/** + * One node per scoring branch the ranking rules distinguish: a semantic target, + * a same-rect actionable descendant, a self-hittable node below the root, a + * node whose only ancestor is the viewport-sized root, and a second semantic + * target with the first one's exact area. + */ +const MIXED_SCORE_NODES: RawSnapshotNode[] = [ + { index: 0, depth: 0, type: 'XCUIElementTypeApplication', rect: VIEWPORT, hittable: true }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Sync', + rect: { x: 20, y: 100, width: 100, height: 40 }, + hittable: false, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeOther', + label: 'Sync', + rect: { x: 20, y: 200, width: 120, height: 40 }, + hittable: false, + }, + { + index: 3, + depth: 2, + parentIndex: 2, + type: 'XCUIElementTypeImage', + identifier: 'sync-hit-area', + rect: { x: 20, y: 200, width: 120, height: 40 }, + hittable: true, + }, + { + index: 4, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeOther', + label: 'Sync', + rect: { x: 20, y: 300, width: 80, height: 30 }, + hittable: true, + }, + { + index: 5, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Sync', + rect: { x: 20, y: 400, width: 60, height: 20 }, + hittable: false, + }, + { + index: 6, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Sync', + rect: { x: 200, y: 100, width: 100, height: 40 }, + hittable: false, + }, +]; + +test('a multi-match ranking pass scans the tree once', () => { + const snapshot = duplicateHeavyCapture(); + const matches = snapshot.nodes.slice(1); + const { observed, scans } = observeWholeTreeScans(snapshot.nodes); + + const ranked = preferOnscreenMatches(matches, observed); + + assert.deepEqual(scans, { iterations: 1, filter: 0, map: 0 }); + assert.deepEqual( + ranked.map((node) => node.ref), + [...matches].reverse().map((node) => node.ref), + ); +}); + +test('a single match returns without indexing the tree', () => { + const snapshot = duplicateHeavyCapture(); + const { observed, scans } = observeWholeTreeScans(snapshot.nodes); + + const ranked = preferOnscreenMatches([snapshot.nodes[1]!], observed); + + assert.deepEqual(scans, { iterations: 0, filter: 0, map: 0 }); + assert.deepEqual( + ranked.map((node) => node.ref), + [snapshot.nodes[1]!.ref], + ); +}); + +test('a capture without a root rect returns matches unranked and unindexed', () => { + const snapshot = makeSnapshotState([ + { index: 0, depth: 0, type: 'XCUIElementTypeApplication', hittable: true }, + ...MIXED_SCORE_NODES.slice(1), + ]); + const matches = snapshot.nodes.slice(1); + const { observed, scans } = observeWholeTreeScans(snapshot.nodes); + + const ranked = preferOnscreenMatches(matches, observed); + + assert.deepEqual(scans, { iterations: 0, filter: 0, map: 0 }); + assert.deepEqual( + ranked.map((node) => node.ref), + matches.map((node) => node.ref), + ); +}); + +test('ranks by actionability score, then smallest rect, then input order', () => { + const snapshot = makeSnapshotState(MIXED_SCORE_NODES); + // Deliberately out of tree order so score, area, and input position each have + // to decide a pair: [static text, second button, first button, wrapper, self-hittable]. + const matches = [5, 6, 1, 2, 4].map((index) => snapshot.nodes[index]!); + + const ranked = preferOnscreenMatches(matches, snapshot.nodes); + + assert.deepEqual( + ranked.map((node) => node.ref), + ['e7', 'e2', 'e3', 'e5', 'e6'], + ); +}); diff --git a/src/daemon/handlers/find-match-ranking.ts b/src/daemon/handlers/find-match-ranking.ts new file mode 100644 index 000000000..0c3a87ca9 --- /dev/null +++ b/src/daemon/handlers/find-match-ranking.ts @@ -0,0 +1,86 @@ +import { centerOfRect, type SnapshotState } from '@agent-device/kernel/snapshot'; +import { + createActionableTouchResolver, + isRootInteractionContainer, + resolveActionableTouchResolution, +} from '../../core/interaction-targeting.ts'; + +/** + * How `find` orders the candidates its locator matched, before the ambiguity + * contract in `find-match-resolution.ts` either refuses them or narrows with + * `--first`/`--last`. Ordering is a separate question from matching: it asks + * which candidate an agent most plausibly meant to touch, using the same + * actionability policy the interaction itself will run. + */ +export function preferOnscreenMatches( + matches: SnapshotState['nodes'], + nodes: SnapshotState['nodes'], +): SnapshotState['nodes'] { + const viewport = nodes[0]?.rect; + if (!viewport) return matches; + const onscreen = matches.filter((node) => { + if (!node.rect) return false; + const center = centerOfRect(node.rect); + return ( + center.x >= viewport.x && + center.x <= viewport.x + viewport.width && + center.y >= viewport.y && + center.y <= viewport.y + viewport.height + ); + }); + const preferred = onscreen.length > 0 ? onscreen : matches; + if (preferred.length < 2) return preferred; + return rankInteractiveMatches(preferred, nodes, createActionableTouchResolver(nodes)); +} + +function rankInteractiveMatches( + matches: SnapshotState['nodes'], + nodes: SnapshotState['nodes'], + resolveTouch: ReturnType, +): SnapshotState['nodes'] { + return matches + .map((node, index) => ({ + node, + index, + score: interactiveMatchScore(node, nodes, resolveTouch), + })) + .sort((left, right) => { + if (right.score !== left.score) return right.score - left.score; + return rectArea(left.node) - rectArea(right.node) || left.index - right.index; + }) + .map((entry) => entry.node); +} + +function interactiveMatchScore( + node: SnapshotState['nodes'][number], + nodes: SnapshotState['nodes'], + resolveTouch: ReturnType, +): number { + const resolution = resolveTouch(node); + if (resolution.reason === 'covered') return 0; + const resolved = resolvedTouchScore(resolution, nodes[0]); + if (resolved > 0) return resolved; + if (node.hittable && node.rect && !isRootInteractionContainer(node, nodes[0])) return 3; + return node.rect ? 1 : 0; +} + +function resolvedTouchScore( + resolution: ReturnType, + root: SnapshotState['nodes'][number] | undefined, +): number { + if (!resolution.node.rect) return 0; + if (resolution.reason === 'semantic-target' || resolution.reason === 'same-rect-descendant') { + return 4; + } + if ( + resolution.reason === 'hittable-ancestor' && + !isRootInteractionContainer(resolution.node, root) + ) { + return 2; + } + return 0; +} + +function rectArea(node: SnapshotState['nodes'][number]): number { + return node.rect ? node.rect.width * node.rect.height : Number.POSITIVE_INFINITY; +} diff --git a/src/daemon/handlers/find-match-resolution.ts b/src/daemon/handlers/find-match-resolution.ts index 0aa569f64..fa5912cab 100644 --- a/src/daemon/handlers/find-match-resolution.ts +++ b/src/daemon/handlers/find-match-resolution.ts @@ -5,11 +5,9 @@ import { } from '@agent-device/selectors'; import { listSelectorPipelineMatches } from '../../core/selector-pipeline.ts'; import { SELECTOR_PIPELINE_POLICIES } from '../../core/selector-pipeline-policy.ts'; -import { centerOfRect, type SnapshotState } from '@agent-device/kernel/snapshot'; -import { - isRootInteractionContainer, - resolveActionableTouchResolution, -} from '../../core/interaction-targeting.ts'; +import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import { isRootInteractionContainer } from '../../core/interaction-targeting.ts'; +import { preferOnscreenMatches } from './find-match-ranking.ts'; import { formatSnapshotLine } from '../../snapshot/snapshot-lines.ts'; import type { ElementMatchCandidateDetails } from '../../utils/error-candidates.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts'; @@ -91,71 +89,6 @@ function narrowMultipleMatches( return null; } -function preferOnscreenMatches( - matches: SnapshotState['nodes'], - nodes: SnapshotState['nodes'], -): SnapshotState['nodes'] { - const viewport = nodes[0]?.rect; - if (!viewport) return matches; - const onscreen = matches.filter((node) => { - if (!node.rect) return false; - const center = centerOfRect(node.rect); - return ( - center.x >= viewport.x && - center.x <= viewport.x + viewport.width && - center.y >= viewport.y && - center.y <= viewport.y + viewport.height - ); - }); - return rankInteractiveMatches(onscreen.length > 0 ? onscreen : matches, nodes); -} - -function rankInteractiveMatches( - matches: SnapshotState['nodes'], - nodes: SnapshotState['nodes'], -): SnapshotState['nodes'] { - if (matches.length < 2) return matches; - return matches - .map((node, index) => ({ node, index, score: interactiveMatchScore(node, nodes) })) - .sort((left, right) => { - if (right.score !== left.score) return right.score - left.score; - return rectArea(left.node) - rectArea(right.node) || left.index - right.index; - }) - .map((entry) => entry.node); -} - -function interactiveMatchScore( - node: SnapshotState['nodes'][number], - nodes: SnapshotState['nodes'], -): number { - const resolution = resolveActionableTouchResolution(nodes, node); - if (resolution.reason === 'covered') return 0; - const resolved = resolvedTouchScore(resolution, nodes[0]); - if (resolved > 0) return resolved; - if (node.hittable && node.rect && !isRootInteractionContainer(node, nodes[0])) return 3; - return node.rect ? 1 : 0; -} - -function resolvedTouchScore( - resolution: ReturnType, - root: SnapshotState['nodes'][number] | undefined, -): number { - if (!resolution.node.rect) return 0; - if (resolution.reason === 'semantic-target' || resolution.reason === 'same-rect-descendant') { - return 4; - } - if ( - resolution.reason === 'hittable-ancestor' && - !isRootInteractionContainer(resolution.node, root) - ) { - return 2; - } - return 0; -} - -function rectArea(node: SnapshotState['nodes'][number]): number { - return node.rect ? node.rect.width * node.rect.height : Number.POSITIVE_INFINITY; -} // #1597: an agent reading an ambiguous-match error must be able to act on the // right @ref immediately, without a follow-up snapshot round trip. Candidate // lines reuse the exact snapshot-line renderer (`formatSnapshotLine`) so a diff --git a/test/integration/ios-simulator-e2e/snapshot-backend-conformance.ts b/test/integration/ios-simulator-e2e/snapshot-backend-conformance.ts index 988505672..57234dc68 100644 --- a/test/integration/ios-simulator-e2e/snapshot-backend-conformance.ts +++ b/test/integration/ios-simulator-e2e/snapshot-backend-conformance.ts @@ -12,7 +12,7 @@ import { type SnapshotPreferredBackend, } from '@agent-device/kernel/snapshot'; import { SNAPSHOT_BACKEND_CAPABILITIES } from '../../../src/snapshot-quality/backend-capabilities.ts'; -import { isSemanticTouchTarget } from '../../../src/core/interaction-targeting.ts'; +import { isSemanticTouchTarget } from '../../../src/core/touch-semantics.ts'; export type SnapshotBackendConformanceFixture = { screen: string;