From 65d0513a1a143caf57beb7bedcde6d5e05cc1a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 22 Aug 2026 13:46:15 +0200 Subject: [PATCH 1/3] refactor(find): isolate match ranking policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move find's candidate ordering — on-screen preference, actionability scoring, and the area/input-order tie-break — out of find-match-resolution.ts into a focused sibling. Pure move: no score input, ranked order, ambiguity refusal, or --first/--last behavior changes. The extraction establishes the seam the follow-up indexing change needs, so the ranking pass has one production entry point to build a topology in. Refs #1690 --- src/daemon/handlers/find-match-ranking.ts | 78 ++++++++++++++++++++ src/daemon/handlers/find-match-resolution.ts | 73 +----------------- 2 files changed, 81 insertions(+), 70 deletions(-) create mode 100644 src/daemon/handlers/find-match-ranking.ts diff --git a/src/daemon/handlers/find-match-ranking.ts b/src/daemon/handlers/find-match-ranking.ts new file mode 100644 index 000000000..c1576e06d --- /dev/null +++ b/src/daemon/handlers/find-match-ranking.ts @@ -0,0 +1,78 @@ +import { centerOfRect, type SnapshotState } from '@agent-device/kernel/snapshot'; +import { + isRootInteractionContainer, + resolveActionableTouchResolution, +} from '../../core/interaction-targeting.ts'; + +/** + * How `find` orders the candidates its locator matched, before the ambiguity + * contract in `find-match-resolution.ts` either refuses them or narrows with + * `--first`/`--last`. Ordering is a separate question from matching: it asks + * which candidate an agent most plausibly meant to touch, using the same + * actionability policy the interaction itself will run. + */ +export function preferOnscreenMatches( + matches: SnapshotState['nodes'], + nodes: SnapshotState['nodes'], +): SnapshotState['nodes'] { + const viewport = nodes[0]?.rect; + if (!viewport) return matches; + const onscreen = matches.filter((node) => { + if (!node.rect) return false; + const center = centerOfRect(node.rect); + return ( + center.x >= viewport.x && + center.x <= viewport.x + viewport.width && + center.y >= viewport.y && + center.y <= viewport.y + viewport.height + ); + }); + return rankInteractiveMatches(onscreen.length > 0 ? onscreen : matches, nodes); +} + +function rankInteractiveMatches( + matches: SnapshotState['nodes'], + nodes: SnapshotState['nodes'], +): SnapshotState['nodes'] { + if (matches.length < 2) return matches; + return matches + .map((node, index) => ({ node, index, score: interactiveMatchScore(node, nodes) })) + .sort((left, right) => { + if (right.score !== left.score) return right.score - left.score; + return rectArea(left.node) - rectArea(right.node) || left.index - right.index; + }) + .map((entry) => entry.node); +} + +function interactiveMatchScore( + node: SnapshotState['nodes'][number], + nodes: SnapshotState['nodes'], +): number { + const resolution = resolveActionableTouchResolution(nodes, node); + if (resolution.reason === 'covered') return 0; + const resolved = resolvedTouchScore(resolution, nodes[0]); + if (resolved > 0) return resolved; + if (node.hittable && node.rect && !isRootInteractionContainer(node, nodes[0])) return 3; + return node.rect ? 1 : 0; +} + +function resolvedTouchScore( + resolution: ReturnType, + root: SnapshotState['nodes'][number] | undefined, +): number { + if (!resolution.node.rect) return 0; + if (resolution.reason === 'semantic-target' || resolution.reason === 'same-rect-descendant') { + return 4; + } + if ( + resolution.reason === 'hittable-ancestor' && + !isRootInteractionContainer(resolution.node, root) + ) { + return 2; + } + return 0; +} + +function rectArea(node: SnapshotState['nodes'][number]): number { + return node.rect ? node.rect.width * node.rect.height : Number.POSITIVE_INFINITY; +} diff --git a/src/daemon/handlers/find-match-resolution.ts b/src/daemon/handlers/find-match-resolution.ts index 0aa569f64..fa5912cab 100644 --- a/src/daemon/handlers/find-match-resolution.ts +++ b/src/daemon/handlers/find-match-resolution.ts @@ -5,11 +5,9 @@ import { } from '@agent-device/selectors'; import { listSelectorPipelineMatches } from '../../core/selector-pipeline.ts'; import { SELECTOR_PIPELINE_POLICIES } from '../../core/selector-pipeline-policy.ts'; -import { centerOfRect, type SnapshotState } from '@agent-device/kernel/snapshot'; -import { - isRootInteractionContainer, - resolveActionableTouchResolution, -} from '../../core/interaction-targeting.ts'; +import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import { isRootInteractionContainer } from '../../core/interaction-targeting.ts'; +import { preferOnscreenMatches } from './find-match-ranking.ts'; import { formatSnapshotLine } from '../../snapshot/snapshot-lines.ts'; import type { ElementMatchCandidateDetails } from '../../utils/error-candidates.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts'; @@ -91,71 +89,6 @@ function narrowMultipleMatches( return null; } -function preferOnscreenMatches( - matches: SnapshotState['nodes'], - nodes: SnapshotState['nodes'], -): SnapshotState['nodes'] { - const viewport = nodes[0]?.rect; - if (!viewport) return matches; - const onscreen = matches.filter((node) => { - if (!node.rect) return false; - const center = centerOfRect(node.rect); - return ( - center.x >= viewport.x && - center.x <= viewport.x + viewport.width && - center.y >= viewport.y && - center.y <= viewport.y + viewport.height - ); - }); - return rankInteractiveMatches(onscreen.length > 0 ? onscreen : matches, nodes); -} - -function rankInteractiveMatches( - matches: SnapshotState['nodes'], - nodes: SnapshotState['nodes'], -): SnapshotState['nodes'] { - if (matches.length < 2) return matches; - return matches - .map((node, index) => ({ node, index, score: interactiveMatchScore(node, nodes) })) - .sort((left, right) => { - if (right.score !== left.score) return right.score - left.score; - return rectArea(left.node) - rectArea(right.node) || left.index - right.index; - }) - .map((entry) => entry.node); -} - -function interactiveMatchScore( - node: SnapshotState['nodes'][number], - nodes: SnapshotState['nodes'], -): number { - const resolution = resolveActionableTouchResolution(nodes, node); - if (resolution.reason === 'covered') return 0; - const resolved = resolvedTouchScore(resolution, nodes[0]); - if (resolved > 0) return resolved; - if (node.hittable && node.rect && !isRootInteractionContainer(node, nodes[0])) return 3; - return node.rect ? 1 : 0; -} - -function resolvedTouchScore( - resolution: ReturnType, - root: SnapshotState['nodes'][number] | undefined, -): number { - if (!resolution.node.rect) return 0; - if (resolution.reason === 'semantic-target' || resolution.reason === 'same-rect-descendant') { - return 4; - } - if ( - resolution.reason === 'hittable-ancestor' && - !isRootInteractionContainer(resolution.node, root) - ) { - return 2; - } - return 0; -} - -function rectArea(node: SnapshotState['nodes'][number]): number { - return node.rect ? node.rect.width * node.rect.height : Number.POSITIVE_INFINITY; -} // #1597: an agent reading an ambiguous-match error must be able to act on the // right @ref immediately, without a follow-up snapshot round trip. Candidate // lines reuse the exact snapshot-line renderer (`formatSnapshotLine`) so a From aa12f07d81cdb244a9f79bd3e9ca8b9997180074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 22 Aug 2026 13:52:57 +0200 Subject: [PATCH 2/3] perf(find): index snapshot topology once per match ranking pass Ranking a mutating find's candidates asked the whole tree the same three questions once per candidate: same-rect descendants filtered every node, the nearest hittable ancestor rebuilt a full index map, and the overly-broad-ancestor check re-filtered every node for viewport roots. With m matches over an n-node capture that is O(m x n) full-tree work before find can act or refuse. buildActionableTouchTopology reads those three collections in one pass (nodes by index, children by parent index, normalized viewport-root rects). preferOnscreenMatches builds exactly one per multi-match pass and threads it through every score; resolveActionableTouchResolution takes it as an optional argument so one-off interaction callers keep the cheap two-argument shape. findNearestAncestor gained the same optional prebuilt map its snapshot-presentation sibling already accepted, and classifyActionableTouchCandidates now reuses one topology instead of building a bare index map and re-resolving per candidate. Score inputs, ranked order, area and input-order tie-breaks, ambiguity refusal, and --first/--last are unchanged; only the derivation is shared. Observed red first: with the wiring hunk removed, the new ranking regression reports builder calls 0 (expected 1) and 64 filter + 32 map whole-tree scans (expected 0) over 32 candidates. The topology docstring records two seams the reviewer asked for. #1690 names src/snapshot/snapshot-processing.ts as findNearestAncestor's home; that path is gone and packages/contracts/src/snapshot-tree.ts is the seam that replaced it, so the issue's file list is drifted rather than a second site to change. And viewportRootRects is not interchangeable with snapshot-visibility's precomputedViewportRects: normalizeRect drops negative width/height where hasValidRect keeps them, which changes which rect wins pickLargestRect. Refs #1690 --- packages/contracts/src/snapshot-tree.ts | 13 +- packages/contracts/src/snapshot.test.ts | 9 +- src/core/actionable-touch-topology.test.ts | 102 +++++++++ src/core/actionable-touch-topology.ts | 65 ++++++ src/core/interaction-targeting.fixtures.ts | 107 +++++++++ src/core/interaction-targeting.test.ts | 34 +++ src/core/interaction-targeting.ts | 73 ++++-- .../__tests__/find-match-ranking.test.ts | 211 ++++++++++++++++++ src/daemon/handlers/find-match-ranking.ts | 18 +- 9 files changed, 602 insertions(+), 30 deletions(-) create mode 100644 src/core/actionable-touch-topology.test.ts create mode 100644 src/core/actionable-touch-topology.ts create mode 100644 src/daemon/handlers/__tests__/find-match-ranking.test.ts diff --git a/packages/contracts/src/snapshot-tree.ts b/packages/contracts/src/snapshot-tree.ts index dfb6233ea..8677787ef 100644 --- a/packages/contracts/src/snapshot-tree.ts +++ b/packages/contracts/src/snapshot-tree.ts @@ -33,13 +33,22 @@ export function findSnapshotAncestor( /** * Returns the nearest ancestor matching `predicate`; false means keep walking. + * + * Pass `nodeByIndex` when the caller already holds an index over this exact + * node array — a batch asking about many nodes otherwise rebuilds the whole map + * per lookup. Omitting it keeps the one-off call shape, which is what an + * ordinary single-target interaction wants. */ export function findNearestAncestor( nodes: SnapshotNode[], node: SnapshotNode, predicate: (ancestor: SnapshotNode) => boolean, + nodeByIndex?: ReadonlyMap, ): SnapshotNode | null { - return findSnapshotAncestor(nodes, node, buildSnapshotNodeMap(nodes), (ancestor) => - predicate(ancestor) ? ancestor : null, + return findSnapshotAncestor( + nodes, + node, + nodeByIndex ?? buildSnapshotNodeMap(nodes), + (ancestor) => (predicate(ancestor) ? ancestor : null), ); } diff --git a/packages/contracts/src/snapshot.test.ts b/packages/contracts/src/snapshot.test.ts index 485f5d350..0b5350cb4 100644 --- a/packages/contracts/src/snapshot.test.ts +++ b/packages/contracts/src/snapshot.test.ts @@ -74,10 +74,17 @@ test('findNearestAncestor adapts a predicate to the shared tree walk', () => { { ref: 'e20', index: 20, parentIndex: 10, type: 'Cell' }, ]; + const isWindow = (ancestor: SnapshotNode) => ancestor.type === 'Window'; + + assert.equal(findNearestAncestor(nodes, nodes[1]!, isWindow)?.index, 10); + // A caller that already holds an index over this array hands it in instead of + // paying for a rebuilt map per lookup; an index that omits the chain proves + // the supplied map is the one walked, not a quietly rebuilt one. assert.equal( - findNearestAncestor(nodes, nodes[1]!, (ancestor) => ancestor.type === 'Window')?.index, + findNearestAncestor(nodes, nodes[1]!, isWindow, buildSnapshotNodeMap(nodes))?.index, 10, ); + assert.equal(findNearestAncestor(nodes, nodes[1]!, isWindow, new Map()), null); }); test('snapshot tree and scroll semantics identify nodes through their stable indexes', () => { diff --git a/src/core/actionable-touch-topology.test.ts b/src/core/actionable-touch-topology.test.ts new file mode 100644 index 000000000..23c41ed5b --- /dev/null +++ b/src/core/actionable-touch-topology.test.ts @@ -0,0 +1,102 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { makeSnapshotState } from '../__tests__/test-utils/snapshot-builders.ts'; +import { buildActionableTouchTopology } from './actionable-touch-topology.ts'; + +/** + * Non-contiguous indexes, two parents, and every root vocabulary the canonical + * predicate recognizes — the three questions a ranking pass asks the tree. + */ +const MIXED_TOPOLOGY_NODES = [ + { + index: 10, + depth: 0, + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: true, + }, + { + index: 20, + depth: 1, + parentIndex: 10, + type: 'AXUnknown', + role: 'AXWindow', + rect: { x: 0, y: 0, width: 390, height: 400 }, + hittable: true, + }, + { + index: 30, + depth: 1, + parentIndex: 10, + type: 'XCUIElementTypeOther', + subrole: 'AXFloatingWindow', + rect: { x: 10, y: 10, width: 100, height: 100 }, + hittable: false, + }, + { + index: 40, + depth: 2, + parentIndex: 20, + type: 'XCUIElementTypeButton', + label: 'Save', + rect: { x: 20, y: 20, width: 80, height: 30 }, + hittable: true, + }, + { + index: 50, + depth: 2, + parentIndex: 20, + type: 'XCUIElementTypeStaticText', + label: 'Saved', + rect: { x: 20, y: 60, width: 80, height: 20 }, + hittable: false, + }, + // A viewport root with an unusable rect: the same drop the per-candidate + // filter/map chain performed, so an indexed pass cannot start measuring + // against NaN geometry. + { + index: 60, + depth: 1, + parentIndex: 10, + type: 'XCUIElementTypeWindow', + rect: { x: Number.NaN, y: 0, width: 390, height: 844 }, + hittable: true, + }, +]; + +test('indexes every node by its snapshot index, not its array position', () => { + const snapshot = makeSnapshotState(MIXED_TOPOLOGY_NODES); + + const topology = buildActionableTouchTopology(snapshot.nodes); + + assert.deepEqual([...topology.nodesByIndex.keys()], [10, 20, 30, 40, 50, 60]); + assert.strictEqual(topology.nodesByIndex.get(40), snapshot.nodes[3]); +}); + +test('groups children under each parent index in input order', () => { + const snapshot = makeSnapshotState(MIXED_TOPOLOGY_NODES); + + const topology = buildActionableTouchTopology(snapshot.nodes); + + assert.deepEqual( + topology.childrenByParentIndex.get(10)?.map((node) => node.index), + [20, 30, 60], + ); + assert.deepEqual( + topology.childrenByParentIndex.get(20)?.map((node) => node.index), + [40, 50], + ); + assert.equal(topology.childrenByParentIndex.get(40), undefined); +}); + +test('collects canonical viewport-root rects from type, role, and subrole', () => { + const snapshot = makeSnapshotState(MIXED_TOPOLOGY_NODES); + + const topology = buildActionableTouchTopology(snapshot.nodes); + + assert.deepEqual(topology.viewportRootRects, [ + { x: 0, y: 0, width: 390, height: 844 }, + { x: 0, y: 0, width: 390, height: 400 }, + { x: 10, y: 10, width: 100, height: 100 }, + ]); +}); diff --git a/src/core/actionable-touch-topology.ts b/src/core/actionable-touch-topology.ts new file mode 100644 index 000000000..151679493 --- /dev/null +++ b/src/core/actionable-touch-topology.ts @@ -0,0 +1,65 @@ +import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; +import { isViewportRootNode } from '@agent-device/contracts/snapshot'; +import { normalizeRect } from '../utils/rect-center.ts'; + +/** + * The three whole-tree lookups actionability resolution performs per node, read + * once instead of rebuilt per node: parents by index, children by parent, and + * the viewport rects an overly-broad ancestor is measured against. + * + * It exists because ranking asks the SAME policy about many candidates. One + * `find click` with `m` matches over an `n`-node capture used to walk the + * whole tree three times per candidate before it could refuse or act; with this + * it walks it once for the pass. + * + * Read-only and scoped to one pass over one node array. A topology that outlived + * its capture would answer for a screen that has already moved, so nothing caches + * it on a session — build it from the exact array being ranked and drop it with + * that pass. + * + * `nodesByIndex` is handed to `findNearestAncestor` in + * `@agent-device/contracts/snapshot-tree`. #1690 names + * `src/snapshot/snapshot-processing.ts` as that function's home; the path no + * longer exists and the contracts module is the seam that replaced it, so read + * the issue's file list as drifted, not as a second place to change. + */ +export type ActionableTouchTopology = { + /** Parent resolution by `node.index` — snapshot identity, not array position. */ + readonly nodesByIndex: ReadonlyMap; + /** Children of each parent index, in the input array's order. */ + readonly childrenByParentIndex: ReadonlyMap; + /** + * Normalized rects of the canonical viewport roots, in the input array's order. + * NOT interchangeable with `snapshot-visibility`'s `precomputedViewportRects`: + * `normalizeRect` drops negative width/height, `hasValidRect` there keeps them, + * so substituting one for the other changes which rect wins `pickLargestRect`. + */ + readonly viewportRootRects: readonly Rect[]; +}; + +/** + * One pass, three collections. Deliberately written with `for...of` rather than + * `filter`/`map`: the ranking regression counts whole-array scans on the node + * array, and the point of this builder is that a ranking pass performs exactly + * one of them. + */ +export function buildActionableTouchTopology( + nodes: readonly SnapshotNode[], +): ActionableTouchTopology { + const nodesByIndex = new Map(); + const childrenByParentIndex = new Map(); + const viewportRootRects: Rect[] = []; + for (const node of nodes) { + nodesByIndex.set(node.index, node); + if (typeof node.parentIndex === 'number') { + const siblings = childrenByParentIndex.get(node.parentIndex); + if (siblings) siblings.push(node); + else childrenByParentIndex.set(node.parentIndex, [node]); + } + if (isViewportRootNode(node)) { + const rect = normalizeRect(node.rect); + if (rect) viewportRootRects.push(rect); + } + } + return { nodesByIndex, childrenByParentIndex, viewportRootRects }; +} diff --git a/src/core/interaction-targeting.fixtures.ts b/src/core/interaction-targeting.fixtures.ts index 87ffc8e00..1f88de3d2 100644 --- a/src/core/interaction-targeting.fixtures.ts +++ b/src/core/interaction-targeting.fixtures.ts @@ -74,3 +74,110 @@ export const ELEMENT14_DISTINCT_SUBTREE_NODES: RawSnapshotNode[] = [ hittable: true, }, ]; + +/** + * One node per decision `resolveActionableTouchResolution` can reach, so an + * indexed pass and an unindexed one can be compared across the whole policy + * rather than on the branch a single example happens to hit: a same-rect + * actionable descendant (1 -> 2), semantic targets (3, 4), a nonhittable leaf + * under a hittable ancestor (5), both overly-broad shapes — a scrolling + * container (7) and the viewport-sized root (10) — a covered node (8), and a + * parentless, rectless node with no usable target at all (9). + */ +export const INDEXED_PARITY_POLICY_NODES: RawSnapshotNode[] = [ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeOther', + label: 'Save wrapper', + rect: { x: 20, y: 100, width: 120, height: 44 }, + hittable: false, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'XCUIElementTypeImage', + identifier: 'save-hit-area', + rect: { x: 20, y: 100, width: 120, height: 44 }, + hittable: true, + }, + { + index: 3, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Save', + rect: { x: 20, y: 200, width: 100, height: 40 }, + hittable: false, + }, + { + index: 4, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeCell', + label: 'Account row', + rect: { x: 10, y: 260, width: 370, height: 60 }, + hittable: true, + }, + { + index: 5, + depth: 2, + parentIndex: 4, + type: 'XCUIElementTypeStaticText', + label: 'Account', + rect: { x: 24, y: 272, width: 80, height: 20 }, + hittable: false, + }, + { + index: 6, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeScrollView', + rect: { x: 0, y: 340, width: 390, height: 300 }, + hittable: true, + }, + { + index: 7, + depth: 2, + parentIndex: 6, + type: 'XCUIElementTypeOther', + label: 'Feed item', + rect: { x: 20, y: 360, width: 200, height: 40 }, + hittable: false, + }, + { + index: 8, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Under overlay', + rect: { x: 20, y: 700, width: 100, height: 20 }, + hittable: false, + interactionBlocked: 'covered', + }, + { + index: 9, + depth: 0, + type: 'XCUIElementTypeOther', + label: 'Virtual item', + hittable: false, + }, + { + index: 10, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Status', + rect: { x: 20, y: 760, width: 60, height: 20 }, + hittable: false, + }, +]; diff --git a/src/core/interaction-targeting.test.ts b/src/core/interaction-targeting.test.ts index 250e1aa06..a0c369fd8 100644 --- a/src/core/interaction-targeting.test.ts +++ b/src/core/interaction-targeting.test.ts @@ -7,6 +7,7 @@ import { scrollingContainerTypeArb, } from '../__tests__/test-utils/property-arbitraries.ts'; import { makeSnapshotState } from '../__tests__/test-utils/snapshot-builders.ts'; +import { buildActionableTouchTopology } from './actionable-touch-topology.ts'; import { classifyActionableTouchCandidates, resolveActionableTouchResolution, @@ -14,6 +15,7 @@ import { import { ELEMENT14_DISTINCT_SUBTREE_NODES, EQUIVALENT_WRAPPER_CHAIN_NODES, + INDEXED_PARITY_POLICY_NODES, } from './interaction-targeting.fixtures.ts'; test('collapses one same-label wrapper chain to its shared actionable node', () => { @@ -201,3 +203,35 @@ test('falls back to the original node when no usable touch target exists', () => assert.equal(resolution.reason, 'original'); assert.equal(resolution.node.label, 'Virtual item'); }); + +test('a prebuilt topology answers every policy branch exactly as the unindexed walk does', () => { + const snapshot = makeSnapshotState(INDEXED_PARITY_POLICY_NODES); + const topology = buildActionableTouchTopology(snapshot.nodes); + + const unindexed = snapshot.nodes.map((node) => + resolveActionableTouchResolution(snapshot.nodes, node), + ); + const indexed = snapshot.nodes.map((node) => + resolveActionableTouchResolution(snapshot.nodes, node, topology), + ); + + assert.deepEqual(indexed, unindexed); + // Pinned rather than merely equal: two identically broken implementations + // would also be deeply equal to each other. + assert.deepEqual( + indexed.map((resolution) => [resolution.node.index, resolution.reason]), + [ + [0, 'hittable-ancestor'], + [2, 'same-rect-descendant'], + [2, 'hittable-ancestor'], + [3, 'semantic-target'], + [4, 'semantic-target'], + [4, 'hittable-ancestor'], + [6, 'hittable-ancestor'], + [7, 'overly-broad-ancestor'], + [8, 'covered'], + [9, 'original'], + [10, 'overly-broad-ancestor'], + ], + ); +}); diff --git a/src/core/interaction-targeting.ts b/src/core/interaction-targeting.ts index 364a17360..dce1643e8 100644 --- a/src/core/interaction-targeting.ts +++ b/src/core/interaction-targeting.ts @@ -6,6 +6,10 @@ import { normalizeType, isViewportRootNode, } from '@agent-device/contracts/snapshot'; +import { + buildActionableTouchTopology, + type ActionableTouchTopology, +} from './actionable-touch-topology.ts'; import { isSnapshotNodeInteractionBlocked } from '../snapshot/snapshot-occlusion.ts'; import { areRectsApproximatelyEqual, @@ -57,13 +61,17 @@ export function classifyActionableTouchCandidates( ): ActionableTouchCandidateClassification { const first = candidates[0]; if (!first) return { kind: 'ambiguous', candidates }; - const byIndex = new Map(nodes.map((node) => [node.index, node])); - if (!candidatesFormSingleAncestryChain(candidates, byIndex)) { + // One index for the whole classification: the ancestry chain and every + // candidate's resolution ask the same tree the same three questions. + const topology = buildActionableTouchTopology(nodes); + if (!candidatesFormSingleAncestryChain(candidates, topology.nodesByIndex)) { return { kind: 'ambiguous', candidates }; } - const actionable = resolveActionableTouchResolution(nodes, first).node; + const actionable = resolveActionableTouchResolution(nodes, first, topology).node; for (const candidate of candidates.slice(1)) { - if (resolveActionableTouchResolution(nodes, candidate).node.index !== actionable.index) { + if ( + resolveActionableTouchResolution(nodes, candidate, topology).node.index !== actionable.index + ) { return { kind: 'ambiguous', candidates }; } } @@ -124,28 +132,36 @@ export function isRootInteractionContainer( ); } -/** @internal Exposed for focused policy tests. */ +/** + * @internal Exposed for focused policy tests. + * + * `topology` is an optional prebuilt index over THIS `nodes` array. It changes + * only how the three whole-tree lookups below are answered, never which node or + * reason comes back — a caller resolving one target may keep omitting it rather + * than pay for an index it uses once. + */ export function resolveActionableTouchResolution( nodes: SnapshotNode[], node: SnapshotNode, + topology?: ActionableTouchTopology, ): ActionableTouchResolution { if (isSnapshotNodeInteractionBlocked(node)) { return { node, reason: 'covered' }; } - const descendant = findPreferredActionableDescendant(nodes, node); + const descendant = findPreferredActionableDescendant(nodes, node, topology); if (descendant?.rect && resolveRectCenter(descendant.rect)) { return { node: descendant, reason: 'same-rect-descendant' }; } if (isSemanticTouchTarget(node) && node.rect && resolveRectCenter(node.rect)) { return { node, reason: 'semantic-target' }; } - const ancestor = findNearestHittableAncestor(nodes, node); + const ancestor = findNearestHittableAncestor(nodes, node, topology); if ( ancestor?.rect && !isSnapshotNodeInteractionBlocked(ancestor) && resolveRectCenter(ancestor.rect) ) { - if (isOverlyBroadAncestor(node, ancestor, nodes)) { + if (isOverlyBroadAncestor(node, ancestor, nodes, topology)) { return { node, reason: 'overly-broad-ancestor' }; } return { node: ancestor, reason: 'hittable-ancestor' }; @@ -156,14 +172,21 @@ export function resolveActionableTouchResolution( function findNearestHittableAncestor( nodes: SnapshotNode[], node: SnapshotNode, + topology: ActionableTouchTopology | undefined, ): SnapshotNode | null { if (node.hittable) return node; - return findNearestAncestor(nodes, node, (parent) => parent.hittable === true); + return findNearestAncestor( + nodes, + node, + (parent) => parent.hittable === true, + topology?.nodesByIndex, + ); } function findPreferredActionableDescendant( nodes: SnapshotNode[], node: SnapshotNode, + topology: ActionableTouchTopology | undefined, ): SnapshotNode | null { const targetRect = normalizeRect(node.rect); if (!targetRect) return null; @@ -172,14 +195,11 @@ function findPreferredActionableDescendant( const visited = new Set(); while (!visited.has(current.ref)) { visited.add(current.ref); - const sameRectChildren = nodes.filter((candidate) => { - if ( - candidate.parentIndex !== current.index || - !candidate.hittable || - isSnapshotNodeInteractionBlocked(candidate) - ) { - return false; - } + const children = topology + ? (topology.childrenByParentIndex.get(current.index) ?? []) + : nodes.filter((candidate) => candidate.parentIndex === current.index); + const sameRectChildren = children.filter((candidate) => { + if (!candidate.hittable || isSnapshotNodeInteractionBlocked(candidate)) return false; const candidateRect = normalizeRect(candidate.rect); return candidateRect ? areRectsApproximatelyEqual(candidateRect, targetRect) : false; }); @@ -215,6 +235,7 @@ function isOverlyBroadAncestor( node: SnapshotNode, ancestor: SnapshotNode, nodes: SnapshotNode[], + topology: ActionableTouchTopology | undefined, ): boolean { const nodeRect = normalizeRect(node.rect); const ancestorRect = normalizeRect(ancestor.rect); @@ -222,7 +243,7 @@ function isOverlyBroadAncestor( if (isScrollingContainer(ancestor) && !areRectsApproximatelyEqual(nodeRect, ancestorRect)) { return true; } - const rootViewportRect = resolveRootViewportRect(nodes, nodeRect); + const rootViewportRect = resolveRootViewportRect(nodes, nodeRect, topology); if (!rootViewportRect) return false; if (!isRectViewportSized(ancestorRect, rootViewportRect)) return false; return !areRectsApproximatelyEqual(nodeRect, ancestorRect); @@ -242,12 +263,18 @@ function isScrollingContainer(node: SnapshotNode): boolean { ); } -function resolveRootViewportRect(nodes: SnapshotNode[], targetRect: Rect): Rect | null { +function resolveRootViewportRect( + nodes: SnapshotNode[], + targetRect: Rect, + topology: ActionableTouchTopology | undefined, +): Rect | null { const targetCenter = centerOfRect(targetRect); - const viewportRects = nodes - .filter(isViewportRootNode) - .map((node) => normalizeRect(node.rect)) - .filter((rect): rect is Rect => rect !== null); + const viewportRects = + topology?.viewportRootRects ?? + nodes + .filter(isViewportRootNode) + .map((node) => normalizeRect(node.rect)) + .filter((rect): rect is Rect => rect !== null); if (viewportRects.length === 0) return null; const containingRects = viewportRects.filter((rect) => diff --git a/src/daemon/handlers/__tests__/find-match-ranking.test.ts b/src/daemon/handlers/__tests__/find-match-ranking.test.ts new file mode 100644 index 000000000..da8f48fa8 --- /dev/null +++ b/src/daemon/handlers/__tests__/find-match-ranking.test.ts @@ -0,0 +1,211 @@ +import assert from 'node:assert/strict'; +import { beforeEach, test, vi } from 'vitest'; +import type { RawSnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; +import { makeSnapshotState } from '../../../__tests__/test-utils/snapshot-builders.ts'; +import { buildActionableTouchTopology } from '../../../core/actionable-touch-topology.ts'; +import { preferOnscreenMatches } from '../find-match-ranking.ts'; + +/** + * The builder stays REAL — `vi.fn` only wraps it, so ranking consumes production + * topology data and the wrapper reports how many times the pass built one. The + * alternative (a test double) would prove the call happened and nothing about + * what ranking then read. + */ +vi.mock('../../../core/actionable-touch-topology.ts', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + buildActionableTouchTopology: vi.fn(actual.buildActionableTouchTopology), + }; +}); + +const buildTopology = vi.mocked(buildActionableTouchTopology); + +beforeEach(() => { + buildTopology.mockClear(); +}); + +const VIEWPORT = { x: 0, y: 0, width: 390, height: 844 }; +const DUPLICATE_MATCH_COUNT = 32; + +/** + * Counts whole-array `filter`/`map` calls on the snapshot tree. Every function + * is bound to the target so a counted scan cannot re-enter the proxy and + * inflate its own count; `matches` is a different array, so ranking's own + * legitimate on-screen filter is never mistaken for a full-tree scan. + */ +function observeWholeTreeScans(nodes: SnapshotState['nodes']): { + observed: SnapshotState['nodes']; + scans: { filter: number; map: number }; +} { + const scans = { filter: 0, map: 0 }; + const observed = new Proxy(nodes, { + get(target, property) { + if (property === 'filter' || property === 'map') scans[property] += 1; + const value = Reflect.get(target, property) as unknown; + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + return { observed, scans }; +} + +/** + * Duplicate-heavy and deliberately worst-case: every match is nonsemantic, + * nonhittable and childless, so the old implementation had to run all three + * whole-tree lookups — same-rect descendants, the nearest hittable ancestor's + * index map, and the viewport roots an overly-broad ancestor is measured + * against — once per candidate. Widths shrink as the list goes on so the + * area tie-break has to reverse the input order. + */ +function duplicateHeavyCapture(): SnapshotState { + const raw: RawSnapshotNode[] = [ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeApplication', + rect: VIEWPORT, + hittable: true, + }, + ]; + for (let position = 0; position < DUPLICATE_MATCH_COUNT; position += 1) { + raw.push({ + index: position + 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeOther', + label: 'Item', + rect: { + x: 20, + y: 40 + position * 20, + width: 100 + (DUPLICATE_MATCH_COUNT - position), + height: 16, + }, + hittable: false, + }); + } + return makeSnapshotState(raw); +} + +/** + * One node per scoring branch the ranking rules distinguish: a semantic target, + * a same-rect actionable descendant, a self-hittable node below the root, a + * node whose only ancestor is the viewport-sized root, and a second semantic + * target with the first one's exact area. + */ +const MIXED_SCORE_NODES: RawSnapshotNode[] = [ + { index: 0, depth: 0, type: 'XCUIElementTypeApplication', rect: VIEWPORT, hittable: true }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Sync', + rect: { x: 20, y: 100, width: 100, height: 40 }, + hittable: false, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeOther', + label: 'Sync', + rect: { x: 20, y: 200, width: 120, height: 40 }, + hittable: false, + }, + { + index: 3, + depth: 2, + parentIndex: 2, + type: 'XCUIElementTypeImage', + identifier: 'sync-hit-area', + rect: { x: 20, y: 200, width: 120, height: 40 }, + hittable: true, + }, + { + index: 4, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeOther', + label: 'Sync', + rect: { x: 20, y: 300, width: 80, height: 30 }, + hittable: true, + }, + { + index: 5, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Sync', + rect: { x: 20, y: 400, width: 60, height: 20 }, + hittable: false, + }, + { + index: 6, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Sync', + rect: { x: 200, y: 100, width: 100, height: 40 }, + hittable: false, + }, +]; + +test('a multi-match ranking pass indexes the tree once and never rescans it', () => { + const snapshot = duplicateHeavyCapture(); + const matches = snapshot.nodes.slice(1); + const { observed, scans } = observeWholeTreeScans(snapshot.nodes); + + const ranked = preferOnscreenMatches(matches, observed); + + assert.equal(buildTopology.mock.calls.length, 1); + assert.deepEqual(scans, { filter: 0, map: 0 }); + assert.deepEqual( + ranked.map((node) => node.ref), + [...matches].reverse().map((node) => node.ref), + ); +}); + +test('a single match returns without indexing the tree', () => { + const snapshot = duplicateHeavyCapture(); + const { observed, scans } = observeWholeTreeScans(snapshot.nodes); + + const ranked = preferOnscreenMatches([snapshot.nodes[1]!], observed); + + assert.equal(buildTopology.mock.calls.length, 0); + assert.deepEqual(scans, { filter: 0, map: 0 }); + assert.deepEqual( + ranked.map((node) => node.ref), + [snapshot.nodes[1]!.ref], + ); +}); + +test('a capture without a root rect returns matches unranked and unindexed', () => { + const snapshot = makeSnapshotState([ + { index: 0, depth: 0, type: 'XCUIElementTypeApplication', hittable: true }, + ...MIXED_SCORE_NODES.slice(1), + ]); + const matches = snapshot.nodes.slice(1); + + const ranked = preferOnscreenMatches(matches, snapshot.nodes); + + assert.equal(buildTopology.mock.calls.length, 0); + assert.deepEqual( + ranked.map((node) => node.ref), + matches.map((node) => node.ref), + ); +}); + +test('ranks by actionability score, then smallest rect, then input order', () => { + const snapshot = makeSnapshotState(MIXED_SCORE_NODES); + // Deliberately out of tree order so score, area, and input position each have + // to decide a pair: [static text, second button, first button, wrapper, self-hittable]. + const matches = [5, 6, 1, 2, 4].map((index) => snapshot.nodes[index]!); + + const ranked = preferOnscreenMatches(matches, snapshot.nodes); + + assert.deepEqual( + ranked.map((node) => node.ref), + ['e7', 'e2', 'e3', 'e5', 'e6'], + ); +}); diff --git a/src/daemon/handlers/find-match-ranking.ts b/src/daemon/handlers/find-match-ranking.ts index c1576e06d..4c322382e 100644 --- a/src/daemon/handlers/find-match-ranking.ts +++ b/src/daemon/handlers/find-match-ranking.ts @@ -1,4 +1,8 @@ import { centerOfRect, type SnapshotState } from '@agent-device/kernel/snapshot'; +import { + buildActionableTouchTopology, + type ActionableTouchTopology, +} from '../../core/actionable-touch-topology.ts'; import { isRootInteractionContainer, resolveActionableTouchResolution, @@ -27,16 +31,21 @@ export function preferOnscreenMatches( center.y <= viewport.y + viewport.height ); }); - return rankInteractiveMatches(onscreen.length > 0 ? onscreen : matches, nodes); + const preferred = onscreen.length > 0 ? onscreen : matches; + // The only topology build site for the pass, and only once there is really + // something to order: one candidate answers no comparative question, so it + // must not pay for an index over the whole capture. + if (preferred.length < 2) return preferred; + return rankInteractiveMatches(preferred, nodes, buildActionableTouchTopology(nodes)); } function rankInteractiveMatches( matches: SnapshotState['nodes'], nodes: SnapshotState['nodes'], + topology: ActionableTouchTopology, ): SnapshotState['nodes'] { - if (matches.length < 2) return matches; return matches - .map((node, index) => ({ node, index, score: interactiveMatchScore(node, nodes) })) + .map((node, index) => ({ node, index, score: interactiveMatchScore(node, nodes, topology) })) .sort((left, right) => { if (right.score !== left.score) return right.score - left.score; return rectArea(left.node) - rectArea(right.node) || left.index - right.index; @@ -47,8 +56,9 @@ function rankInteractiveMatches( function interactiveMatchScore( node: SnapshotState['nodes'][number], nodes: SnapshotState['nodes'], + topology: ActionableTouchTopology, ): number { - const resolution = resolveActionableTouchResolution(nodes, node); + const resolution = resolveActionableTouchResolution(nodes, node, topology); if (resolution.reason === 'covered') return 0; const resolved = resolvedTouchScore(resolution, nodes[0]); if (resolved > 0) return resolved; From 18220fb811b5072f0489ec807d1be8ddbb426de2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 23 Aug 2026 08:01:59 +0200 Subject: [PATCH 3/3] refactor: hide actionable touch indexing --- packages/contracts/src/snapshot-tree.ts | 17 +- packages/contracts/src/snapshot.test.ts | 8 - src/core/actionable-touch-topology.test.ts | 102 ---------- src/core/actionable-touch-topology.ts | 65 ------ src/core/interaction-targeting.test.ts | 12 +- src/core/interaction-targeting.ts | 191 +++++++++--------- src/core/interaction-touch-point.ts | 2 +- src/core/press-retarget.test.ts | 2 +- src/core/press-retarget.ts | 2 +- src/core/touch-semantics.ts | 23 +++ .../__tests__/find-match-ranking.test.ts | 48 +---- src/daemon/handlers/find-match-ranking.ts | 22 +- .../snapshot-backend-conformance.ts | 2 +- 13 files changed, 149 insertions(+), 347 deletions(-) delete mode 100644 src/core/actionable-touch-topology.test.ts delete mode 100644 src/core/actionable-touch-topology.ts create mode 100644 src/core/touch-semantics.ts diff --git a/packages/contracts/src/snapshot-tree.ts b/packages/contracts/src/snapshot-tree.ts index 8677787ef..e98345976 100644 --- a/packages/contracts/src/snapshot-tree.ts +++ b/packages/contracts/src/snapshot-tree.ts @@ -31,24 +31,13 @@ export function findSnapshotAncestor( return null; } -/** - * Returns the nearest ancestor matching `predicate`; false means keep walking. - * - * Pass `nodeByIndex` when the caller already holds an index over this exact - * node array — a batch asking about many nodes otherwise rebuilds the whole map - * per lookup. Omitting it keeps the one-off call shape, which is what an - * ordinary single-target interaction wants. - */ +/** Returns the nearest ancestor matching `predicate`; false means keep walking. */ export function findNearestAncestor( nodes: SnapshotNode[], node: SnapshotNode, predicate: (ancestor: SnapshotNode) => boolean, - nodeByIndex?: ReadonlyMap, ): SnapshotNode | null { - return findSnapshotAncestor( - nodes, - node, - nodeByIndex ?? buildSnapshotNodeMap(nodes), - (ancestor) => (predicate(ancestor) ? ancestor : null), + return findSnapshotAncestor(nodes, node, buildSnapshotNodeMap(nodes), (ancestor) => + predicate(ancestor) ? ancestor : null, ); } diff --git a/packages/contracts/src/snapshot.test.ts b/packages/contracts/src/snapshot.test.ts index 0b5350cb4..672eb655f 100644 --- a/packages/contracts/src/snapshot.test.ts +++ b/packages/contracts/src/snapshot.test.ts @@ -77,14 +77,6 @@ test('findNearestAncestor adapts a predicate to the shared tree walk', () => { const isWindow = (ancestor: SnapshotNode) => ancestor.type === 'Window'; assert.equal(findNearestAncestor(nodes, nodes[1]!, isWindow)?.index, 10); - // A caller that already holds an index over this array hands it in instead of - // paying for a rebuilt map per lookup; an index that omits the chain proves - // the supplied map is the one walked, not a quietly rebuilt one. - assert.equal( - findNearestAncestor(nodes, nodes[1]!, isWindow, buildSnapshotNodeMap(nodes))?.index, - 10, - ); - assert.equal(findNearestAncestor(nodes, nodes[1]!, isWindow, new Map()), null); }); test('snapshot tree and scroll semantics identify nodes through their stable indexes', () => { diff --git a/src/core/actionable-touch-topology.test.ts b/src/core/actionable-touch-topology.test.ts deleted file mode 100644 index 23c41ed5b..000000000 --- a/src/core/actionable-touch-topology.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { makeSnapshotState } from '../__tests__/test-utils/snapshot-builders.ts'; -import { buildActionableTouchTopology } from './actionable-touch-topology.ts'; - -/** - * Non-contiguous indexes, two parents, and every root vocabulary the canonical - * predicate recognizes — the three questions a ranking pass asks the tree. - */ -const MIXED_TOPOLOGY_NODES = [ - { - index: 10, - depth: 0, - type: 'XCUIElementTypeApplication', - rect: { x: 0, y: 0, width: 390, height: 844 }, - hittable: true, - }, - { - index: 20, - depth: 1, - parentIndex: 10, - type: 'AXUnknown', - role: 'AXWindow', - rect: { x: 0, y: 0, width: 390, height: 400 }, - hittable: true, - }, - { - index: 30, - depth: 1, - parentIndex: 10, - type: 'XCUIElementTypeOther', - subrole: 'AXFloatingWindow', - rect: { x: 10, y: 10, width: 100, height: 100 }, - hittable: false, - }, - { - index: 40, - depth: 2, - parentIndex: 20, - type: 'XCUIElementTypeButton', - label: 'Save', - rect: { x: 20, y: 20, width: 80, height: 30 }, - hittable: true, - }, - { - index: 50, - depth: 2, - parentIndex: 20, - type: 'XCUIElementTypeStaticText', - label: 'Saved', - rect: { x: 20, y: 60, width: 80, height: 20 }, - hittable: false, - }, - // A viewport root with an unusable rect: the same drop the per-candidate - // filter/map chain performed, so an indexed pass cannot start measuring - // against NaN geometry. - { - index: 60, - depth: 1, - parentIndex: 10, - type: 'XCUIElementTypeWindow', - rect: { x: Number.NaN, y: 0, width: 390, height: 844 }, - hittable: true, - }, -]; - -test('indexes every node by its snapshot index, not its array position', () => { - const snapshot = makeSnapshotState(MIXED_TOPOLOGY_NODES); - - const topology = buildActionableTouchTopology(snapshot.nodes); - - assert.deepEqual([...topology.nodesByIndex.keys()], [10, 20, 30, 40, 50, 60]); - assert.strictEqual(topology.nodesByIndex.get(40), snapshot.nodes[3]); -}); - -test('groups children under each parent index in input order', () => { - const snapshot = makeSnapshotState(MIXED_TOPOLOGY_NODES); - - const topology = buildActionableTouchTopology(snapshot.nodes); - - assert.deepEqual( - topology.childrenByParentIndex.get(10)?.map((node) => node.index), - [20, 30, 60], - ); - assert.deepEqual( - topology.childrenByParentIndex.get(20)?.map((node) => node.index), - [40, 50], - ); - assert.equal(topology.childrenByParentIndex.get(40), undefined); -}); - -test('collects canonical viewport-root rects from type, role, and subrole', () => { - const snapshot = makeSnapshotState(MIXED_TOPOLOGY_NODES); - - const topology = buildActionableTouchTopology(snapshot.nodes); - - assert.deepEqual(topology.viewportRootRects, [ - { x: 0, y: 0, width: 390, height: 844 }, - { x: 0, y: 0, width: 390, height: 400 }, - { x: 10, y: 10, width: 100, height: 100 }, - ]); -}); diff --git a/src/core/actionable-touch-topology.ts b/src/core/actionable-touch-topology.ts deleted file mode 100644 index 151679493..000000000 --- a/src/core/actionable-touch-topology.ts +++ /dev/null @@ -1,65 +0,0 @@ -import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; -import { isViewportRootNode } from '@agent-device/contracts/snapshot'; -import { normalizeRect } from '../utils/rect-center.ts'; - -/** - * The three whole-tree lookups actionability resolution performs per node, read - * once instead of rebuilt per node: parents by index, children by parent, and - * the viewport rects an overly-broad ancestor is measured against. - * - * It exists because ranking asks the SAME policy about many candidates. One - * `find click` with `m` matches over an `n`-node capture used to walk the - * whole tree three times per candidate before it could refuse or act; with this - * it walks it once for the pass. - * - * Read-only and scoped to one pass over one node array. A topology that outlived - * its capture would answer for a screen that has already moved, so nothing caches - * it on a session — build it from the exact array being ranked and drop it with - * that pass. - * - * `nodesByIndex` is handed to `findNearestAncestor` in - * `@agent-device/contracts/snapshot-tree`. #1690 names - * `src/snapshot/snapshot-processing.ts` as that function's home; the path no - * longer exists and the contracts module is the seam that replaced it, so read - * the issue's file list as drifted, not as a second place to change. - */ -export type ActionableTouchTopology = { - /** Parent resolution by `node.index` — snapshot identity, not array position. */ - readonly nodesByIndex: ReadonlyMap; - /** Children of each parent index, in the input array's order. */ - readonly childrenByParentIndex: ReadonlyMap; - /** - * Normalized rects of the canonical viewport roots, in the input array's order. - * NOT interchangeable with `snapshot-visibility`'s `precomputedViewportRects`: - * `normalizeRect` drops negative width/height, `hasValidRect` there keeps them, - * so substituting one for the other changes which rect wins `pickLargestRect`. - */ - readonly viewportRootRects: readonly Rect[]; -}; - -/** - * One pass, three collections. Deliberately written with `for...of` rather than - * `filter`/`map`: the ranking regression counts whole-array scans on the node - * array, and the point of this builder is that a ranking pass performs exactly - * one of them. - */ -export function buildActionableTouchTopology( - nodes: readonly SnapshotNode[], -): ActionableTouchTopology { - const nodesByIndex = new Map(); - const childrenByParentIndex = new Map(); - const viewportRootRects: Rect[] = []; - for (const node of nodes) { - nodesByIndex.set(node.index, node); - if (typeof node.parentIndex === 'number') { - const siblings = childrenByParentIndex.get(node.parentIndex); - if (siblings) siblings.push(node); - else childrenByParentIndex.set(node.parentIndex, [node]); - } - if (isViewportRootNode(node)) { - const rect = normalizeRect(node.rect); - if (rect) viewportRootRects.push(rect); - } - } - return { nodesByIndex, childrenByParentIndex, viewportRootRects }; -} diff --git a/src/core/interaction-targeting.test.ts b/src/core/interaction-targeting.test.ts index a0c369fd8..5e544e20f 100644 --- a/src/core/interaction-targeting.test.ts +++ b/src/core/interaction-targeting.test.ts @@ -7,9 +7,9 @@ import { scrollingContainerTypeArb, } from '../__tests__/test-utils/property-arbitraries.ts'; import { makeSnapshotState } from '../__tests__/test-utils/snapshot-builders.ts'; -import { buildActionableTouchTopology } from './actionable-touch-topology.ts'; import { classifyActionableTouchCandidates, + createActionableTouchResolver, resolveActionableTouchResolution, } from './interaction-targeting.ts'; import { @@ -204,20 +204,16 @@ test('falls back to the original node when no usable touch target exists', () => assert.equal(resolution.node.label, 'Virtual item'); }); -test('a prebuilt topology answers every policy branch exactly as the unindexed walk does', () => { +test('the batch resolver preserves every actionability policy branch', () => { const snapshot = makeSnapshotState(INDEXED_PARITY_POLICY_NODES); - const topology = buildActionableTouchTopology(snapshot.nodes); + const resolveTouch = createActionableTouchResolver(snapshot.nodes); const unindexed = snapshot.nodes.map((node) => resolveActionableTouchResolution(snapshot.nodes, node), ); - const indexed = snapshot.nodes.map((node) => - resolveActionableTouchResolution(snapshot.nodes, node, topology), - ); + const indexed = snapshot.nodes.map(resolveTouch); assert.deepEqual(indexed, unindexed); - // Pinned rather than merely equal: two identically broken implementations - // would also be deeply equal to each other. assert.deepEqual( indexed.map((resolution) => [resolution.node.index, resolution.reason]), [ diff --git a/src/core/interaction-targeting.ts b/src/core/interaction-targeting.ts index dce1643e8..9004df070 100644 --- a/src/core/interaction-targeting.ts +++ b/src/core/interaction-targeting.ts @@ -3,13 +3,10 @@ import { centerOfRect } from '@agent-device/kernel/snapshot'; import { containsPoint, pickLargestRect } from '@agent-device/kernel/rect'; import { findNearestAncestor, + findSnapshotAncestor, normalizeType, isViewportRootNode, } from '@agent-device/contracts/snapshot'; -import { - buildActionableTouchTopology, - type ActionableTouchTopology, -} from './actionable-touch-topology.ts'; import { isSnapshotNodeInteractionBlocked } from '../snapshot/snapshot-occlusion.ts'; import { areRectsApproximatelyEqual, @@ -17,20 +14,7 @@ import { resolveRectCenter, } from '../utils/rect-center.ts'; import { intersectArea } from '../utils/screenshot-geometry.ts'; - -const SEMANTIC_TOUCH_ROLE_FRAGMENTS = [ - 'button', - 'link', - 'menuitem', - 'tabitem', - 'textfield', - 'searchfield', - 'securetextfield', - 'checkbox', - 'radio', - 'switch', - 'cell', -]; +import { isSemanticTouchTarget } from './touch-semantics.ts'; type ActionableTouchResolutionReason = | 'same-rect-descendant' @@ -45,32 +29,31 @@ type ActionableTouchResolution = { reason: ActionableTouchResolutionReason; }; +type ActionableTouchIndex = { + nodesByIndex: ReadonlyMap; + childrenByParentIndex: ReadonlyMap; + viewportRootRects: readonly Rect[]; +}; + type ActionableTouchCandidateClassification = | { kind: 'equivalent'; node: SnapshotNode } | { kind: 'ambiguous'; candidates: SnapshotNode[] }; -/** - * Mutating selector matches may collapse only when their tree structure proves - * that they describe one action: every candidate is on one ancestor/descendant - * chain and every candidate resolves to the same actionable node. Geometry may - * help resolve a wrapper to its control, but never chooses between branches. - */ export function classifyActionableTouchCandidates( nodes: SnapshotNode[], candidates: SnapshotNode[], ): ActionableTouchCandidateClassification { const first = candidates[0]; if (!first) return { kind: 'ambiguous', candidates }; - // One index for the whole classification: the ancestry chain and every - // candidate's resolution ask the same tree the same three questions. - const topology = buildActionableTouchTopology(nodes); - if (!candidatesFormSingleAncestryChain(candidates, topology.nodesByIndex)) { + const index = buildActionableTouchIndex(nodes); + if (!candidatesFormSingleAncestryChain(candidates, index.nodesByIndex)) { return { kind: 'ambiguous', candidates }; } - const actionable = resolveActionableTouchResolution(nodes, first, topology).node; + const actionable = resolveActionableTouchResolutionWithIndex(nodes, first, index).node; for (const candidate of candidates.slice(1)) { if ( - resolveActionableTouchResolution(nodes, candidate, topology).node.index !== actionable.index + resolveActionableTouchResolutionWithIndex(nodes, candidate, index).node.index !== + actionable.index ) { return { kind: 'ambiguous', candidates }; } @@ -109,13 +92,6 @@ function isAncestorOf( return false; } -/** - * The tree's viewport root as an interaction target: a viewport root node whose - * rect is exactly the tree root's. Promotion that lands here has retargeted to - * "the screen" rather than to the thing that matched, which is why the - * `hittable-ancestor-below-root` promotion stage declines it and `find` - * excludes it from candidacy and ranking. - */ export function isRootInteractionContainer( node: SnapshotNode, root: SnapshotNode | undefined, @@ -132,61 +108,84 @@ export function isRootInteractionContainer( ); } -/** - * @internal Exposed for focused policy tests. - * - * `topology` is an optional prebuilt index over THIS `nodes` array. It changes - * only how the three whole-tree lookups below are answered, never which node or - * reason comes back — a caller resolving one target may keep omitting it rather - * than pay for an index it uses once. - */ export function resolveActionableTouchResolution( nodes: SnapshotNode[], node: SnapshotNode, - topology?: ActionableTouchTopology, +): ActionableTouchResolution { + return resolveActionableTouchResolutionWithIndex(nodes, node); +} + +/** Resolves many candidates against one snapshot without rebuilding its indexes. */ +export function createActionableTouchResolver( + nodes: SnapshotNode[], +): (node: SnapshotNode) => ActionableTouchResolution { + const index = buildActionableTouchIndex(nodes); + return (node) => resolveActionableTouchResolutionWithIndex(nodes, node, index); +} + +function resolveActionableTouchResolutionWithIndex( + nodes: SnapshotNode[], + node: SnapshotNode, + index?: ActionableTouchIndex, ): ActionableTouchResolution { if (isSnapshotNodeInteractionBlocked(node)) { return { node, reason: 'covered' }; } - const descendant = findPreferredActionableDescendant(nodes, node, topology); - if (descendant?.rect && resolveRectCenter(descendant.rect)) { - return { node: descendant, reason: 'same-rect-descendant' }; - } - if (isSemanticTouchTarget(node) && node.rect && resolveRectCenter(node.rect)) { - return { node, reason: 'semantic-target' }; - } - const ancestor = findNearestHittableAncestor(nodes, node, topology); - if ( - ancestor?.rect && - !isSnapshotNodeInteractionBlocked(ancestor) && - resolveRectCenter(ancestor.rect) - ) { - if (isOverlyBroadAncestor(node, ancestor, nodes, topology)) { - return { node, reason: 'overly-broad-ancestor' }; - } - return { node: ancestor, reason: 'hittable-ancestor' }; + return ( + resolvePreferredDescendant(nodes, node, index) ?? + resolveSemanticTarget(node) ?? + resolveHittableAncestor(nodes, node, index) ?? { node, reason: 'original' } + ); +} + +function resolvePreferredDescendant( + nodes: SnapshotNode[], + node: SnapshotNode, + index: ActionableTouchIndex | undefined, +): ActionableTouchResolution | null { + const descendant = findPreferredActionableDescendant(nodes, node, index); + return descendant?.rect && resolveRectCenter(descendant.rect) + ? { node: descendant, reason: 'same-rect-descendant' } + : null; +} + +function resolveSemanticTarget(node: SnapshotNode): ActionableTouchResolution | null { + return isSemanticTouchTarget(node) && node.rect && resolveRectCenter(node.rect) + ? { node, reason: 'semantic-target' } + : null; +} + +function resolveHittableAncestor( + nodes: SnapshotNode[], + node: SnapshotNode, + index: ActionableTouchIndex | undefined, +): ActionableTouchResolution | null { + const ancestor = findNearestHittableAncestor(nodes, node, index); + if (!ancestor?.rect || isSnapshotNodeInteractionBlocked(ancestor)) return null; + if (!resolveRectCenter(ancestor.rect)) return null; + if (isOverlyBroadAncestor(node, ancestor, nodes, index)) { + return { node, reason: 'overly-broad-ancestor' }; } - return { node, reason: 'original' }; + return { node: ancestor, reason: 'hittable-ancestor' }; } function findNearestHittableAncestor( nodes: SnapshotNode[], node: SnapshotNode, - topology: ActionableTouchTopology | undefined, + index: ActionableTouchIndex | undefined, ): SnapshotNode | null { if (node.hittable) return node; - return findNearestAncestor( - nodes, - node, - (parent) => parent.hittable === true, - topology?.nodesByIndex, + const isHittable = (parent: SnapshotNode) => parent.hittable === true; + if (!index) return findNearestAncestor(nodes, node, isHittable); + return findSnapshotAncestor(nodes, node, index.nodesByIndex, (parent) => + isHittable(parent) ? parent : null, ); } function findPreferredActionableDescendant( nodes: SnapshotNode[], node: SnapshotNode, - topology: ActionableTouchTopology | undefined, + index: ActionableTouchIndex | undefined, ): SnapshotNode | null { const targetRect = normalizeRect(node.rect); if (!targetRect) return null; @@ -195,8 +194,8 @@ function findPreferredActionableDescendant( const visited = new Set(); while (!visited.has(current.ref)) { visited.add(current.ref); - const children = topology - ? (topology.childrenByParentIndex.get(current.index) ?? []) + const children = index + ? (index.childrenByParentIndex.get(current.index) ?? []) : nodes.filter((candidate) => candidate.parentIndex === current.index); const sameRectChildren = children.filter((candidate) => { if (!candidate.hittable || isSnapshotNodeInteractionBlocked(candidate)) return false; @@ -212,30 +211,11 @@ function findPreferredActionableDescendant( return current === node ? null : current; } -/** - * THE canonical interactive-role classification for touch: a node whose - * type/role/subrole names a control that independently receives taps. Shared - * by the hittable-ancestor promotion above and #1280's press-retarget - * competing-descendant guard (`press-retarget.ts`) — one list, never a - * parallel copy. - */ -export function isSemanticTouchTarget(node: SnapshotNode): boolean { - const roles = [node.type, node.role, node.subrole].map((value) => normalizeType(value ?? '')); - return roles.some(isSemanticTouchRole); -} - -function isSemanticTouchRole(role: string): boolean { - // Match Tab exactly so broad roles like Table/TabBar do not become touch targets. - return ( - role === 'tab' || SEMANTIC_TOUCH_ROLE_FRAGMENTS.some((fragment) => role.includes(fragment)) - ); -} - function isOverlyBroadAncestor( node: SnapshotNode, ancestor: SnapshotNode, nodes: SnapshotNode[], - topology: ActionableTouchTopology | undefined, + index: ActionableTouchIndex | undefined, ): boolean { const nodeRect = normalizeRect(node.rect); const ancestorRect = normalizeRect(ancestor.rect); @@ -243,7 +223,7 @@ function isOverlyBroadAncestor( if (isScrollingContainer(ancestor) && !areRectsApproximatelyEqual(nodeRect, ancestorRect)) { return true; } - const rootViewportRect = resolveRootViewportRect(nodes, nodeRect, topology); + const rootViewportRect = resolveRootViewportRect(nodes, nodeRect, index); if (!rootViewportRect) return false; if (!isRectViewportSized(ancestorRect, rootViewportRect)) return false; return !areRectsApproximatelyEqual(nodeRect, ancestorRect); @@ -266,11 +246,11 @@ function isScrollingContainer(node: SnapshotNode): boolean { function resolveRootViewportRect( nodes: SnapshotNode[], targetRect: Rect, - topology: ActionableTouchTopology | undefined, + index: ActionableTouchIndex | undefined, ): Rect | null { const targetCenter = centerOfRect(targetRect); const viewportRects = - topology?.viewportRootRects ?? + index?.viewportRootRects ?? nodes .filter(isViewportRootNode) .map((node) => normalizeRect(node.rect)) @@ -283,6 +263,25 @@ function resolveRootViewportRect( return pickLargestRect(containingRects.length > 0 ? containingRects : viewportRects); } +function buildActionableTouchIndex(nodes: readonly SnapshotNode[]): ActionableTouchIndex { + const nodesByIndex = new Map(); + const childrenByParentIndex = new Map(); + const viewportRootRects: Rect[] = []; + for (const node of nodes) { + nodesByIndex.set(node.index, node); + if (typeof node.parentIndex === 'number') { + const children = childrenByParentIndex.get(node.parentIndex); + if (children) children.push(node); + else childrenByParentIndex.set(node.parentIndex, [node]); + } + if (isViewportRootNode(node)) { + const rect = normalizeRect(node.rect); + if (rect) viewportRootRects.push(rect); + } + } + return { nodesByIndex, childrenByParentIndex, viewportRootRects }; +} + function isRectViewportSized(rect: Rect, viewportRect: Rect): boolean { const overlapArea = intersectArea(rect, viewportRect); const rectArea = rect.width * rect.height; diff --git a/src/core/interaction-touch-point.ts b/src/core/interaction-touch-point.ts index 20edf683d..f2751a4c3 100644 --- a/src/core/interaction-touch-point.ts +++ b/src/core/interaction-touch-point.ts @@ -5,7 +5,7 @@ import { normalizeRect, resolveRectCenter, } from '../utils/rect-center.ts'; -import { isSemanticTouchTarget } from './interaction-targeting.ts'; +import { isSemanticTouchTarget } from './touch-semantics.ts'; export type InteractionTouchPointResolution = | { kind: 'resolved'; point: Point; strategy: 'center' | 'parent-owned' } diff --git a/src/core/press-retarget.test.ts b/src/core/press-retarget.test.ts index 5eba30fbd..0ce8e9a35 100644 --- a/src/core/press-retarget.test.ts +++ b/src/core/press-retarget.test.ts @@ -397,7 +397,7 @@ test('resolvePressRecordingTarget P2a contrast: a container with a UNIQUE id kee // --------------------------------------------------------------------------- // P2b (#1280 re-review): the guard is built from the canonical interactive -// classification (`isSemanticTouchTarget`, core/interaction-targeting.ts), +// classification (`isSemanticTouchTarget`, core/touch-semantics.ts), // not a parallel list — roles the old private fragment list missed must // block. And a geometry condition: the selected descendant's rect center // must lie INSIDE the container's rect, else the replay tap point is not diff --git a/src/core/press-retarget.ts b/src/core/press-retarget.ts index c3e7565e9..b062fb77e 100644 --- a/src/core/press-retarget.ts +++ b/src/core/press-retarget.ts @@ -18,7 +18,7 @@ import { resolveRectCenter } from '../utils/rect-center.ts'; import { demoteNonUniqueLocalIdentity, readNodeLocalIdentity } from '@agent-device/ad-script'; import { buildIndexMap } from '../replay/target-evidence-tree.ts'; import { normalizeSelectorText } from '@agent-device/selectors'; -import { isSemanticTouchTarget } from './interaction-targeting.ts'; +import { isSemanticTouchTarget } from './touch-semantics.ts'; /** * Rule 3's fail-closed guard: a descendant that could independently receive diff --git a/src/core/touch-semantics.ts b/src/core/touch-semantics.ts new file mode 100644 index 000000000..32c3b5e9b --- /dev/null +++ b/src/core/touch-semantics.ts @@ -0,0 +1,23 @@ +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { normalizeType } from '@agent-device/contracts/snapshot'; + +const TOUCH_ROLE_FRAGMENTS = [ + 'button', + 'link', + 'menuitem', + 'tabitem', + 'textfield', + 'searchfield', + 'securetextfield', + 'checkbox', + 'radio', + 'switch', + 'cell', +]; + +export function isSemanticTouchTarget(node: SnapshotNode): boolean { + const roles = [node.type, node.role, node.subrole].map((value) => normalizeType(value ?? '')); + return roles.some( + (role) => role === 'tab' || TOUCH_ROLE_FRAGMENTS.some((fragment) => role.includes(fragment)), + ); +} diff --git a/src/daemon/handlers/__tests__/find-match-ranking.test.ts b/src/daemon/handlers/__tests__/find-match-ranking.test.ts index da8f48fa8..72dc4cb55 100644 --- a/src/daemon/handlers/__tests__/find-match-ranking.test.ts +++ b/src/daemon/handlers/__tests__/find-match-ranking.test.ts @@ -1,47 +1,20 @@ import assert from 'node:assert/strict'; -import { beforeEach, test, vi } from 'vitest'; +import { test } from 'vitest'; import type { RawSnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; import { makeSnapshotState } from '../../../__tests__/test-utils/snapshot-builders.ts'; -import { buildActionableTouchTopology } from '../../../core/actionable-touch-topology.ts'; import { preferOnscreenMatches } from '../find-match-ranking.ts'; -/** - * The builder stays REAL — `vi.fn` only wraps it, so ranking consumes production - * topology data and the wrapper reports how many times the pass built one. The - * alternative (a test double) would prove the call happened and nothing about - * what ranking then read. - */ -vi.mock('../../../core/actionable-touch-topology.ts', async (importOriginal) => { - const actual = - await importOriginal(); - return { - ...actual, - buildActionableTouchTopology: vi.fn(actual.buildActionableTouchTopology), - }; -}); - -const buildTopology = vi.mocked(buildActionableTouchTopology); - -beforeEach(() => { - buildTopology.mockClear(); -}); - const VIEWPORT = { x: 0, y: 0, width: 390, height: 844 }; const DUPLICATE_MATCH_COUNT = 32; -/** - * Counts whole-array `filter`/`map` calls on the snapshot tree. Every function - * is bound to the target so a counted scan cannot re-enter the proxy and - * inflate its own count; `matches` is a different array, so ranking's own - * legitimate on-screen filter is never mistaken for a full-tree scan. - */ function observeWholeTreeScans(nodes: SnapshotState['nodes']): { observed: SnapshotState['nodes']; - scans: { filter: number; map: number }; + scans: { iterations: number; filter: number; map: number }; } { - const scans = { filter: 0, map: 0 }; + const scans = { iterations: 0, filter: 0, map: 0 }; const observed = new Proxy(nodes, { get(target, property) { + if (property === Symbol.iterator) scans.iterations += 1; if (property === 'filter' || property === 'map') scans[property] += 1; const value = Reflect.get(target, property) as unknown; return typeof value === 'function' ? value.bind(target) : value; @@ -151,15 +124,14 @@ const MIXED_SCORE_NODES: RawSnapshotNode[] = [ }, ]; -test('a multi-match ranking pass indexes the tree once and never rescans it', () => { +test('a multi-match ranking pass scans the tree once', () => { const snapshot = duplicateHeavyCapture(); const matches = snapshot.nodes.slice(1); const { observed, scans } = observeWholeTreeScans(snapshot.nodes); const ranked = preferOnscreenMatches(matches, observed); - assert.equal(buildTopology.mock.calls.length, 1); - assert.deepEqual(scans, { filter: 0, map: 0 }); + assert.deepEqual(scans, { iterations: 1, filter: 0, map: 0 }); assert.deepEqual( ranked.map((node) => node.ref), [...matches].reverse().map((node) => node.ref), @@ -172,8 +144,7 @@ test('a single match returns without indexing the tree', () => { const ranked = preferOnscreenMatches([snapshot.nodes[1]!], observed); - assert.equal(buildTopology.mock.calls.length, 0); - assert.deepEqual(scans, { filter: 0, map: 0 }); + assert.deepEqual(scans, { iterations: 0, filter: 0, map: 0 }); assert.deepEqual( ranked.map((node) => node.ref), [snapshot.nodes[1]!.ref], @@ -186,10 +157,11 @@ test('a capture without a root rect returns matches unranked and unindexed', () ...MIXED_SCORE_NODES.slice(1), ]); const matches = snapshot.nodes.slice(1); + const { observed, scans } = observeWholeTreeScans(snapshot.nodes); - const ranked = preferOnscreenMatches(matches, snapshot.nodes); + const ranked = preferOnscreenMatches(matches, observed); - assert.equal(buildTopology.mock.calls.length, 0); + assert.deepEqual(scans, { iterations: 0, filter: 0, map: 0 }); assert.deepEqual( ranked.map((node) => node.ref), matches.map((node) => node.ref), diff --git a/src/daemon/handlers/find-match-ranking.ts b/src/daemon/handlers/find-match-ranking.ts index 4c322382e..0c3a87ca9 100644 --- a/src/daemon/handlers/find-match-ranking.ts +++ b/src/daemon/handlers/find-match-ranking.ts @@ -1,9 +1,6 @@ import { centerOfRect, type SnapshotState } from '@agent-device/kernel/snapshot'; import { - buildActionableTouchTopology, - type ActionableTouchTopology, -} from '../../core/actionable-touch-topology.ts'; -import { + createActionableTouchResolver, isRootInteractionContainer, resolveActionableTouchResolution, } from '../../core/interaction-targeting.ts'; @@ -32,20 +29,21 @@ export function preferOnscreenMatches( ); }); const preferred = onscreen.length > 0 ? onscreen : matches; - // The only topology build site for the pass, and only once there is really - // something to order: one candidate answers no comparative question, so it - // must not pay for an index over the whole capture. if (preferred.length < 2) return preferred; - return rankInteractiveMatches(preferred, nodes, buildActionableTouchTopology(nodes)); + return rankInteractiveMatches(preferred, nodes, createActionableTouchResolver(nodes)); } function rankInteractiveMatches( matches: SnapshotState['nodes'], nodes: SnapshotState['nodes'], - topology: ActionableTouchTopology, + resolveTouch: ReturnType, ): SnapshotState['nodes'] { return matches - .map((node, index) => ({ node, index, score: interactiveMatchScore(node, nodes, topology) })) + .map((node, index) => ({ + node, + index, + score: interactiveMatchScore(node, nodes, resolveTouch), + })) .sort((left, right) => { if (right.score !== left.score) return right.score - left.score; return rectArea(left.node) - rectArea(right.node) || left.index - right.index; @@ -56,9 +54,9 @@ function rankInteractiveMatches( function interactiveMatchScore( node: SnapshotState['nodes'][number], nodes: SnapshotState['nodes'], - topology: ActionableTouchTopology, + resolveTouch: ReturnType, ): number { - const resolution = resolveActionableTouchResolution(nodes, node, topology); + const resolution = resolveTouch(node); if (resolution.reason === 'covered') return 0; const resolved = resolvedTouchScore(resolution, nodes[0]); if (resolved > 0) return resolved; diff --git a/test/integration/ios-simulator-e2e/snapshot-backend-conformance.ts b/test/integration/ios-simulator-e2e/snapshot-backend-conformance.ts index 988505672..57234dc68 100644 --- a/test/integration/ios-simulator-e2e/snapshot-backend-conformance.ts +++ b/test/integration/ios-simulator-e2e/snapshot-backend-conformance.ts @@ -12,7 +12,7 @@ import { type SnapshotPreferredBackend, } from '@agent-device/kernel/snapshot'; import { SNAPSHOT_BACKEND_CAPABILITIES } from '../../../src/snapshot-quality/backend-capabilities.ts'; -import { isSemanticTouchTarget } from '../../../src/core/interaction-targeting.ts'; +import { isSemanticTouchTarget } from '../../../src/core/touch-semantics.ts'; export type SnapshotBackendConformanceFixture = { screen: string;