Inspect native interaction regions in Gesture Debug

LookInside.xcodeproj/project.pbxproj

@@ -220,6 +220,7 @@
 				GestureDebug/LKGestureDebugView.swift,
 				GestureDebug/LKGestureDebugWindowController.swift,
 				GestureDebug/LKGestureNodeDetailView.swift,
+				GestureDebug/LKNativeInteractionDetailView.swift,
 				MCPBridge/LKMCPBridgeAttributeValueDecoder.swift,
 				MCPBridge/LKMCPBridgeConnection.swift,
 				MCPBridge/LKMCPBridgeDetailsService.swift,

LookInside/GestureDebug/LKGestureCaptureModels.swift

@@ -14,6 +14,27 @@ struct LKGestureCaptureBatch: Codable {
     var message: String
     var pollDurationMS: Double
     var overlayStatus: LKGestureOverlayStatus?
+    var nativeRegions: [LKNativeInteractionRegion]?
+}
+
+struct LKNativeInteractionRegion: Codable, Identifiable {
+    var id: String
+    var source: String
+    var viewClass: String
+    var viewAddress: String
+    var windowAddress: String
+    var label: String
+    var handlers: [LKNativeInteractionHandler]
+    var geometry: LKGestureCaptureGeometry
+    var isActive: Bool
+    var geometryKind: String
+}
+
+struct LKNativeInteractionHandler: Codable {
+    var kind: String
+    var name: String
+    var target: String?
+    var events: UInt?
 }
 
 struct LKGestureOverlayStatus: Codable {
@@ -101,6 +122,7 @@ struct LKGestureCaptureArchive: Codable {
     var sessionID: String?
     var snapshots: [LKGestureCaptureSnapshot]
     var records: [LKGestureCaptureRecord]
+    var nativeRegions: [LKNativeInteractionRegion]?
 }
 
 struct LKGestureTreeItem: Identifiable {

LookInside/GestureDebug/LKGestureDebugSession.swift

@@ -20,6 +20,8 @@ final class LKGestureDebugSession: ObservableObject {
     @Published var followsLatest = true
     @Published var overlayEnabled = true
     @Published private(set) var overlayStatus: LKGestureOverlayStatus?
+    @Published private(set) var nativeRegions: [LKNativeInteractionRegion] = []
+    @Published private var lastNativeRegions: [LKNativeInteractionRegion] = []
     @Published private(set) var appName = "No target"
     @Published private(set) var supported = false
 
@@ -35,6 +37,10 @@ final class LKGestureDebugSession: ObservableObject {
         snapshots.first { $0.id == selectedSnapshotID }
     }
 
+    var canExport: Bool {
+        !records.isEmpty || !snapshots.isEmpty || !lastNativeRegions.isEmpty
+    }
+
     init() {
         let manager = LKConnectionManager.sharedInstance()
         if let disposable = manager?.didReceivePush?.subscribeNext({ [weak self] value in
@@ -69,6 +75,7 @@ final class LKGestureDebugSession: ObservableObject {
         supported = (app?.appInfo?.gestureDebugProtocolVersion ?? 0) >= 1
         snapshots.removeAll()
         records.removeAll()
+        lastNativeRegions.removeAll()
         selectedSnapshotID = nil
         selectedNodeID = nil
         message = supported
@@ -87,6 +94,8 @@ final class LKGestureDebugSession: ObservableObject {
         redactedCount = 0
         droppedCount = 0
         overlayStatus = nil
+        nativeRegions.removeAll()
+        lastNativeRegions.removeAll()
         snapshots.removeAll()
         records.removeAll()
         selectedSnapshotID = nil
@@ -122,6 +131,7 @@ final class LKGestureDebugSession: ObservableObject {
         isRunning = false
         isStarting = false
         overlayStatus = nil
+        nativeRegions.removeAll()
         state = "stopped"
         if token != nil {
             message = "Capture stopped. Recorded events remain available."
@@ -179,7 +189,7 @@ final class LKGestureDebugSession: ObservableObject {
     func archiveData() throws -> Data {
         let archive = LKGestureCaptureArchive(
             appName: appName, bundleIdentifier: app?.appInfo?.appBundleIdentifier ?? "",
-            sessionID: lastSessionID, snapshots: snapshots, records: records
+            sessionID: lastSessionID, snapshots: snapshots, records: records, nativeRegions: lastNativeRegions
         )
         let encoder = JSONEncoder()
         encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
@@ -207,6 +217,10 @@ final class LKGestureDebugSession: ObservableObject {
             droppedCount = batch.droppedCount
             pollDurationMS = batch.pollDurationMS
             overlayStatus = batch.overlayStatus
+            nativeRegions = batch.nativeRegions ?? []
+            if batch.state != "stopped", batch.state != "error" {
+                lastNativeRegions = nativeRegions
+            }
             state = batch.state
             isStarting = state == "starting"
             isRunning = state == "waiting" || state == "capturing" || state == "redacted"

LookInside/GestureDebug/LKGestureDebugView.swift

@@ -10,6 +10,7 @@ struct LKGestureDebugView: View {
         case responders = "Responders"
         case gestures = "Gestures"
         case bindings = "Bindings"
+        case native = "Native"
         case hitTest = "Hit Test"
         case raw = "Raw Log"
     }
@@ -28,7 +29,9 @@ struct LKGestureDebugView: View {
                     }
                     .pickerStyle(.segmented)
                     .padding(12)
-                    if let snapshot = session.selectedSnapshot {
+                    if section == .native {
+                        LKNativeInteractionDetailView(regions: session.nativeRegions, isRunning: session.isRunning, overlayEnabled: session.overlayEnabled)
+                    } else if let snapshot = session.selectedSnapshot {
                         snapshotContent(snapshot)
                     } else {
                         emptyState
@@ -49,7 +52,7 @@ struct LKGestureDebugView: View {
             Spacer()
             Toggle("Gesture borders", isOn: $session.overlayEnabled)
                 .help("Show gesture regions on the current page. A translucent fill marks the current input binding.")
-            Button(session.overlayStatus?.mode == "livePage" ? "Refresh Borders" : "Clear Borders") { session.clearBorders() }
+            Button(session.overlayStatus?.mode == "persistentObserved" ? "Clear Borders" : "Refresh Borders") { session.clearBorders() }
                 .disabled(!session.isRunning || !session.overlayEnabled)
             Button(session.isRunning || session.isStarting ? "Stop Capture" : "Start Capture") {
                 if session.isRunning || session.isStarting {
@@ -61,7 +64,7 @@ struct LKGestureDebugView: View {
             .disabled(!session.supported)
             .keyboardShortcut("r", modifiers: .command)
             Button("Export…", action: export)
-                .disabled(session.records.isEmpty && session.snapshots.isEmpty)
+                .disabled(!session.canExport)
         }
         .padding(12)
     }
@@ -88,7 +91,13 @@ struct LKGestureDebugView: View {
             if let overlay = session.overlayStatus, overlay.isEnabled, session.isRunning {
                 Text(overlay.mode == "livePage"
                     ? "\(overlay.regionCount) borders on the current page · Updates automatically with the page."
-                    : "\(overlay.regionCount) observed borders · Whole-page reading is unavailable on this runtime. Clear after page changes.")
+                    : (overlay.mode == "nativePage"
+                        ? "\(session.nativeRegions.count) native borders on the current page · SwiftUI whole-page reading is unavailable on this runtime."
+                        : "\(overlay.regionCount) observed borders · Whole-page reading is unavailable on this runtime. Clear after page changes."))
+                    .font(.caption).foregroundStyle(.secondary)
+            }
+            if !session.nativeRegions.isEmpty {
+                Text("\(session.nativeRegions.count) native controls / list rows · See Native for handler details.")
                     .font(.caption).foregroundStyle(.secondary)
             }
             if session.state == "redacted" {
@@ -172,6 +181,7 @@ struct LKGestureDebugView: View {
                     .padding(.horizontal, 14).padding(.bottom, 8)
             }
             switch section {
+            case .native: EmptyView()
             case .responders: tree(snapshot.responders, snapshot: snapshot)
             case .gestures: tree(snapshot.gestures, snapshot: snapshot)
             case .bindings:

LookInside/GestureDebug/LKNativeInteractionDetailView.swift

@@ -0,0 +1,86 @@
+import SwiftUI
+
+/// Current native regions are independent of historical SwiftUI log events.
+struct LKNativeInteractionDetailView: View {
+    let regions: [LKNativeInteractionRegion]
+    let isRunning: Bool
+    let overlayEnabled: Bool
+    @State private var selection: String?
+
+    var body: some View {
+        VStack(alignment: .leading, spacing: 0) {
+            Text("Current page · \(regions.count) native regions")
+                .font(.headline).padding(12)
+            Text("Native controls and actionable list rows appear before interaction. Dashed outlines show view bounds; custom hit-test shapes may differ.")
+                .font(.caption).foregroundStyle(.secondary).padding(.horizontal, 12).padding(.bottom, 12)
+            Divider()
+            if regions.isEmpty {
+                Text(isRunning && overlayEnabled ? "No supported native interactions on this page. Native collection currently supports UIKit." : "Start capture with Gesture borders enabled to inspect native regions.")
+                    .foregroundStyle(.secondary).padding()
+                    .frame(maxWidth: .infinity, maxHeight: .infinity)
+            } else {
+                HSplitView {
+                    List(regions, selection: $selection) { region in
+                        VStack(alignment: .leading, spacing: 5) {
+                            HStack {
+                                Text(region.label).lineLimit(2)
+                                if region.isActive {
+                                    Text("Active").foregroundStyle(.green)
+                                }
+                            }
+                            Text(region.source == "listSelection" ? "List selection" : "UIControl")
+                                .font(.caption).foregroundStyle(.secondary)
+                            Text(region.viewAddress).font(.system(.caption, design: .monospaced)).foregroundStyle(.secondary)
+                        }.padding(.vertical, 3).tag(region.id)
+                    }.frame(minWidth: 220, idealWidth: 270)
+                    if let region = regions.first(where: { $0.id == selection }) {
+                        detail(region).frame(minWidth: 280)
+                    } else {
+                        Text("Select a native region to inspect its handlers.")
+                            .foregroundStyle(.secondary).frame(minWidth: 280, maxWidth: .infinity, maxHeight: .infinity)
+                    }
+                }
+            }
+        }
+        .onChange(of: regions.map(\.id)) { ids in
+            if let selection, !ids.contains(selection) {
+                self.selection = nil
+            }
+        }
+    }
+
+    private func detail(_ region: LKNativeInteractionRegion) -> some View {
+        ScrollView {
+            VStack(alignment: .leading, spacing: 16) {
+                field("View", region.viewClass)
+                field("Address", region.viewAddress)
+                field("Window", region.windowAddress)
+                field("State", region.isActive ? "Tracking / highlighted" : "Idle")
+                field("Geometry", region.geometryKind == "swiftUIContentShape" ? "SwiftUI content shape on native control" : "View bounds (hit-test approximation)")
+                if let x = region.geometry.x, let y = region.geometry.y, let width = region.geometry.width, let height = region.geometry.height {
+                    field("Window coordinates", String(format: "(%.1f, %.1f) · %.1f × %.1f", x, y, width, height))
+                }
+                ForEach(Array(region.handlers.enumerated()), id: \.offset) { _, handler in
+                    VStack(alignment: .leading, spacing: 6) {
+                        Text(handler.kind == "uiAction" ? "UIAction" : (handler.kind == "listSelection" ? "List selection callback" : "Target-action"))
+                            .font(.caption).foregroundStyle(.secondary)
+                        Text(handler.name).font(.system(.callout, design: .monospaced))
+                        if let target = handler.target {
+                            Text(target).font(.system(.caption, design: .monospaced))
+                        }
+                        if let events = handler.events {
+                            Text("UIControl events: 0x\(String(events, radix: 16))").font(.caption)
+                        }
+                    }
+                }
+            }.textSelection(.enabled).padding(16).frame(maxWidth: .infinity, alignment: .leading)
+        }
+    }
+
+    private func field(_ title: String, _ value: String) -> some View {
+        VStack(alignment: .leading, spacing: 6) {
+            Text(title).font(.caption).foregroundStyle(.secondary)
+            Text(value).font(.system(.callout, design: .monospaced))
+        }
+    }
+}

Scripts/test-gesture-models.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+GESTURE_TEST_DIR="$(mktemp -d "${TMPDIR:-/tmp}/lookinside-gesture-models.XXXXXX")"
+trap 'rm -rf "$GESTURE_TEST_DIR"' EXIT
+swiftc -parse-as-library "$ROOT/LookInside/GestureDebug/LKGestureCaptureModels.swift" \
+	"$ROOT/Tests/GestureDebug/LKGestureCaptureModelsTests.swift" -o "$GESTURE_TEST_DIR/gesture-models"
+"$GESTURE_TEST_DIR/gesture-models"

Scripts/test.sh

@@ -58,6 +58,7 @@ run_xcode_build() {
 }
 
 run_command "sync derived source" bash Scripts/sync-derived-source.sh
+run_command "gesture capture model tests" bash Scripts/test-gesture-models.sh
 run_swift_build "swift build" build
 run_swift_build "swift build target=LookinServer" build -c debug --target LookinServer
 run_xcode_build "LookInside" "Debug"

Tests/GestureDebug/LKGestureCaptureModelsTests.swift

@@ -0,0 +1,27 @@
+import Foundation
+
+@main
+struct LKGestureCaptureModelsTests {
+    static func main() throws {
+        let legacy = Data(#"{"schemaVersion":1,"sessionID":"s","sequence":1,"snapshots":[],"records":[],"recordCount":0,"redactedCount":0,"droppedCount":0,"state":"capturing","message":"","pollDurationMS":0}"#.utf8)
+        let decoder = JSONDecoder()
+        var batch = try decoder.decode(LKGestureCaptureBatch.self, from: legacy)
+        precondition(batch.nativeRegions == nil, "Old Servers remain compatible")
+        batch.nativeRegions = [LKNativeInteractionRegion(
+            id: "0x1", source: "listSelection", viewClass: "SwiftUI.ListCollectionViewCell", viewAddress: "0x1", windowAddress: "0x2",
+            label: "List row 0:1", handlers: [LKNativeInteractionHandler(kind: "listSelection", name: "selectionBehavior.onSelect")],
+            geometry: LKGestureCaptureGeometry(x: 20, y: 40, width: 320, height: 44, source: "nativeViewBounds", coordinateSpace: "window"),
+            isActive: false, geometryKind: "viewBounds"
+        )]
+        let current = try decoder.decode(LKGestureCaptureBatch.self, from: JSONEncoder().encode(batch))
+        precondition(current.nativeRegions?.first?.handlers.first?.name == "selectionBehavior.onSelect")
+        precondition(current.nativeRegions?.first?.geometry.width == 320)
+        let archive = LKGestureCaptureArchive(appName: "Fixture", bundleIdentifier: "fixture", sessionID: "s", snapshots: [], records: [], nativeRegions: current.nativeRegions)
+        let roundTrip = try decoder.decode(LKGestureCaptureArchive.self, from: JSONEncoder().encode(archive))
+        precondition(roundTrip.nativeRegions?.first?.id == "0x1", "Native-only captures can be exported")
+        batch.nativeRegions = []
+        let cleared = try decoder.decode(LKGestureCaptureBatch.self, from: JSONEncoder().encode(batch))
+        precondition(cleared.nativeRegions?.isEmpty == true)
+        print("Gesture capture model compatibility and native archive tests passed")
+    }
+}