diff --git a/LookInside.xcodeproj/project.pbxproj b/LookInside.xcodeproj/project.pbxproj index b1cbaf5..c11ba71 100644 --- a/LookInside.xcodeproj/project.pbxproj +++ b/LookInside.xcodeproj/project.pbxproj @@ -220,6 +220,7 @@ GestureDebug/LKGestureDebugView.swift, GestureDebug/LKGestureDebugWindowController.swift, GestureDebug/LKGestureNodeDetailView.swift, + GestureDebug/LKNativeInteractionDetailView.swift, MCPBridge/LKMCPBridgeAttributeValueDecoder.swift, MCPBridge/LKMCPBridgeConnection.swift, MCPBridge/LKMCPBridgeDetailsService.swift, diff --git a/LookInside/GestureDebug/LKGestureCaptureModels.swift b/LookInside/GestureDebug/LKGestureCaptureModels.swift index a03ff09..7da9835 100644 --- a/LookInside/GestureDebug/LKGestureCaptureModels.swift +++ b/LookInside/GestureDebug/LKGestureCaptureModels.swift @@ -14,6 +14,27 @@ struct LKGestureCaptureBatch: Codable { var message: String var pollDurationMS: Double var overlayStatus: LKGestureOverlayStatus? + var nativeRegions: [LKNativeInteractionRegion]? +} + +struct LKNativeInteractionRegion: Codable, Identifiable { + var id: String + var source: String + var viewClass: String + var viewAddress: String + var windowAddress: String + var label: String + var handlers: [LKNativeInteractionHandler] + var geometry: LKGestureCaptureGeometry + var isActive: Bool + var geometryKind: String +} + +struct LKNativeInteractionHandler: Codable { + var kind: String + var name: String + var target: String? + var events: UInt? } struct LKGestureOverlayStatus: Codable { @@ -101,6 +122,7 @@ struct LKGestureCaptureArchive: Codable { var sessionID: String? var snapshots: [LKGestureCaptureSnapshot] var records: [LKGestureCaptureRecord] + var nativeRegions: [LKNativeInteractionRegion]? } struct LKGestureTreeItem: Identifiable { diff --git a/LookInside/GestureDebug/LKGestureDebugSession.swift b/LookInside/GestureDebug/LKGestureDebugSession.swift index 4e3a611..4520475 100644 --- a/LookInside/GestureDebug/LKGestureDebugSession.swift +++ b/LookInside/GestureDebug/LKGestureDebugSession.swift @@ -20,6 +20,8 @@ final class LKGestureDebugSession: ObservableObject { @Published var followsLatest = true @Published var overlayEnabled = true @Published private(set) var overlayStatus: LKGestureOverlayStatus? + @Published private(set) var nativeRegions: [LKNativeInteractionRegion] = [] + @Published private var lastNativeRegions: [LKNativeInteractionRegion] = [] @Published private(set) var appName = "No target" @Published private(set) var supported = false @@ -35,6 +37,10 @@ final class LKGestureDebugSession: ObservableObject { snapshots.first { $0.id == selectedSnapshotID } } + var canExport: Bool { + !records.isEmpty || !snapshots.isEmpty || !lastNativeRegions.isEmpty + } + init() { let manager = LKConnectionManager.sharedInstance() if let disposable = manager?.didReceivePush?.subscribeNext({ [weak self] value in @@ -69,6 +75,7 @@ final class LKGestureDebugSession: ObservableObject { supported = (app?.appInfo?.gestureDebugProtocolVersion ?? 0) >= 1 snapshots.removeAll() records.removeAll() + lastNativeRegions.removeAll() selectedSnapshotID = nil selectedNodeID = nil message = supported @@ -87,6 +94,8 @@ final class LKGestureDebugSession: ObservableObject { redactedCount = 0 droppedCount = 0 overlayStatus = nil + nativeRegions.removeAll() + lastNativeRegions.removeAll() snapshots.removeAll() records.removeAll() selectedSnapshotID = nil @@ -122,6 +131,7 @@ final class LKGestureDebugSession: ObservableObject { isRunning = false isStarting = false overlayStatus = nil + nativeRegions.removeAll() state = "stopped" if token != nil { message = "Capture stopped. Recorded events remain available." @@ -179,7 +189,7 @@ final class LKGestureDebugSession: ObservableObject { func archiveData() throws -> Data { let archive = LKGestureCaptureArchive( appName: appName, bundleIdentifier: app?.appInfo?.appBundleIdentifier ?? "", - sessionID: lastSessionID, snapshots: snapshots, records: records + sessionID: lastSessionID, snapshots: snapshots, records: records, nativeRegions: lastNativeRegions ) let encoder = JSONEncoder() encoder.outputFormatting = [.prettyPrinted, .sortedKeys] @@ -207,6 +217,10 @@ final class LKGestureDebugSession: ObservableObject { droppedCount = batch.droppedCount pollDurationMS = batch.pollDurationMS overlayStatus = batch.overlayStatus + nativeRegions = batch.nativeRegions ?? [] + if batch.state != "stopped", batch.state != "error" { + lastNativeRegions = nativeRegions + } state = batch.state isStarting = state == "starting" isRunning = state == "waiting" || state == "capturing" || state == "redacted" diff --git a/LookInside/GestureDebug/LKGestureDebugView.swift b/LookInside/GestureDebug/LKGestureDebugView.swift index 6e61157..553ce14 100644 --- a/LookInside/GestureDebug/LKGestureDebugView.swift +++ b/LookInside/GestureDebug/LKGestureDebugView.swift @@ -10,6 +10,7 @@ struct LKGestureDebugView: View { case responders = "Responders" case gestures = "Gestures" case bindings = "Bindings" + case native = "Native" case hitTest = "Hit Test" case raw = "Raw Log" } @@ -28,7 +29,9 @@ struct LKGestureDebugView: View { } .pickerStyle(.segmented) .padding(12) - if let snapshot = session.selectedSnapshot { + if section == .native { + LKNativeInteractionDetailView(regions: session.nativeRegions, isRunning: session.isRunning, overlayEnabled: session.overlayEnabled) + } else if let snapshot = session.selectedSnapshot { snapshotContent(snapshot) } else { emptyState @@ -49,7 +52,7 @@ struct LKGestureDebugView: View { Spacer() Toggle("Gesture borders", isOn: $session.overlayEnabled) .help("Show gesture regions on the current page. A translucent fill marks the current input binding.") - Button(session.overlayStatus?.mode == "livePage" ? "Refresh Borders" : "Clear Borders") { session.clearBorders() } + Button(session.overlayStatus?.mode == "persistentObserved" ? "Clear Borders" : "Refresh Borders") { session.clearBorders() } .disabled(!session.isRunning || !session.overlayEnabled) Button(session.isRunning || session.isStarting ? "Stop Capture" : "Start Capture") { if session.isRunning || session.isStarting { @@ -61,7 +64,7 @@ struct LKGestureDebugView: View { .disabled(!session.supported) .keyboardShortcut("r", modifiers: .command) Button("Export…", action: export) - .disabled(session.records.isEmpty && session.snapshots.isEmpty) + .disabled(!session.canExport) } .padding(12) } @@ -88,7 +91,13 @@ struct LKGestureDebugView: View { if let overlay = session.overlayStatus, overlay.isEnabled, session.isRunning { Text(overlay.mode == "livePage" ? "\(overlay.regionCount) borders on the current page · Updates automatically with the page." - : "\(overlay.regionCount) observed borders · Whole-page reading is unavailable on this runtime. Clear after page changes.") + : (overlay.mode == "nativePage" + ? "\(session.nativeRegions.count) native borders on the current page · SwiftUI whole-page reading is unavailable on this runtime." + : "\(overlay.regionCount) observed borders · Whole-page reading is unavailable on this runtime. Clear after page changes.")) + .font(.caption).foregroundStyle(.secondary) + } + if !session.nativeRegions.isEmpty { + Text("\(session.nativeRegions.count) native controls / list rows · See Native for handler details.") .font(.caption).foregroundStyle(.secondary) } if session.state == "redacted" { @@ -172,6 +181,7 @@ struct LKGestureDebugView: View { .padding(.horizontal, 14).padding(.bottom, 8) } switch section { + case .native: EmptyView() case .responders: tree(snapshot.responders, snapshot: snapshot) case .gestures: tree(snapshot.gestures, snapshot: snapshot) case .bindings: diff --git a/LookInside/GestureDebug/LKNativeInteractionDetailView.swift b/LookInside/GestureDebug/LKNativeInteractionDetailView.swift new file mode 100644 index 0000000..5f657d5 --- /dev/null +++ b/LookInside/GestureDebug/LKNativeInteractionDetailView.swift @@ -0,0 +1,86 @@ +import SwiftUI + +/// Current native regions are independent of historical SwiftUI log events. +struct LKNativeInteractionDetailView: View { + let regions: [LKNativeInteractionRegion] + let isRunning: Bool + let overlayEnabled: Bool + @State private var selection: String? + + var body: some View { + VStack(alignment: .leading, spacing: 0) { + Text("Current page · \(regions.count) native regions") + .font(.headline).padding(12) + Text("Native controls and actionable list rows appear before interaction. Dashed outlines show view bounds; custom hit-test shapes may differ.") + .font(.caption).foregroundStyle(.secondary).padding(.horizontal, 12).padding(.bottom, 12) + Divider() + if regions.isEmpty { + Text(isRunning && overlayEnabled ? "No supported native interactions on this page. Native collection currently supports UIKit." : "Start capture with Gesture borders enabled to inspect native regions.") + .foregroundStyle(.secondary).padding() + .frame(maxWidth: .infinity, maxHeight: .infinity) + } else { + HSplitView { + List(regions, selection: $selection) { region in + VStack(alignment: .leading, spacing: 5) { + HStack { + Text(region.label).lineLimit(2) + if region.isActive { + Text("Active").foregroundStyle(.green) + } + } + Text(region.source == "listSelection" ? "List selection" : "UIControl") + .font(.caption).foregroundStyle(.secondary) + Text(region.viewAddress).font(.system(.caption, design: .monospaced)).foregroundStyle(.secondary) + }.padding(.vertical, 3).tag(region.id) + }.frame(minWidth: 220, idealWidth: 270) + if let region = regions.first(where: { $0.id == selection }) { + detail(region).frame(minWidth: 280) + } else { + Text("Select a native region to inspect its handlers.") + .foregroundStyle(.secondary).frame(minWidth: 280, maxWidth: .infinity, maxHeight: .infinity) + } + } + } + } + .onChange(of: regions.map(\.id)) { ids in + if let selection, !ids.contains(selection) { + self.selection = nil + } + } + } + + private func detail(_ region: LKNativeInteractionRegion) -> some View { + ScrollView { + VStack(alignment: .leading, spacing: 16) { + field("View", region.viewClass) + field("Address", region.viewAddress) + field("Window", region.windowAddress) + field("State", region.isActive ? "Tracking / highlighted" : "Idle") + field("Geometry", region.geometryKind == "swiftUIContentShape" ? "SwiftUI content shape on native control" : "View bounds (hit-test approximation)") + if let x = region.geometry.x, let y = region.geometry.y, let width = region.geometry.width, let height = region.geometry.height { + field("Window coordinates", String(format: "(%.1f, %.1f) · %.1f × %.1f", x, y, width, height)) + } + ForEach(Array(region.handlers.enumerated()), id: \.offset) { _, handler in + VStack(alignment: .leading, spacing: 6) { + Text(handler.kind == "uiAction" ? "UIAction" : (handler.kind == "listSelection" ? "List selection callback" : "Target-action")) + .font(.caption).foregroundStyle(.secondary) + Text(handler.name).font(.system(.callout, design: .monospaced)) + if let target = handler.target { + Text(target).font(.system(.caption, design: .monospaced)) + } + if let events = handler.events { + Text("UIControl events: 0x\(String(events, radix: 16))").font(.caption) + } + } + } + }.textSelection(.enabled).padding(16).frame(maxWidth: .infinity, alignment: .leading) + } + } + + private func field(_ title: String, _ value: String) -> some View { + VStack(alignment: .leading, spacing: 6) { + Text(title).font(.caption).foregroundStyle(.secondary) + Text(value).font(.system(.callout, design: .monospaced)) + } + } +} diff --git a/Scripts/test-gesture-models.sh b/Scripts/test-gesture-models.sh new file mode 100755 index 0000000..1e442fa --- /dev/null +++ b/Scripts/test-gesture-models.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +GESTURE_TEST_DIR="$(mktemp -d "${TMPDIR:-/tmp}/lookinside-gesture-models.XXXXXX")" +trap 'rm -rf "$GESTURE_TEST_DIR"' EXIT +swiftc -parse-as-library "$ROOT/LookInside/GestureDebug/LKGestureCaptureModels.swift" \ + "$ROOT/Tests/GestureDebug/LKGestureCaptureModelsTests.swift" -o "$GESTURE_TEST_DIR/gesture-models" +"$GESTURE_TEST_DIR/gesture-models" diff --git a/Scripts/test.sh b/Scripts/test.sh index d5d3b12..66c55ba 100755 --- a/Scripts/test.sh +++ b/Scripts/test.sh @@ -58,6 +58,7 @@ run_xcode_build() { } run_command "sync derived source" bash Scripts/sync-derived-source.sh +run_command "gesture capture model tests" bash Scripts/test-gesture-models.sh run_swift_build "swift build" build run_swift_build "swift build target=LookinServer" build -c debug --target LookinServer run_xcode_build "LookInside" "Debug" diff --git a/Tests/GestureDebug/LKGestureCaptureModelsTests.swift b/Tests/GestureDebug/LKGestureCaptureModelsTests.swift new file mode 100644 index 0000000..36ab12d --- /dev/null +++ b/Tests/GestureDebug/LKGestureCaptureModelsTests.swift @@ -0,0 +1,27 @@ +import Foundation + +@main +struct LKGestureCaptureModelsTests { + static func main() throws { + let legacy = Data(#"{"schemaVersion":1,"sessionID":"s","sequence":1,"snapshots":[],"records":[],"recordCount":0,"redactedCount":0,"droppedCount":0,"state":"capturing","message":"","pollDurationMS":0}"#.utf8) + let decoder = JSONDecoder() + var batch = try decoder.decode(LKGestureCaptureBatch.self, from: legacy) + precondition(batch.nativeRegions == nil, "Old Servers remain compatible") + batch.nativeRegions = [LKNativeInteractionRegion( + id: "0x1", source: "listSelection", viewClass: "SwiftUI.ListCollectionViewCell", viewAddress: "0x1", windowAddress: "0x2", + label: "List row 0:1", handlers: [LKNativeInteractionHandler(kind: "listSelection", name: "selectionBehavior.onSelect")], + geometry: LKGestureCaptureGeometry(x: 20, y: 40, width: 320, height: 44, source: "nativeViewBounds", coordinateSpace: "window"), + isActive: false, geometryKind: "viewBounds" + )] + let current = try decoder.decode(LKGestureCaptureBatch.self, from: JSONEncoder().encode(batch)) + precondition(current.nativeRegions?.first?.handlers.first?.name == "selectionBehavior.onSelect") + precondition(current.nativeRegions?.first?.geometry.width == 320) + let archive = LKGestureCaptureArchive(appName: "Fixture", bundleIdentifier: "fixture", sessionID: "s", snapshots: [], records: [], nativeRegions: current.nativeRegions) + let roundTrip = try decoder.decode(LKGestureCaptureArchive.self, from: JSONEncoder().encode(archive)) + precondition(roundTrip.nativeRegions?.first?.id == "0x1", "Native-only captures can be exported") + batch.nativeRegions = [] + let cleared = try decoder.decode(LKGestureCaptureBatch.self, from: JSONEncoder().encode(batch)) + precondition(cleared.nativeRegions?.isEmpty == true) + print("Gesture capture model compatibility and native archive tests passed") + } +}