diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index c21eb9c3..3e5d06f0 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -7,7 +7,7 @@ const siteUrl = "https://simdeck.nativescript.org"; export default defineConfig({ title: "SimDeck", description: - "A local-first iOS Simulator control plane with a browser UI, REST API, and WebRTC video.", + "Stream, inspect, and automate iOS Simulators and Android emulators from a browser, CLI, or test.", lang: "en-US", cleanUrls: true, lastUpdated: true, @@ -21,7 +21,7 @@ export default defineConfig({ { property: "og:description", content: - "A local iOS Simulator control plane with a browser UI, REST API, and WebRTC video.", + "Stream, inspect, and automate iOS Simulators and Android emulators from a browser, CLI, or test.", }, ], ["meta", { property: "og:url", content: `${siteUrl}/` }], @@ -36,7 +36,7 @@ export default defineConfig({ { text: "CLI", link: "/cli/", activeMatch: "/cli/" }, { text: "API", link: "/api/rest", activeMatch: "/api/" }, { - text: "Inspector", + text: "Inspectors", link: "/inspector/", activeMatch: "/inspector/", }, @@ -46,16 +46,10 @@ export default defineConfig({ activeMatch: "/extensions/", }, { - text: "0.1.0", + text: "0.1.5", items: [ - { - text: "Changelog", - link: `${githubUrl}/releases`, - }, - { - text: "Contributing", - link: "/contributing", - }, + { text: "Changelog", link: `${githubUrl}/releases` }, + { text: "Contributing", link: "/contributing" }, ], }, ], @@ -63,28 +57,28 @@ export default defineConfig({ sidebar: { "/guide/": [ { - text: "Getting Started", + text: "Start", items: [ - { text: "Introduction", link: "/guide/" }, - { text: "Installation", link: "/guide/installation" }, + { text: "Overview", link: "/guide/" }, + { text: "Install", link: "/guide/installation" }, { text: "Quick Start", link: "/guide/quick-start" }, ], }, { - text: "Concepts", + text: "Use", items: [ - { text: "Architecture", link: "/guide/architecture" }, - { text: "Video Pipeline", link: "/guide/video" }, + { text: "Daemon", link: "/guide/daemon" }, + { text: "Video & Streaming", link: "/guide/video" }, { text: "LAN Access", link: "/guide/lan-access" }, - { text: "Project Daemon", link: "/guide/daemon" }, { text: "Testing", link: "/guide/testing" }, { text: "GitHub Actions", link: "/guide/github-actions" }, ], }, { - text: "Operating SimDeck", + text: "Reference", items: [ { text: "Troubleshooting", link: "/guide/troubleshooting" }, + { text: "How It Works", link: "/guide/architecture" }, { text: "Contributing", link: "/contributing" }, ], }, @@ -95,23 +89,18 @@ export default defineConfig({ text: "CLI", items: [ { text: "Overview", link: "/cli/" }, - { text: "Command Reference", link: "/cli/commands" }, - { text: "Flags & Options", link: "/cli/flags" }, + { text: "Commands", link: "/cli/commands" }, + { text: "Flags", link: "/cli/flags" }, ], }, ], "/api/": [ { - text: "HTTP API", + text: "API", items: [ - { text: "REST Endpoints", link: "/api/rest" }, + { text: "REST", link: "/api/rest" }, { text: "Health & Metrics", link: "/api/health" }, - ], - }, - { - text: "Inspectors", - items: [ { text: "Inspector Protocol", link: "/api/inspector-protocol" }, ], }, @@ -119,26 +108,14 @@ export default defineConfig({ "/inspector/": [ { - text: "Inspector", + text: "Inspectors", items: [ { text: "Overview", link: "/inspector/" }, { text: "Accessibility", link: "/inspector/accessibility" }, - { - text: "Swift In-App Agent", - link: "/inspector/swift", - }, - { - text: "NativeScript Runtime", - link: "/inspector/nativescript", - }, - { - text: "React Native Runtime", - link: "/inspector/react-native", - }, - { - text: "Flutter Runtime", - link: "/inspector/flutter", - }, + { text: "Swift", link: "/inspector/swift" }, + { text: "NativeScript", link: "/inspector/nativescript" }, + { text: "React Native", link: "/inspector/react-native" }, + { text: "Flutter", link: "/inspector/flutter" }, ], }, ], @@ -167,7 +144,7 @@ export default defineConfig({ footer: { message: "Released under the Apache-2.0 License.", - copyright: `Copyright © 2026 SimDeck contributors.`, + copyright: "Copyright (c) 2026 SimDeck contributors.", }, outline: { diff --git a/docs/api/health.md b/docs/api/health.md index d0dfb7e6..079bcb25 100644 --- a/docs/api/health.md +++ b/docs/api/health.md @@ -1,10 +1,14 @@ # Health & Metrics -Two endpoints expose every observable signal SimDeck collects: `GET /api/health` for the bootstrap surface and `GET /api/metrics` for the running counters. +Use these endpoints to check whether a daemon is reachable and to diagnose stream performance. -## `GET /api/health` +## Health -Returns the static bootstrap information the browser client needs, plus a freshness timestamp. +```http +GET /api/health +``` + +Example: ```json { @@ -13,6 +17,11 @@ Returns the static bootstrap information the browser client needs, plus a freshn "timestamp": 1714094761.234, "videoCodec": "auto", "lowLatency": false, + "realtimeStream": true, + "localStreamFps": 60, + "streamQuality": { + "profile": "full" + }, "webRtc": { "iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }], "iceTransportPolicy": "all" @@ -20,118 +29,42 @@ Returns the static bootstrap information the browser client needs, plus a freshn } ``` -| Field | Notes | -| --------------------------- | ----------------------------------------------------------------------------------------------------- | -| `ok` | Always `true` if the route is reachable. Network failures are signalled by HTTP errors. | -| `httpPort` | HTTP port for the REST API, browser UI, and WebRTC offer endpoint. | -| `timestamp` | Server-side `time.now()` as a fractional Unix epoch in seconds. | -| `videoCodec` | Requested encoder mode. One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). | -| `lowLatency` | `true` when software H.264 low-latency mode was enabled at daemon startup. | -| `realtimeStream` | `true` when the WebRTC stream is configured to favor freshness and realtime pacing. | -| `localStreamFps` | Local quality stream frame target, from 15 to 240 fps. Defaults to 60. | -| `streamQuality` | Active realtime quality profile and encoder limits such as `maxEdge`, `fps`, and bitrate. | -| `webRtc.iceServers` | ICE servers the browser should use when creating the WebRTC peer connection. | -| `webRtc.iceTransportPolicy` | Browser ICE transport policy. One of `all` or `relay`. | +Important fields: -The default access token is regenerated every time the server restarts. A client should refetch `/api/health` after any disconnection. +| Field | Meaning | +| --------------- | ------------------------------------------------------- | +| `ok` | Server is alive | +| `httpPort` | Port serving UI and API | +| `videoCodec` | Requested codec mode: `auto`, `hardware`, or `software` | +| `streamQuality` | Active stream profile and limits | +| `webRtc` | ICE settings the browser should use | -## `GET /api/metrics` +## Metrics -Returns a snapshot of every server-side counter and the rolling buffer of client-reported stats: - -```json -{ - "frames_encoded": 12039, - "keyframes_encoded": 17, - "frames_sent": 11982, - "frames_dropped_server": 21, - "keyframe_requests": 4, - "active_streams": 1, - "subscribers_connected": 3, - "subscribers_disconnected": 2, - "avg_send_queue_depth": 0.91, - "max_send_queue_depth": 2, - "latest_first_frame_ms": 412, - "encoders": [ - { - "udid": "9D7E5BB7-...", - "encoder": { - "encoderMode": "auto", - "activeEncoderMode": "hardware", - "clientForeground": true, - "autoSoftwareFallbackActive": false, - "hardwareAccelerated": true, - "overloadState": "nominal", - "averageEncoderLoadPercent": 42.1, - "averageEncodeLatencyUs": 7021.0, - "encoderBudgetUs": 16667, - "overloadEvents": 0 - } - } - ], - "client_streams": [ - { - "clientId": "browser-ABC", - "kind": "viewport", - "udid": "9D7E5BB7-...", - "timestampMs": 1714094761234.0, - "codec": "h264", - "width": 1170, - "height": 2532, - "decodedFps": 59.7, - "renderedFps": 59.7, - "droppedFps": 0.0, - "latestRenderMs": 6.2 - } - ] -} +```http +GET /api/metrics ``` -### Counter glossary - -| Counter | Increments when… | -| -------------------------- | --------------------------------------------------------------------------------------- | -| `frames_encoded` | The native bridge produces a frame. | -| `keyframes_encoded` | The encoder emits a keyframe (always a subset of `frames_encoded`). | -| `frames_sent` | A frame is written to a WebRTC client. | -| `frames_dropped_server` | A client is too slow and the broadcast channel skips frames for them. | -| `keyframe_requests` | The transport hub asks the encoder for a fresh keyframe (e.g. on reconnect or refresh). | -| `active_streams` | Currently open WebRTC streams. | -| `subscribers_connected` | Lifetime count of WebRTC streams opened. | -| `subscribers_disconnected` | Lifetime count of WebRTC streams closed. | -| `avg_send_queue_depth` | Running average of broadcast channel pressure. | -| `max_send_queue_depth` | Peak broadcast channel pressure. | -| `latest_first_frame_ms` | First-frame latency for the most recent connect, in milliseconds. | - -### Encoder overload state - -`encoders` contains one entry per active simulator session. `encoder.overloadState` -is derived from native VideoToolbox submit-to-output latency: +Useful fields: -| State | Meaning | -| ------------ | --------------------------------------------------------------------------------------- | -| `nominal` | Smoothed encode latency is comfortably below the active frame budget. | -| `strained` | Smoothed latency is near the frame budget or several frames are close to budget. | -| `overloaded` | Smoothed latency exceeds the budget or several frames in a row took longer than budget. | +| Field | What to look for | +| ---------------------------------- | -------------------------------------------- | +| `latest_first_frame_ms` | First-frame startup time | +| `frames_dropped_server` | Server dropping stale frames to stay current | +| `keyframe_requests` | Stream refresh or recovery activity | +| `active_streams` | Open browser streams | +| `encoders[].encoder.overloadState` | `nominal`, `strained`, or `overloaded` | +| `client_streams` | Recent browser decoder and render reports | -This is an inferred pressure signal rather than a private macOS hardware queue -counter. It is useful for deciding when to lower stream resolution/FPS or switch -from hardware to software encoding. When `encoderMode` is `auto`, SimDeck uses -this signal to temporarily rebuild the active session with software H.264 if the -hardware encoder is overloaded, then retries hardware after a cooldown. The -`activeEncoderMode`, `autoSoftwareFallbackActive`, `autoSoftwareFallbacks`, and -`autoHardwareRetries` fields expose that state. `clientForeground` is `false` -when all current browser viewers for the simulator are hidden or unfocused; in -that state the native session uses software H.264 until a viewer returns -foreground. +If `overloadState` is `overloaded` or dropped frames keep increasing, lower stream quality or restart with software encoding: -### Client stream stats - -`client_streams` is a rolling buffer of the most recent reports from browser stream transports. WebRTC clients normally send reports over the telemetry data channel, H.264 WebSocket clients send them on the stream socket, and simple clients can still post to `POST /api/client-stream-stats`. The server keeps the last 48 entries per `(clientId, kind)` pair. +```sh +simdeck daemon restart --video-codec software --stream-quality low +``` -The browser client uses these to render its in-app diagnostics overlay and to size its decoder workers. Every field is optional except `clientId` and `kind`; see [`ClientStreamStats`](https://github.com/NativeScript/SimDeck/blob/main/server/src/metrics/counters.rs) for the full schema. +## Submit Client Stats -## Submitting client stats +Custom clients can report their own stream stats: ```http POST /api/client-stream-stats @@ -142,31 +75,16 @@ Content-Type: application/json "kind": "viewport", "udid": "9D7E5BB7-...", "codec": "h264", - "width": 1170, - "height": 2532, "decodedFps": 59.7, "droppedFps": 0.0, "latestRenderMs": 6.2 } ``` -Required fields: - -- `clientId` — any stable identifier you pick. -- `kind` — what slice of the client is reporting (`viewport`, `decoder`, `renderer`, …). +Required fields are `clientId` and `kind`. -Anything else is optional but typed; unknown fields are rejected. +Read only the client buffer: -A successful submission returns: - -```json -{ "ok": true } -``` - -## `GET /api/client-stream-stats` - -Returns the same `clientStreams` array that `GET /api/metrics` includes, in case you only want the client-side view: - -```json -{ "clientStreams": [ ... ] } +```http +GET /api/client-stream-stats ``` diff --git a/docs/api/inspector-protocol.md b/docs/api/inspector-protocol.md index 60bc8fb8..9c655517 100644 --- a/docs/api/inspector-protocol.md +++ b/docs/api/inspector-protocol.md @@ -1,60 +1,45 @@ # Inspector Protocol -In-app inspectors talk to SimDeck using a small newline-delimited JSON protocol called `SDI/0.1`. Both transports (TCP and WebSocket) speak the same envelope and method set, so app-side code is interchangeable between them. +In-app inspectors use SimDeck's small JSON protocol to publish richer UI trees and handle debug actions. -## Transports +Most users do not need this page. Use it when you are building or debugging an inspector runtime. -There are two equivalent transports: +## Transports -### Newline-delimited TCP +| Transport | Used by | +| ------------------------------------------------------------ | ---------------------------------------------------------------- | +| TCP on `127.0.0.1:47370-47402` | Swift in-app agent | +| WebSocket `/api/inspector/connect` | NativeScript, React Native, Flutter, and other outbound runtimes | +| Polling `/api/inspector/poll` plus `/api/inspector/response` | Fallback when WebSocket is unavailable | -The original Swift in-app agent listens on TCP. The default port is `47370`; if it is busy the agent tries the next 32 ports and listens on the first free one. Clients should probe `47370–47402` and call `Inspector.getInfo` to disambiguate. +TCP is newline-delimited JSON: ```sh printf '{"id":1,"method":"Inspector.getInfo"}\n' | nc 127.0.0.1 47370 ``` -The agent also advertises Bonjour service type `_simdeckinspector._tcp`. - -### WebSocket via the server - -NativeScript apps connect outbound to the SimDeck server: - -```text -GET /api/inspector/connect -``` - -After connection the server sends `Inspector.getInfo` and waits for a response that includes a `processIdentifier`. Once that arrives, the server treats the WebSocket as the preferred transport for that PID and routes inspector requests there. - -While the inspector is registered, the daemon advertises it in `~/.simdeck/inspectors.json` so other local SimDeck daemons can find the owning daemon and relay requests to the same app inspector. - -A polling fallback is available for environments without WebSocket support: - -- `GET /api/inspector/poll?processIdentifier=` — long-polls for the next request, returning `204 No Content` if nothing arrives within 25 seconds. -- `POST /api/inspector/response` — submits the response body. - -## Envelopes +## Envelope -### Request +Request: ```json { "id": 1, "method": "View.getHierarchy", - "params": { "includeHidden": false } + "params": { "maxDepth": 4 } } ``` -### Response +Success: ```json { "id": 1, - "result": { "protocolVersion": "0.1", "roots": [] } + "result": { "roots": [] } } ``` -### Error +Error: ```json { @@ -66,65 +51,50 @@ A polling fallback is available for environments without WebSocket support: } ``` -### Event +Event: ```json { "event": "Inspector.connected", - "params": { "protocolVersion": "0.1", "framing": "ndjson" } + "params": { "protocolVersion": "0.1" } } ``` -If the agent is started with an `authToken`, every request must include a matching top-level `token` field. +If an inspector is started with an auth token, requests must include a matching top-level `token`. -All point input and `frameInScreen` values use UIKit screen points, **not pixels**. Multiply by `displayScale` from `Inspector.getInfo` to convert to native pixels. +## Core Methods -## Methods +| Method | Purpose | +| ---------------------- | ------------------------------------------------------ | +| `Runtime.ping` | Connectivity check | +| `Inspector.getInfo` | Protocol version, app metadata, display scale, sources | +| `View.getHierarchy` | Current UI hierarchy | +| `View.get` | One subtree by ID | +| `View.hitTest` | Topmost view at a point | +| `View.describeAtPoint` | Hit view plus ancestors | +| `View.listActions` | Supported actions for a view | +| `View.perform` | Run an action such as `tap`, `setText`, or `scrollBy` | +| `View.getProperties` | Editable debug properties | +| `View.setProperty` | Best-effort runtime property edit | +| `View.evaluateScript` | Debug script evaluation | -The full method list. The SimDeck HTTP proxy at `POST /api/simulators/{udid}/inspector/request` only allows a curated subset (see [REST endpoints](/api/rest#post-api-simulators-udid-inspector-request)); direct TCP/WebSocket clients can call any of them. +Coordinates and frames use UIKit screen points, not pixels. Multiply by `displayScale` when you need pixels. -### `Runtime.ping` - -Quick connectivity check. Returns: - -```json -{ "result": { "ok": true } } -``` - -### `Inspector.getInfo` - -Returns protocol version, app process metadata, display scale, coordinate space, and the available method list. Required first call after connect. +## Hierarchy Request ```json { - "result": { - "protocolVersion": "0.1", - "processIdentifier": 73214, - "bundleIdentifier": "com.example.MyApp", - "bundleName": "MyApp", - "displayScale": 3, - "coordinateSpace": "uikit-screen-points", - "appHierarchy": { - "available": true, - "source": "nativescript" - } + "id": 2, + "method": "View.getHierarchy", + "params": { + "includeHidden": false, + "maxDepth": 20, + "source": "uikit" } } ``` -### `View.getHierarchy` - -Returns the current hierarchy rooted at every visible window. - -Params: - -```json -{ "includeHidden": false, "maxDepth": 20, "source": "uikit" } -``` - -By default the agent returns the published framework hierarchy (for example NativeScript, React Native, Flutter, or SwiftUI) when one exists. Pass `"source": "uikit"` to force the raw UIKit tree when the runtime supports UIKit inspection. - -Published framework nodes may include `sourceLocation`: +Framework runtimes may return logical nodes with source locations: ```json { @@ -133,124 +103,29 @@ Published framework nodes may include `sourceLocation`: "sourceLocation": { "file": "src/app/home.component.html", "line": 12, - "column": 5, - "offset": 238 + "column": 5 } } ``` -`line` and `column` are one-based when produced by NativeScript or React Native development metadata. - -### `View.get` - -Returns one view subtree by id: - -```json -{ - "id": 4, - "method": "View.get", - "params": { "id": "view:0x1234", "maxDepth": 2 } -} -``` - -IDs are process-local and valid until the underlying object is destroyed. - -### `View.hitTest` - -Returns the topmost hit-tested view for a screen point: - -```json -{ - "id": 5, - "method": "View.hitTest", - "params": { "x": 120, "y": 240, "maxDepth": 1 } -} -``` - -### `View.describeAtPoint` - -Returns the hit view plus its ancestor chain. - -```json -{ "id": 6, "method": "View.describeAtPoint", "params": { "x": 120, "y": 240 } } -``` - -### `View.listActions` - -Lists the safe interactions a view supports. - -```json -{ "id": 7, "method": "View.listActions", "params": { "id": "view:0x1234" } } -``` - -### `View.perform` - -Performs a high-level action on a view. - -Supported actions: `tap`, `focus`, `resignFirstResponder`, `accessibilityActivate`, `setText`, `setValue`, `toggle`, `scrollBy`, `scrollTo`. - -Examples: - -```json -{ - "id": 8, - "method": "View.perform", - "params": { "id": "view:0x1234", "action": "tap" } -} -``` - -```json -{ - "id": 9, - "method": "View.perform", - "params": { "id": "view:0x1234", "action": "setText", "value": "hello" } -} -``` +## Actions ```json { - "id": 10, + "id": 3, "method": "View.perform", "params": { "id": "view:0x1234", - "action": "scrollBy", - "y": 400, - "animated": true + "action": "tap" } } ``` -### `View.getProperties` - -Returns editable runtime properties for a view: - -```json -{ "id": 11, "method": "View.getProperties", "params": { "id": "view:0x1234" } } -``` +Common actions include `tap`, `focus`, `resignFirstResponder`, `setText`, `setValue`, `toggle`, `scrollBy`, and `scrollTo`. Support depends on the inspector runtime and selected view. -### `View.setProperty` +## SwiftUI Publishing -Sets a UIKit property dynamically. This is a debug-only escape hatch; agents reject unsafe property names and coerce structured UIKit values such as `UIColor`, `CGRect`, `CGPoint`, `CGSize`, and `UIEdgeInsets`. - -```json -{ - "id": 12, - "method": "View.setProperty", - "params": { - "id": "view:0x1234", - "property": "backgroundColor", - "value": { "$type": "UIColor", "hex": "#FF6600FF" } - } -} -``` - -### `View.evaluateScript` - -Evaluates a small UIKit script against a view. Used by the browser inspector to run pre-canned diagnostics. - -## SwiftUI - -For SwiftUI apps you control, attach the root publisher to the top of your scene: +Swift apps can publish a SwiftUI root tree: ```swift WindowGroup { @@ -259,23 +134,16 @@ WindowGroup { } ``` -The agent reflects the current SwiftUI value/body tree and publishes it as the `swiftui` hierarchy source. `View.getHierarchy` returns that tree by default; pass `"source": "uikit"` to inspect the backing hosting views instead. - -This is a debug aid built on Swift reflection. It can show the declared view/body structure, including custom subviews, containers, labels, modifier names, active conditional branches, and `ForEach` rows whose data and content builder are available through SwiftUI's public API. Private/custom containers may still be opaque when they do not expose a child view value or content builder. - -The agent also exposes SwiftUI in the raw UIKit tree: - -1. **Automatic detection.** UIKit bridge or hosting views whose runtime classes contain `SwiftUI` or `UIHosting` are reported with `swiftUI.isHost` or `swiftUI.isProbe` markers. -2. **Source-level tags.** Apps can tag SwiftUI views with `View.simDeckInspectorTag(_:id:metadata:)` from the Swift agent. Tagged views appear as lightweight probe `UIView`s with `swiftUI.isProbe = true`. +They can also tag specific SwiftUI views so the inspector can find them: ```swift Text("Continue") .simDeckInspectorTag("continue-label", id: "onboarding.continue.label") ``` -## Allowed proxy methods +## HTTP Proxy Allow List -When you call the inspector via `POST /api/simulators/{udid}/inspector/request`, the SimDeck server enforces an allow-list to keep the HTTP surface small: +The public proxy at `POST /api/simulators/{udid}/inspector/request` allows: - `Runtime.ping` - `Inspector.getInfo` @@ -287,4 +155,4 @@ When you call the inspector via `POST /api/simulators/{udid}/inspector/request`, - `View.listActions` - `View.perform` -For anything not in this list, talk directly to the inspector over TCP or WebSocket. +For other methods, talk directly to the inspector transport. diff --git a/docs/api/rest.md b/docs/api/rest.md index 5d16c27b..b1164d11 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -1,776 +1,201 @@ -# REST Endpoints +# REST API -The SimDeck server exposes one REST API over plain HTTP. Every route lives under `/api/`. Responses are JSON unless explicitly noted otherwise. Errors return a JSON body with `{"error": "..."}` and an appropriate HTTP status. +SimDeck serves the browser UI and API from the same HTTP server. Routes under `/api/*` return JSON unless noted. -The served browser UI receives the generated access token automatically through a strict same-site cookie. Direct API callers must send `X-SimDeck-Token: ` or `Authorization: Bearer `. +Use the CLI for most automation. Use the API when you are building a custom client, test harness, or editor integration. -## Conventions +## Authentication -- Method casing follows REST conventions. `GET` for queries, `POST` for state changes. -- Path parameters use `{name}` notation in this reference. UDIDs come from `GET /api/simulators` (or `simdeck list`). -- Most mutation endpoints return `{ "ok": true }`; boot and shutdown return refreshed simulator metadata. -- Timestamps are numeric unless a route documents otherwise. +Browser sessions loaded from the SimDeck server receive auth automatically. Direct callers should send the daemon token from `simdeck daemon status`: -## Health and metrics - -### `GET /api/health` - -Returns server health and the active video encoder mode. - -```json -{ - "ok": true, - "httpPort": 4310, - "timestamp": 1714094761.234, - "videoCodec": "auto", - "lowLatency": false, - "webRtc": { - "iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }], - "iceTransportPolicy": "all" - } -} +```text +X-SimDeck-Token: +Authorization: Bearer ``` -The browser client polls this endpoint at startup to detect server restarts and -to mirror the daemon's WebRTC ICE configuration. - -### `GET /api/metrics` - -Returns server-side video stats, active encoder overload states, and a rolling -buffer of client-side stats. See [Video Pipeline](/guide/video#tuning-with-metrics) -for an annotated example. - -### `GET /api/client-stream-stats` - -Returns just the client-side stats: - -```json -{ "clientStreams": [{ "clientId": "...", "kind": "viewport", ... }] } -``` - -### `POST /api/client-stream-stats` - -Submit a stats sample from a client. The server keeps the last 48 entries per `(clientId, kind)`: +LAN browsers can pair with the printed six-digit code through: ```http -POST /api/client-stream-stats -Content-Type: application/json - -{ - "clientId": "browser-ABC", - "kind": "viewport", - "codec": "h264", - "width": 1170, - "height": 2532, - "decodedFps": 59.7, - "droppedFps": 0.0, - "latestRenderMs": 6.2 -} +POST /api/pair ``` -Required fields: `clientId` and `kind`. Every other field is optional but typed in `ClientStreamStats`. - -### `GET /api/stream-quality` - -Returns the active stream encoder settings and available quality profiles. - -### `POST /api/stream-quality` +## Quick Examples -Updates the active stream encoder settings for newly encoded frames. Browser -clients normally send these updates on the active WebRTC data channel or H.264 -WebSocket; this endpoint remains for scripts and fallback clients. - -```json -{ - "videoCodec": "hardware", - "fps": 60, - "profile": "full" -} +```sh +curl -H "X-SimDeck-Token: $SIMDECK_TOKEN" \ + http://127.0.0.1:4310/api/simulators ``` -`videoCodec` accepts `hardware` or `software` from the UI, and the API also -accepts `auto`. `fps` is clamped to the local stream range. Browser viewers show -five H.264 resolution profiles: `full` (4096 px at 60 fps), `balanced` -(1280 px at 60 fps), `economy` (1080 px at 30 fps), `low` (720 px at 30 fps), -and `tiny` (540 px at 30 fps). The API still accepts the legacy `quality`, -`fast`, `smooth`, and `ci-software` profiles for CLI/provider compatibility. -When -`profile` is provided, its resolution preset is applied; send `maxEdge` without -`profile` for a custom resolution cap. - -## Simulator inventory - -### `GET /api/simulators` - -Returns every iOS Simulator known to the native bridge plus every Android AVD -found in the Android SDK, enriched with any session state SimDeck has attached: - -```json -{ - "simulators": [ - { - "udid": "9D7E5BB7-...", - "name": "iPhone 15 Pro", - "runtimeName": "iOS 18.0", - "deviceTypeIdentifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-15-Pro", - "isBooted": true, - "platform": "ios-simulator", - "privateDisplay": { - "displayReady": true, - "displayStatus": "running", - "displayWidth": 1170, - "displayHeight": 2532, - "frameSequence": 8124, - "rotationQuarterTurns": 0 - } - } - ] -} +```sh +curl -X POST \ + -H "Content-Type: application/json" \ + -H "X-SimDeck-Token: $SIMDECK_TOKEN" \ + -d '{"url":"https://example.com"}' \ + http://127.0.0.1:4310/api/simulators//open-url ``` -Android emulators use IDs prefixed with `android:` and include Android metadata: +## Server -```json -{ - "udid": "android:SimDeck_Pixel_8_API_36", - "name": "SimDeck_Pixel_8_API_36", - "platform": "android-emulator", - "runtimeName": "Android", - "deviceTypeName": "Android Emulator", - "isBooted": true, - "android": { - "avdName": "SimDeck_Pixel_8_API_36", - "serial": "emulator-5554", - "grpcPort": 8554 - } -} -``` +| Method | Path | Purpose | +| ------ | -------------------------- | ---------------------------------------------------------- | +| `GET` | `/api/health` | Server health, version-ish runtime settings, stream config | +| `GET` | `/api/metrics` | Video, encoder, and client stream counters | +| `GET` | `/api/client-stream-stats` | Recent client stream reports | +| `POST` | `/api/client-stream-stats` | Submit client stream stats | +| `GET` | `/api/stream-quality` | Current stream quality settings | +| `POST` | `/api/stream-quality` | Update stream quality settings | -For iOS, `privateDisplay` is `null` until a stream attaches. For Android, -SimDeck fills display size from `adb shell wm size` when the emulator is booted. +See [Health & Metrics](/api/health) for details. -## Simulator lifecycle +## Devices -### `POST /api/simulators/{udid}/boot` +| Method | Path | Purpose | +| ------ | ------------------------------------------ | ----------------------------------------- | +| `GET` | `/api/simulators` | List iOS Simulators and Android emulators | +| `GET` | `/api/simulators/{udid}/state` | Get one device state | +| `POST` | `/api/simulators/{udid}/boot` | Boot a simulator or emulator | +| `POST` | `/api/simulators/{udid}/shutdown` | Shut it down | +| `POST` | `/api/simulators/{udid}/erase` | Erase data and settings | +| `POST` | `/api/simulators/{udid}/toggle-appearance` | Toggle light/dark appearance | -Boots the simulator or Android emulator and returns the refreshed device metadata: +Device IDs come from `/api/simulators`. Android IDs use the `android:` prefix. -```json -{ "simulator": { ... } } -``` - -### `POST /api/simulators/{udid}/shutdown` +## Apps -Tears down the live session (if any) and shuts the simulator or emulator down. +| Method | Path | Body | +| ------ | ---------------------------------- | ----------------------------------- | +| `POST` | `/api/simulators/{udid}/install` | `{ "appPath": "/path/to/App.app" }` | +| `POST` | `/api/simulators/{udid}/uninstall` | `{ "bundleId": "com.example.App" }` | +| `POST` | `/api/simulators/{udid}/launch` | `{ "bundleId": "com.example.App" }` | +| `POST` | `/api/simulators/{udid}/open-url` | `{ "url": "https://example.com" }` | -### `POST /api/simulators/{udid}/toggle-appearance` +## Live Video -Toggles between light and dark appearance via `simctl ui appearance` on iOS or -`cmd uimode night` on Android. +| Method | Path | Purpose | +| ------ | ------------------------------------- | -------------------------------------- | +| `POST` | `/api/simulators/{udid}/webrtc/offer` | WebRTC offer/answer stream setup | +| `GET` | `/api/simulators/{udid}/h264` | H.264 WebSocket fallback | +| `GET` | `/api/simulators/{udid}/input` | Input WebSocket for fallback transport | +| `GET` | `/api/simulators/{udid}/control` | Alias for input control WebSocket | +| `POST` | `/api/simulators/{udid}/refresh` | Request a fresh frame or keyframe | -```json -{ "ok": true } -``` +For normal clients, copy the browser behavior instead of hand-coding a raw decoder. The UI supports WebRTC first and H.264 WebSocket fallback. -### `POST /api/simulators/{udid}/refresh` - -Forces the iOS encoder to emit a fresh frame. For Android IDs, this route is a -no-op that returns `{ "ok": true, "stream": "screenshot" }`; Android WebRTC -keyframe requests are handled through the WebRTC control channel and RTCP -feedback. - -```json -{ "ok": true } -``` - -### `POST /api/simulators/{udid}/webrtc/offer` - -WebRTC transport for browser-native live video. The browser sends an SDP offer -and the server responds with an SDP answer for a receive-only H.264 video track: +Minimal WebRTC request: ```json { - "sdp": "v=0\r\n...", + "type": "offer", + "sdp": "v=0...", "streamConfig": { + "profile": "balanced", "fps": 60, - "profile": "full", "videoCodec": "auto" - }, - "type": "offer" -} -``` - -```json -{ - "sdp": "v=0\r\n...", - "type": "answer" -} -``` - -For iOS, samples come from the native simulator display session and are sent as -an H.264 media track. Android loopback clients use the same endpoint and control -channels, but receive raw RGBA frames over the `simdeck-rgba` data channel. -Non-loopback Android clients receive VideoToolbox-encoded H.264. - -The browser also opens `simdeck-control` and `simdeck-telemetry` data channels. -In addition to input messages, clients can request a keyframe or tune the -stream attached to that peer: - -```json -{ "type": "streamControl", "forceKeyframe": true } -``` - -Clients can also report page focus/visibility through stream control. When all -current viewers for a simulator report `foreground: false`, the native session -uses software H.264 until a viewer returns foreground: - -```json -{ "type": "streamControl", "clientId": "browser", "foreground": false } -``` - -```json -{ "type": "streamQuality", "config": { "profile": "low", "fps": 30 } } -``` - -The telemetry channel accepts: - -```json -{ "type": "clientStats", "stats": { "clientId": "browser", "kind": "webrtc" } } -``` - -### `GET /api/simulators/{udid}/h264` - -Direct H.264 video over WebSocket for browsers that support WebCodecs but -cannot establish WebRTC media. The server sends binary messages with this -layout: - -| Offset | Size | Field | -| ------ | ---- | --------------------------------------------------- | -| 0 | 4 | Magic bytes `SDH1` | -| 4 | 1 | Version, currently `1` | -| 5 | 1 | Flags: bit 0 keyframe, bit 1 decoder config present | -| 6 | 2 | Header length, big-endian | -| 8 | 8 | Frame sequence, big-endian | -| 16 | 8 | Timestamp in microseconds, big-endian | -| 24 | 4 | Encoded width, big-endian | -| 28 | 4 | Encoded height, big-endian | -| 32 | 4 | Decoder config byte length, big-endian | -| 36 | 4 | H.264 sample byte length, big-endian | - -The optional decoder config bytes follow the header, then the encoded H.264 -sample bytes. Clients can pass initial stream settings as query parameters -(`profile`, `fps`, `videoCodec`) and can send text control messages on the same -socket: - -```json -{ "type": "streamControl", "forceKeyframe": true } -``` - -```json -{ "type": "streamControl", "clientId": "browser", "foreground": false } -``` - -```json -{ "type": "streamQuality", "config": { "profile": "low", "fps": 30 } } -``` - -```json -{ "type": "clientStats", "stats": { "clientId": "browser", "kind": "page" } } -``` - -Touch and keyboard input should use the separate `/api/simulators/{udid}/input` -WebSocket. The video socket is latest-frame oriented: clients should paint the -latest decoded frame locally and request a keyframe if the decoder loses sync, -rather than ACKing every rendered frame. - -### `POST /api/simulators/{udid}/open-url` - -Opens a URL inside the simulator: - -```http -POST /api/simulators/{udid}/open-url -Content-Type: application/json - -{ "url": "https://example.com" } -``` - -```json -{ "ok": true } -``` - -### `POST /api/simulators/{udid}/launch` - -Launches an installed app: - -```http -POST /api/simulators/{udid}/launch -Content-Type: application/json - -{ "bundleId": "com.apple.Preferences" } -``` - -```json -{ "ok": true } -``` - -### `GET /api/simulators/{udid}/screenshot.png` - -Returns a PNG screenshot for the selected device. iOS screenshots come from the -native simulator bridge; Android screenshots come from `adb exec-out screencap --p`. The browser client uses this endpoint for still-image diagnostics and -fallbacks. - -## Input - -### `POST /api/simulators/{udid}/touch` - -Replays a single touch event. For drags, send `began`, one or more `moved`, then `ended` (or `cancelled`). - -```http -POST /api/simulators/{udid}/touch -Content-Type: application/json - -{ "x": 240.0, "y": 480.0, "phase": "began" } -``` - -Allowed `phase` values: `began`, `moved`, `ended`, `cancelled`. - -### `POST /api/simulators/{udid}/touch-sequence` - -Replays multiple normalized touch events through one native input session: - -```http -POST /api/simulators/{udid}/touch-sequence -Content-Type: application/json - -{ - "events": [ - { "x": 0.5, "y": 0.7, "phase": "began", "delayMsAfter": 25 }, - { "x": 0.5, "y": 0.4, "phase": "moved", "delayMsAfter": 25 }, - { "x": 0.5, "y": 0.2, "phase": "ended" } - ] -} -``` - -This is the preferred API for agent gestures because it avoids one HTTP request -per touch phase. - -### `POST /api/simulators/{udid}/key` - -Replays a single keyboard event by HID key code: - -```http -POST /api/simulators/{udid}/key -Content-Type: application/json - -{ "keyCode": 4, "modifiers": 0 } -``` - -`keyCode` is the HID usage value. `modifiers` is a bitmask defined by the HID input subsystem (defaults to `0`). - -### `POST /api/simulators/{udid}/key-sequence` - -Replays multiple HID key codes through one native input session: - -```http -POST /api/simulators/{udid}/key-sequence -Content-Type: application/json - -{ "keyCodes": [11, 8, 15, 15, 18], "delayMs": 5 } -``` - -`delayMs` defaults to `0`. - -### `POST /api/simulators/{udid}/button` - -Presses a hardware button: - -```http -POST /api/simulators/{udid}/button -Content-Type: application/json - -{ "button": "lock", "durationMs": 50 } -``` - -Supported button names match the CLI and chrome controls: `home`, `lock`, -`power`, `side-button`, `volume-up`, `volume-down`, `action`, `mute`, -`digital-crown`, `left-side-button`, `app-switcher`, `siri`, and `apple-pay`. -`durationMs` defaults to `0` and is used for press-and-hold interactions. - -For live chrome interactions, send explicit button edges instead of a completed -press: - -```json -{ "button": "power", "phase": "down" } -``` - -`phase` accepts `down`, `up`, `began`, `ended`, and `cancelled`. Chrome controls -may also pass `usagePage` and `usage` from the device profile when an exact HID -usage is available. - -### `POST /api/simulators/{udid}/crown` - -Rotates the Apple Watch Digital Crown using scroll delta semantics: - -```http -POST /api/simulators/{udid}/crown -Content-Type: application/json - -{ "delta": 50 } -``` - -The browser UI sends this automatically when scrolling over a Watch screen or -crown chrome. - -### `POST /api/simulators/{udid}/home` - -Presses the home button: - -```json -{ "ok": true } -``` - -### `POST /api/simulators/{udid}/app-switcher` - -Invokes the app switcher as one server-side native action. - -### `POST /api/simulators/{udid}/rotate-left` - -Rotates the simulator 90° counter-clockwise. - -### `POST /api/simulators/{udid}/rotate-right` - -Rotates the simulator 90° clockwise. - -## Chrome rendering - -### `GET /api/simulators/{udid}/chrome-profile` - -Returns the bezel layout for the simulator: - -```json -{ - "totalWidth": 1240, - "totalHeight": 2602, - "screenX": 35, - "screenY": 35, - "screenWidth": 1170, - "screenHeight": 2532, - "cornerRadius": 220, - "buttons": [ - { - "name": "power", - "label": "Sleep/Wake", - "x": 1210, - "y": 420, - "width": 18, - "height": 112, - "anchor": "right", - "onTop": false, - "normalOffset": { "x": -2, "y": 420 }, - "rolloverOffset": { "x": -4, "y": 420 }, - "imageName": "SideButton", - "imageDownName": "SideButtonPressed" - } - ] -} -``` - -The browser client uses this to compose chrome around the live frame and to -render physical button sprites over or under the device body. - -### `GET /api/simulators/{udid}/chrome.png` - -Returns the rendered bezel as a PNG. Pass `?buttons=false` to omit physical -button sprites when the client renders them interactively. Cache headers are set -to `no-cache, no-store, must-revalidate` so changes (e.g. after a device -rotation) are picked up immediately. - -### `GET /api/simulators/{udid}/chrome-button/{button}.png` - -Returns a rendered physical button sprite. Pass `?pressed=true` for the -pressed-state sprite when the device profile exposes one. - -## Accessibility - -### `GET /api/simulators/{udid}/accessibility-tree` - -Returns the current accessibility tree. The server merges framework inspectors, the Swift in-app agent, and the native accessibility tree. Query parameters: - -| `source` | Behaviour | -| ---------------------------- | ------------------------------------------------------------------------------------------------------ | -| `auto` _(default)_ / unset | Use the most accurate source available, falling back to AX. | -| `nativescript` / `ns` | Force the NativeScript logical tree if a NativeScript inspector is connected for the foreground app. | -| `react-native` / `rn` | Force the React Native component tree if a React Native inspector is connected for the foreground app. | -| `flutter` / `fl` | Force the Flutter widget tree if a Flutter inspector is connected for the foreground app. | -| `swiftui` / `swift-ui` | Force the published SwiftUI logical tree if the Swift agent root publisher is installed in the app. | -| `uikit` / `in-app-inspector` | Force the raw UIKit hierarchy from the in-app inspector agent (NativeScript or Swift). | -| `native-ax` / `ax` | Always use the native accessibility snapshot. | -| `android-uiautomator` | Force the Android emulator UIAutomator hierarchy. | - -| Parameter | Default | Description | -| --------------- | ------- | ----------------------------------------------------------------------------------------------- | -| `maxDepth` | `80` | Limits returned descendants for in-app inspectors. Native AX responses are trimmed server-side. | -| `includeHidden` | `false` | Includes hidden in-app inspector views when supported by the connected inspector runtime. | - -The response always includes: - -```json -{ - "roots": [...], - "source": "nativescript|react-native|flutter|swiftui|in-app-inspector|native-ax", - "availableSources": ["nativescript", "react-native", "flutter", "swiftui", "in-app-inspector", "native-ax"], - "fallbackReason": "...", - "inspector": { ... } -} -``` - -`fallbackReason` is only present when the server could not honour the requested source. - -### `GET /api/simulators/{udid}/accessibility-point?x=...&y=...` - -Returns the AX-style accessibility description of the topmost element at a screen point. `x` and `y` are in UIKit screen points and must be finite, non-negative numbers. - -## Inspector proxy - -### `POST /api/simulators/{udid}/inspector/request` - -Proxies a single inspector method to the active in-app inspector (NativeScript or Swift) for the simulator. This is used by the browser client to fetch view properties, list available actions, and run debug-only edits. - -```http -POST /api/simulators/{udid}/inspector/request -Content-Type: application/json - -{ - "method": "View.getProperties", - "params": { "id": "view:0x1234" } -} -``` - -Allowed methods (the server enforces this allow-list): - -- `Inspector.getInfo` -- `Runtime.ping` -- `View.get` -- `View.evaluateScript` -- `View.getHierarchy` -- `View.getProperties` -- `View.setProperty` -- `View.listActions` -- `View.perform` - -The response includes both the inspector's `result` and metadata about the inspector that handled the request: - -```json -{ - "result": { "id": "view:0x1234", "properties": [...] }, - "inspector": { - "bundleIdentifier": "com.example.MyApp", - "bundleName": "MyApp", - "transport": "websocket", - "processIdentifier": 73214, - "daemonUrl": null, - "host": "127.0.0.1", - "port": null, - "displayScale": 3, - "protocolVersion": "0.1" } } ``` -For the full method semantics, see the [Inspector Protocol](/api/inspector-protocol). - -## DevTools inspectors - -SimDeck serves WebKit Remote Inspector targets and Chrome DevTools Protocol -targets separately at the API layer. The browser UI combines both sources into -one resizeable DevTools panel with a single target list. - -## WebKit inspector - -### `GET /api/simulators/{udid}/webkit/targets` - -Discovers inspectable WebKit targets exposed by the simulator's `webinspectord` -Remote Inspector socket. These are Safari pages and app web content that WebKit -has made inspectable. On iOS 16.4 and newer, app-owned `WKWebView` instances must -set `isInspectable = true` before they appear here. +Response: ```json { - "udid": "4889B81C-FD88-49A9-BC1D-2087E7C451A2", - "socketPath": "/private/var/tmp/.../com.apple.webinspectord_sim.socket", - "targets": [ - { - "id": "5049443a3731353731-1", - "appId": "PID:71571", - "appName": "Example", - "pageId": 1, - "title": "Example", - "url": "https://example.com/", - "kind": "app-web-content", - "inspectorUrl": "/webkit-inspector-ui/Main.html?ws=127.0.0.1:4310/api/simulators/{udid}/webkit/targets/5049443a3731353731-1/socket", - "webSocketUrl": "/api/simulators/{udid}/webkit/targets/5049443a3731353731-1/socket" - } - ], - "warnings": [] -} -``` - -`kind` is best-effort metadata: - -- `safari-page` for Mobile Safari targets. -- `app-web-content` for app-owned inspectable web content. -- `web-content-proxy` for WebKit proxy targets. - -This endpoint lists **inspectable WebKit targets**, not every `WKWebView` object -in UIKit. AX can still reveal visible web areas that are not inspectable. - -### `GET /api/simulators/{udid}/webkit/targets/{targetId}/socket` - -Upgrades to a WebSocket. The server bridges JSON Web Inspector frontend messages -to the simulator's binary-plist WebKit Remote Inspector protocol for the selected -target. The `inspectorUrl` returned by `webkit/targets` points WebInspectorUI at -this socket. - -### `GET /webkit-inspector-ui/Main.html` - -Serves the local macOS WebInspectorUI resources with a SimDeck browser host shim. -This is authenticated like the rest of the server routes. - -## Chrome DevTools inspector - -### `GET /api/simulators/{udid}/devtools/targets` - -Discovers app runtime inspector sessions that can be opened with the embedded -Chrome DevTools frontend. This includes SimDeck's in-app inspector protocol for -React Native and NativeScript runtimes, UIKit/SwiftUI fallback metadata when -those are the only connected logical sources, Metro React Native DevTools -targets, and generic local Chrome Inspector targets. - -```json -{ - "udid": "4889B81C-FD88-49A9-BC1D-2087E7C451A2", - "targets": [ - { - "id": "sdi-73214", - "title": "React Native: Example", - "type": "page", - "url": "simdeck://com.example.Example", - "description": "SimDeck React Native inspector target", - "devtoolsFrontendUrl": "/chrome-devtools-ui/inspector.html?ws=127.0.0.1:4310/api/simulators/{udid}/devtools/targets/sdi-73214/socket", - "webSocketDebuggerUrl": "ws://127.0.0.1:4310/api/simulators/{udid}/devtools/targets/sdi-73214/socket", - "source": "react-native", - "processIdentifier": 73214, - "bundleIdentifier": "com.example.Example", - "appName": "Example" - } - ], - "warnings": [] -} -``` - -Metro-discovered targets use `source: "react-native-metro"` and point at -`/chrome-devtools-ui/rn_fusebox.html` when the target supports the React Native -Fusebox frontend. SimDeck probes common Metro ports and matching local -Node/React Native listener ports instead of assuming one fixed port. Generic -Chrome Inspector targets use `source: "chrome-inspector"` and are discovered from -common Inspector ports such as `9222-9230` plus matching local Chrome/Node -listener ports. Proxied targets return a SimDeck `webSocketDebuggerUrl`; SimDeck -then connects to the upstream `/inspector/debug` or Chrome DevTools WebSocket. - -### `GET /api/simulators/{udid}/devtools/targets/{targetId}/socket` - -Upgrades to a WebSocket speaking enough of the Chrome DevTools Protocol for the -frontend to open a target, evaluate console expressions through -`View.evaluateScript`, and render the published logical hierarchy as DOM nodes. - -### `GET /chrome-devtools-ui/inspector.html` - -Serves the bundled Chrome DevTools frontend. SimDeck uses the React Native -debugger frontend package for these static assets and copies them into -`client/dist/chrome-devtools-ui` during the client build. - -## NativeScript inspector hub - -### `GET /api/inspector/connect` - -Upgrades to a WebSocket. Used by the [`@nativescript/simdeck-inspector`](/inspector/nativescript) runtime to register itself as an in-app inspector. - -After connection the server sends `Inspector.getInfo` and waits for a response that includes a `processIdentifier`. Once registered, the server uses this socket as the preferred transport for `accessibility-tree` and `inspector/request` calls that target the same process. - -Registered WebSocket and polled inspectors are advertised in `~/.simdeck/inspectors.json` with the owning daemon URL, access token, process id, and advertised hierarchy sources. Other SimDeck daemons read this registry, validate that the process belongs to the requested simulator, and relay inspector requests through the owning daemon. Entries are refreshed while the inspector is alive and expire automatically if the daemon or app exits. - -### `GET /api/inspector/poll?processIdentifier=...` - -Long-poll fallback for environments where the WebSocket transport is not viable. Returns the next pending request as JSON, or `204 No Content` after 25 seconds with no work. - -### `POST /api/inspector/request` - -Protected daemon-to-daemon relay endpoint. A daemon uses this after discovering a matching inspector in `~/.simdeck/inspectors.json`; app runtimes and browsers should use the WebSocket/poll endpoints or `POST /api/simulators/{udid}/inspector/request` instead. - -```http -POST /api/inspector/request -Content-Type: application/json -X-SimDeck-Token: - -{ - "processIdentifier": 73214, - "method": "Runtime.ping", - "params": null + "type": "answer", + "sdp": "v=0..." } ``` -### `POST /api/inspector/response` - -Posts a response to a previous polled request: - -```http -POST /api/inspector/response -Content-Type: application/json - -{ - "processIdentifier": 73214, - "id": 12, - "result": { "ok": true } -} -``` - -Pass `error` instead of `result` to deliver an error. - -## Logs - -### `GET /api/simulators/{udid}/logs` - -Returns recent simulator logs. Without `backfill=true`, the server tails the live `os_log` stream it has already started for the simulator. With `backfill=true`, the server runs a fresh `simctl spawn ... log show` over the requested window. - -| Query parameter | Default | Notes | -| --------------- | ------- | ----------------------------------------------------------------------------- | -| `backfill` | `false` | When `true`, fetch a one-shot history instead of streaming. | -| `seconds` | `30` | Backfill window in seconds. Clamped to `[1, 1800]`. | -| `limit` | `250` | Max entries to return. Clamped to `[1, 1000]`. | -| `levels` | _none_ | Comma-separated list of log levels to keep (`debug,info,notice,error,fault`). | -| `processes` | _none_ | Comma-separated list of process names (case-insensitive substring matches). | -| `q` | _none_ | Free-text filter applied to the rendered log message. | +## Input -```json -{ - "entries": [ - { - "timestamp": "2026-04-23T19:14:12.123Z", - "level": "info", - "process": "MyApp", - "subsystem": "com.example.MyApp", - "category": "ui", - "pid": 73214, - "message": "Loaded 12 items" - } - ] -} -``` +| Method | Path | Body | +| ------ | ----------------------------------------- | ------------------------------------------ | +| `POST` | `/api/simulators/{udid}/tap` | Selector or coordinate tap | +| `POST` | `/api/simulators/{udid}/touch` | `{ "x": 120, "y": 240, "phase": "began" }` | +| `POST` | `/api/simulators/{udid}/touch-sequence` | Multiple touch phases | +| `POST` | `/api/simulators/{udid}/key` | `{ "keyCode": 4, "modifiers": 0 }` | +| `POST` | `/api/simulators/{udid}/key-sequence` | `{ "keyCodes": [11,8,15], "delayMs": 5 }` | +| `POST` | `/api/simulators/{udid}/button` | `{ "button": "lock", "durationMs": 50 }` | +| `POST` | `/api/simulators/{udid}/crown` | `{ "delta": 50 }` | +| `POST` | `/api/simulators/{udid}/dismiss-keyboard` | Dismiss the software keyboard | +| `POST` | `/api/simulators/{udid}/home` | Press Home | +| `POST` | `/api/simulators/{udid}/app-switcher` | Open app switcher | +| `POST` | `/api/simulators/{udid}/rotate-left` | Rotate left | +| `POST` | `/api/simulators/{udid}/rotate-right` | Rotate right | + +Touch coordinates are screen points unless the endpoint body explicitly uses normalized values. + +## UI State And Inspection + +| Method | Path | Purpose | +| ------ | -------------------------------------------------------- | ------------------------------- | +| `GET` | `/api/simulators/{udid}/accessibility-tree` | Current UI tree | +| `GET` | `/api/simulators/{udid}/accessibility-point?x=120&y=240` | Element at a point | +| `POST` | `/api/simulators/{udid}/query` | Query tree by selector | +| `POST` | `/api/simulators/{udid}/wait-for` | Wait until selector appears | +| `POST` | `/api/simulators/{udid}/assert` | Assert selector exists | +| `POST` | `/api/simulators/{udid}/batch` | Run multiple control steps | +| `POST` | `/api/simulators/{udid}/inspector/request` | Call an in-app inspector method | + +Tree query parameters: + +| Parameter | Values | +| --------------- | --------------------------------------------------------------------------------------------------------- | +| `source` | `auto`, `nativescript`, `react-native`, `flutter`, `swiftui`, `uikit`, `native-ax`, `android-uiautomator` | +| `maxDepth` | Integer depth limit | +| `includeHidden` | `true` or `false` | + +Every tree response reports the `source` used and may include a `fallbackReason`. + +## DevTools And WebKit + +| Method | Path | Purpose | +| ------ | ----------------------------------------------------------- | --------------------------------------------------- | +| `GET` | `/api/simulators/{udid}/webkit/targets` | Inspectable Safari or WKWebView targets | +| `GET` | `/api/simulators/{udid}/webkit/targets/{targetId}/socket` | WebKit inspector WebSocket | +| `GET` | `/webkit-inspector-ui/Main.html` | WebInspectorUI frontend | +| `GET` | `/api/simulators/{udid}/devtools/targets` | React Native, app runtime, Metro, or Chrome targets | +| `GET` | `/api/simulators/{udid}/devtools/targets/{targetId}/socket` | DevTools WebSocket | +| `GET` | `/chrome-devtools-ui/inspector.html` | Chrome DevTools frontend | + +For app-owned `WKWebView` on iOS 16.4 or newer, the app must set `isInspectable = true`. + +## Evidence And Chrome + +| Method | Path | Purpose | +| ------ | ----------------------------------------------- | ---------------------------------------------- | +| `GET` | `/api/simulators/{udid}/screenshot.png` | PNG screenshot | +| `GET` | `/api/simulators/{udid}/pasteboard` | Get pasteboard text | +| `POST` | `/api/simulators/{udid}/pasteboard` | Set pasteboard text with `{ "text": "hello" }` | +| `GET` | `/api/simulators/{udid}/logs` | Recent logs | +| `GET` | `/api/simulators/{udid}/chrome-profile` | Screen and chrome geometry | +| `GET` | `/api/simulators/{udid}/chrome.png` | Rendered device chrome PNG | +| `GET` | `/api/simulators/{udid}/chrome-button/{button}` | Rendered button sprite | +| `GET` | `/api/simulators/{udid}/screen-mask.png` | Rendered screen mask PNG | + +Log query parameters: + +| Parameter | Notes | +| -------------------- | -------------------------------------------- | +| `backfill=true` | Fetch recent history instead of current tail | +| `seconds=30` | Time window | +| `limit=250` | Max entries | +| `levels=error,fault` | Filter by level | +| `processes=MyApp` | Filter by process substring | +| `q=Loaded` | Message text filter | + +## Inspector Runtime Hub + +| Method | Path | Purpose | +| ------ | ------------------------------------------- | --------------------------------------- | +| `GET` | `/api/inspector/connect` | WebSocket for in-app runtime inspectors | +| `GET` | `/api/inspector/poll?processIdentifier=...` | Long-poll fallback | +| `POST` | `/api/inspector/request` | Protected daemon-to-daemon relay | +| `POST` | `/api/inspector/response` | Response for polled requests | + +Most clients should call `/api/simulators/{udid}/inspector/request` instead of these hub routes. ## Errors -Error bodies look like: +Errors are JSON: ```json { @@ -780,9 +205,10 @@ Error bodies look like: } ``` -| Status | Cause | -| ------ | ------------------------------------------------------------------------------------------- | -| `400` | Bad request body or query parameter (e.g. missing `url`, invalid `x`/`y`). | -| `404` | Unknown simulator. | -| `408` | Timed out waiting for a downstream component (encoder keyframe, AX, inspector). | -| `500` | Unhandled native bridge error. Always reported as JSON with the original message preserved. | +| Status | Common cause | +| ------ | ---------------------------------------------------- | +| `400` | Bad body or query parameter | +| `401` | Missing or invalid token | +| `404` | Unknown simulator, target, or asset | +| `408` | Timed out waiting for a device, stream, or inspector | +| `500` | Native bridge or server error | diff --git a/docs/cli/commands.md b/docs/cli/commands.md index d7136e12..984b56d7 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -1,174 +1,31 @@ -# Command Reference +# Commands -Every public subcommand exposed by `simdeck`. Replace `simdeck` with `./build/simdeck` when running from a local checkout. +Replace `simdeck` with `./build/simdeck` when running from a source checkout. ## UI And Daemon -### No Subcommand +| Command | Purpose | +| -------------------------------- | ------------------------------------------- | +| `simdeck` | Start a foreground browser session | +| `simdeck ` | Start and select a device | +| `simdeck -d` | Start or reuse the detached project daemon | +| `simdeck -k` | Stop the detached project daemon | +| `simdeck -r` | Restart the detached project daemon | +| `simdeck ui --open` | Open the browser UI from a daemon | +| `simdeck daemon status` | Show daemon URL, PID, token, and log path | +| `simdeck daemon stop` | Stop the current project daemon | +| `simdeck daemon killall` | Stop all project daemons | +| `simdeck service on/off/restart` | Manage the optional always-on macOS service | -Start a foreground workspace daemon and print clean local and LAN browser URLs: +Examples: ```sh -simdeck -simdeck "iPhone 17 Pro Max" -simdeck 9750DF52-0471-48FF-B49A-B184C4BD3A3D -simdeck --detached -simdeck --kill -simdeck --restart +simdeck ui --port 4320 --open +simdeck ui --open +simdeck daemon restart --video-codec software --stream-quality low ``` -The single optional argument is a simulator name or UDID to select by default in the UI. The foreground daemon binds to all interfaces, advertises the detected LAN address when available, prints the LAN HTTP URL plus a six-digit LAN pairing code, and stops when the command exits, when you press `q`, or when you press Ctrl-C. - -Shorthand lifecycle flags are available without a subcommand: - -- `-d`, `--detached` starts or reuses the background project daemon, like `simdeck daemon start`. -- `-k`, `--kill` stops the background project daemon and returns the killed PID. -- `-r`, `--restart` stops the background project daemon, starts a fresh one, and returns the same JSON shape as `daemon start`. - -### `ui` - -Start or reuse the project daemon and serve the browser UI. - -```sh -simdeck ui [--port 4310] [--bind 127.0.0.1] [--advertise-host ] - [--client-root ] [--video-codec auto|hardware|software] - [--low-latency] [--stream-quality ] - [--local-stream-fps <15-240>] [--open] -``` - -`--open` opens the authenticated local URL after the daemon is ready. - -### `studio expose` - -Expose one local simulator through SimDeck Studio: - -```sh -simdeck studio expose [simulator] [--studio-url https://simdeck.djdev.me] - [--port 4310] [--bind 127.0.0.1] - [--video-codec auto|hardware|software] - [--low-latency] [--stream-quality ] -``` - -Studio expose defaults to software H.264, realtime stream delivery, and the -`smooth` stream quality profile. The process prints the Studio URL plus the -active codec/profile, and keeps the outbound bridge alive until Ctrl-C. -`--video-codec hardware` opts back into the hardware encoder when that is preferable. - -### `provider` - -Run a self-hosted SimDeck Studio provider on an always-on Mac: - -```sh -simdeck provider connect --studio-url https://simdeck.djdev.me \ - --host-id --host-token -simdeck provider run [--max-capacity 1] [--simulator-template "iPhone 17 Pro"] -simdeck provider status -``` - -`provider connect` stores the Studio host credential in -`~/.simdeck/studio-provider.json` with user-only permissions. `provider run` -starts or reuses the local daemon, heartbeats to Studio, polls for allocation -jobs, clones simulators from the configured template, installs downloaded -artifacts, launches the configured bundle id, proxies Studio API requests to the -local daemon, and deletes session clones on release. - -### `daemon start` - -Start or reuse the project daemon without opening the browser: - -```sh -simdeck daemon start [--port 4310] [--bind 127.0.0.1] - [--advertise-host ] [--client-root ] - [--video-codec auto|hardware|software] [--low-latency] - [--stream-quality ] [--local-stream-fps <15-240>] -``` - -Output: - -```json -{ - "ok": true, - "projectRoot": "/path/to/app", - "pid": 12345, - "pairingCode": "123456", - "url": "http://127.0.0.1:4310", - "started": true -} -``` - -### `daemon status` - -Print daemon metadata for the current project: - -```sh -simdeck daemon status -``` - -Detached daemons report the supervisor PID and `logPath`; the supervised child -process is restarted automatically after recoverable server exits. - -### `daemon restart` - -Stop the daemon for the current project, then start a fresh one with the same -options as `daemon start`: - -```sh -simdeck daemon restart [--port 4310] [--bind 127.0.0.1] - [--advertise-host ] [--client-root ] - [--video-codec auto|hardware|software] [--low-latency] - [--stream-quality ] [--local-stream-fps <15-240>] -``` - -### `daemon stop` - -Stop the daemon for the current project: - -```sh -simdeck daemon stop -``` - -### `daemon killall` - -Stop SimDeck project daemons across all workspaces: - -```sh -simdeck daemon killall -``` - -### `service` - -Manage the optional always-on macOS user service. Use `simdeck daemon` for the -normal per-project process; use `simdeck service` when you want a LaunchAgent -that starts after login and stays available. - -```sh -simdeck service on [--port 4310] [--bind 127.0.0.1] - [--advertise-host ] [--client-root ] - [--video-codec auto|hardware|software] [--low-latency] - [--stream-quality ] [--local-stream-fps <15-240>] - [--access-token ] -simdeck service restart [same options as service on] -simdeck service off -``` - -`service on` installs `~/Library/LaunchAgents/dev.nativescript.simdeck.plist` -and starts a LaunchAgent that serves SimDeck after login. It is intended for -agents and editor integrations that should be able to open the UI without first -starting a project daemon. - -### `core-simulator` - -Manage Apple's CoreSimulator service layer: - -```sh -simdeck core-simulator restart -simdeck core-simulator start -simdeck core-simulator shutdown -``` - -Use this when `simctl` reports stale service state or simulator display attachment gets stuck before the first frame. - -## Simulator Lifecycle +## Device Lifecycle ```sh simdeck list @@ -177,12 +34,7 @@ simdeck shutdown simdeck erase ``` -`list` returns the same simulator inventory the browser UI renders, including -Android AVDs as IDs like `android:Pixel_8_API_36`. iOS lifecycle commands use -the native bridge. `boot` uses the private CoreSimulator path and does not fall -back to `xcrun simctl boot`; other iOS lifecycle commands still use `xcrun -simctl` where Apple exposes stable subcommands. Android lifecycle commands use -the Android SDK `emulator` and `adb` tools. +Android emulators appear as IDs such as `android:Pixel_8_API_36`. ## Apps And URLs @@ -195,9 +47,7 @@ simdeck open-url https://example.com simdeck toggle-appearance ``` -`launch` and `open-url` use the warm daemon fast path when available. - -## Inspect +## Inspect UI ```sh simdeck describe @@ -208,14 +58,10 @@ simdeck describe --source react-native simdeck describe --source flutter simdeck describe --source uikit simdeck describe --source native-ax -simdeck describe --source android-uiautomator simdeck describe --point 120,240 -simdeck describe --direct ``` -By default, `describe` uses the project daemon so it can prefer connected NativeScript, React Native, Flutter, or UIKit in-app inspectors, then fall back to the private CoreSimulator accessibility bridge. `--direct` skips the daemon and uses the native accessibility bridge directly. - -Use `--format agent` for compact hierarchy text intended for agent planning, and `--format compact-json` when a script needs parseable lower-token output. +Default source selection prefers a connected framework inspector, then the Swift in-app agent, then native accessibility. ## Input @@ -225,23 +71,25 @@ Coordinates are screen points unless `--normalized` is present. simdeck tap 120 240 simdeck tap 0.5 0.5 --normalized simdeck tap --label "Continue" --wait-timeout-ms 5000 -simdeck touch 0.5 0.5 --phase began --normalized -simdeck touch 120 240 --down --up --delay-ms 800 simdeck swipe 200 700 200 200 simdeck gesture scroll-down simdeck pinch --start-distance 160 --end-distance 80 simdeck rotate-gesture --radius 100 --degrees 90 +simdeck type "hello" +simdeck type --file message.txt simdeck key enter simdeck key-sequence --keycodes h,e,l,l,o simdeck key-combo --modifiers cmd --key a -simdeck type "hello" -simdeck type --file message.txt +``` + +System controls: + +```sh simdeck button lock --duration-ms 1000 simdeck button volume-up -simdeck button action --duration-ms 1000 +simdeck button action simdeck button digital-crown simdeck crown --delta 50 -simdeck button left-side-button simdeck dismiss-keyboard simdeck home simdeck app-switcher @@ -249,50 +97,53 @@ simdeck rotate-left simdeck rotate-right ``` -Use selectors (`--id`, `--label`, `--value`, `--element-type`) when possible. Use `--stdin` or `--file` for text containing quotes, newlines, or shell-sensitive characters. - ## Batch -Run a known sequence through one command: - ```sh simdeck batch \ --step "tap --label Continue --wait-timeout-ms 5000" \ --step "type 'hello world'" \ - --step "wait-for --label 'hello world' --timeout-ms 5000" \ - --step "gesture scroll-down" + --step "wait-for --label 'hello world' --timeout-ms 5000" ``` -Batch input can come from `--step`, `--file`, or `--stdin`. Use `wait-for` or `assert` with selector flags (`--id`, `--label`, `--value`, `--element-type`) to wait for UI state instead of fixed delays. `sleep 500` waits 500 ms; suffix seconds explicitly with `s`, as in `sleep 0.5s`. It fails fast by default; pass `--continue-on-error` for best-effort execution. +Use `wait-for` or `assert` steps instead of fixed sleeps when possible. ## Evidence ```sh simdeck screenshot --output screen.png simdeck screenshot --stdout > screen.png -simdeck stream --frames 120 > stream.h264 simdeck pasteboard set "hello" simdeck pasteboard get simdeck logs --seconds 30 --limit 200 simdeck chrome-profile ``` -`stream` writes Annex B H.264 samples to stdout and runs until interrupted, or -until `--frames` samples have been written. It is intended for diagnostics and -external tools, and is iOS-only. Android live viewing in the browser uses the -WebRTC H.264 endpoint; raw frames come from emulator gRPC and are encoded -through VideoToolbox. +Diagnostic iOS H.264 stream: + +```sh +simdeck stream --frames 120 > stream.h264 +``` + +## Studio And Providers -`logs` fetches recent simulator logs or Android `logcat` output. `chrome-profile` -returns the CoreSimulator chrome layout for iOS and a screen-sized profile for -Android. +For hosted Studio workflows: -## HTTP Fast Path +```sh +simdeck studio expose [simulator] +simdeck provider connect --studio-url --host-id --host-token +simdeck provider run +simdeck provider status +``` + +These commands are mainly for managed remote simulator hosts. -Supported hot controls use the daemon automatically. To target a specific daemon, set: +## CoreSimulator Service ```sh -export SIMDECK_SERVER_URL=http://127.0.0.1:4310 +simdeck core-simulator restart +simdeck core-simulator start +simdeck core-simulator shutdown ``` -This avoids repeated native setup in short-lived CLI processes. Commands that need local files, screenshots, pasteboard, or direct AX point queries still use the direct native path when appropriate. +Use this when Apple's simulator service is stale or unresponsive. diff --git a/docs/cli/flags.md b/docs/cli/flags.md index 784e7562..c7574066 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -1,96 +1,75 @@ -# Flags & Options +# Flags -A consolidated list of the public SimDeck CLI flags, grouped by command. - -::: tip Help output -Pass `--help` to any command to see the generated flag list from the binary: +Pass `--help` to any command for the generated flag list: ```sh -simdeck ui --help -simdeck daemon start --help +simdeck --help simdeck tap --help +simdeck daemon start --help ``` -::: - -## Global Flags - -### `--server-url ` - -| Default | unset | -| ------- | -------------------- | -| Env | `SIMDECK_SERVER_URL` | -| Type | `http://` URL | - -Targets a specific running SimDeck daemon for commands that support the HTTP fast path. If unset, commands start or reuse the current project's daemon when needed. +## Global -## `ui`, `daemon start`, And `daemon restart` +| Flag | Env | Purpose | +| -------------------- | -------------------- | -------------------------------- | +| `--server-url ` | `SIMDECK_SERVER_URL` | Target a specific running daemon | -`ui`, `daemon start`, and `daemon restart` accept the same server options. `ui` also accepts `--open`. +## Server Options -| Flag | Default | Description | -| -------------------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------------- | -| `--port ` | `4310` | HTTP port for the REST API, browser UI, and WebRTC offer endpoint. | -| `--bind ` | `127.0.0.1` | Bind address (`0.0.0.0` for [LAN access](/guide/lan-access), `::` for IPv6). | -| `--advertise-host` | matches local host | Hostname or IP printed for LAN browser access. | -| `--client-root` | bundled `client/dist` | Override the static browser client directory. | -| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video Pipeline](/guide/video). | -| `--low-latency` | `false` | Software H.264 profile for slower runners: caps at 15 fps and favors freshness. | -| `--stream-quality` | `full` | Realtime stream quality profile: `full`, `quality`, `balanced`, `fast`, `smooth`, `economy`, `low`, `tiny`, or `ci-software`. | -| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | -| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | +Used by `simdeck ui`, `daemon start`, `daemon restart`, `service on`, and `service restart`. -`studio expose` defaults to software H.264. Pass `--video-codec hardware` to -opt into the hardware encoder when that is preferable. - -The public commands generate an access token automatically. Use `simdeck daemon status` to read it for direct API callers. +| Flag | Default | Notes | +| ---------------------------- | -------------- | --------------------------------------------------------------------------------- | ------ | ------------ | +| `--port ` | `4310` | HTTP port | +| `--bind ` | `127.0.0.1` | Use `0.0.0.0` or `::` for LAN access | +| `--advertise-host ` | detected | Host printed for remote browsers | +| `--client-root ` | bundled client | Static client directory | +| `--video-codec auto | hardware | software` | `auto` | Encoder mode | +| `--stream-quality ` | `full` | `full`, `balanced`, `economy`, `low`, `tiny`, `ci-software`, and related profiles | +| `--local-stream-fps ` | `60` | Local stream frame target | +| `--low-latency` | off | Conservative software H.264 profile | +| `--open` | off | `ui` only | ## `describe` -| Flag | Default | Description | -| ------------------ | ------------------------------ | -------------------------------------------------------------------------------------------------------------------- | -| `--format` | `json` | Output format: `json`, `compact-json`, or `agent`. | -| `--source` | `auto` | Hierarchy source: `auto`, `nativescript`, `react-native`, `flutter`, `uikit`, `native-ax`, or `android-uiautomator`. | -| `--max-depth` | unlimited native / `80` daemon | Trim descendants after the requested depth. | -| `--include-hidden` | `false` | Include hidden in-app inspector views when supported. | -| `--direct` | `false` | Skip the daemon and use the private native accessibility bridge directly. | -| `--point ,` | unset | Return the native element at a screen point. | - -## Input Flags - -Common input commands: - -| Command | Important flags | -| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `tap` | `--id`, `--label`, `--value`, `--element-type`, `--wait-timeout-ms`, `--poll-interval-ms`, `--normalized`, `--duration-ms`, `--pre-delay-ms`, `--post-delay-ms` | -| `touch` | `--phase`, `--normalized`, `--down`, `--up`, `--delay-ms` | -| `swipe` | `--normalized`, `--duration-ms`, `--steps`, `--pre-delay-ms`, `--post-delay-ms` | -| `gesture` | `--screen-width`, `--screen-height`, `--normalized`, `--duration-ms`, `--delta`, `--pre-delay-ms`, `--post-delay-ms` | -| `pinch` | `--start-distance`, `--end-distance`, `--angle-degrees`, `--normalized`, `--duration-ms`, `--steps` | -| `rotate-gesture` | `--radius`, `--degrees`, `--normalized`, `--duration-ms`, `--steps` | -| `type` | `--stdin`, `--file`, `--delay-ms` | -| `key` | `--modifiers`, `--duration-ms`, `--pre-delay-ms`, `--post-delay-ms` | -| `key-sequence` | `--keycodes`, `--delay-ms` | -| `key-combo` | `--modifiers`, `--key` | -| `button` | `--duration-ms` | - -Coordinates are screen points unless `--normalized` is present. Normalized coordinates are clamped to `0.0..1.0`. - -## Evidence And Batch Flags +| Flag | Purpose | +| ------------------ | ------------------------------------------------- | ------------ | ------------------- | ----- | --------- | -------------------- | --------------------- | +| `--format json | compact-json | agent` | Choose output shape | +| `--source auto | nativescript | react-native | flutter | uikit | native-ax | android-uiautomator` | Pick inspector source | +| `--max-depth ` | Trim hierarchy depth | +| `--include-hidden` | Include hidden nodes when supported | +| `--point ,` | Describe the element at a screen point | +| `--direct` | Skip daemon and use native accessibility directly | + +## Input + +| Command | Useful flags | +| ---------------- | ---------------------------------------------------------------------------------------------------- | +| `tap` | `--id`, `--label`, `--value`, `--element-type`, `--wait-timeout-ms`, `--normalized`, `--duration-ms` | +| `touch` | `--phase`, `--normalized`, `--down`, `--up`, `--delay-ms` | +| `swipe` | `--normalized`, `--duration-ms`, `--steps` | +| `gesture` | `--normalized`, `--duration-ms`, `--delta` | +| `pinch` | `--start-distance`, `--end-distance`, `--angle-degrees`, `--normalized` | +| `rotate-gesture` | `--radius`, `--degrees`, `--normalized` | +| `type` | `--stdin`, `--file`, `--delay-ms` | +| `key` | `--modifiers`, `--duration-ms` | +| `key-sequence` | `--keycodes`, `--delay-ms` | +| `key-combo` | `--modifiers`, `--key` | +| `button` | `--duration-ms` | + +## Evidence And Batch | Command | Flags | | ---------------- | ---------------------------------------------------- | | `screenshot` | `--output `, `--stdout` | -| `logs` | `--seconds `, `--limit ` | +| `logs` | `--seconds `, `--limit ` | | `pasteboard set` | `--stdin`, `--file` | | `batch` | `--step`, `--file`, `--stdin`, `--continue-on-error` | ## Exit Codes -| Exit code | Meaning | -| --------- | -------------------------------------------------------------------------- | -| `0` | Success. | -| `1` | Command-level failure (bad usage, missing simulator, native bridge error). | -| `2` | Clap parser errors. | - -Errors print a short message to stderr; structured JSON is reserved for success output. +| Code | Meaning | +| ---- | -------------------------- | +| `0` | Success | +| `1` | Runtime or command failure | +| `2` | Argument parsing failure | diff --git a/docs/cli/index.md b/docs/cli/index.md index ca335ad5..e021b742 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -1,50 +1,58 @@ # CLI -The `simdeck` binary is the only entrypoint SimDeck ships. It opens the browser UI, manages a warm project daemon, and exposes simulator-control subcommands for scripts and tests. +`simdeck` is the main entrypoint for opening the browser UI, managing the daemon, and scripting simulator actions. -## Synopsis +## Common Use ```sh -simdeck [SIMULATOR_NAME_OR_UDID] -simdeck [-d|--detached] -simdeck [-k|--kill] -simdeck [-r|--restart] -simdeck [--server-url ] [OPTIONS] +simdeck +simdeck "iPhone 17 Pro Max" +simdeck -d +simdeck -k +simdeck -r ``` -With no subcommand, `simdeck` starts a foreground workspace daemon and prints local/LAN browser URLs. A single argument selects that simulator by name or UDID in the UI. Use `-d`, `-k`, and `-r` as short aliases for detached start, stop, and restart. - -Most commands automatically start or reuse the project daemon when that is the fastest path. Set `SIMDECK_SERVER_URL=http://127.0.0.1:4310` or pass `--server-url` to target a specific already-running daemon. +With no subcommand, SimDeck starts a foreground server and prints browser URLs. A single simulator name or UDID selects that device in the UI. The shorthand flags start, stop, and restart the detached project daemon. -## Top-level commands +## Command Shape -| Command | Purpose | -| ------------------------------------------------------- | ----------------------------------------------------------- | -| _(none)_ | Start a foreground UI daemon until `q` or Ctrl-C. | -| `ui` | Start or reuse the project daemon and serve the browser UI. | -| `daemon start/status/stop` | Manage the project daemon explicitly. | -| `core-simulator ...` | Restart or manage Apple's CoreSimulator service layer. | -| `list` | Print every simulator known to the native bridge as JSON. | -| `boot` / `shutdown` / `erase` | Manage simulator lifecycle. | -| `install` / `uninstall` / `launch` / `open-url` | Manage apps and URLs inside a simulator. | -| `describe` | Print accessibility or in-app inspector hierarchy data. | -| `tap` / `swipe` / `gesture` / `type` / `key` / `button` | Drive simulator input. | -| `screenshot` / `logs` / `pasteboard` / `chrome-profile` | Collect evidence and device metadata. | +```sh +simdeck [SIMULATOR_NAME_OR_UDID] +simdeck [--server-url ] [options] +``` -Every subcommand returns an exit code that follows shell conventions: zero on success, non-zero on failure. +Use `--server-url` or `SIMDECK_SERVER_URL` when a script should target a specific daemon: -## Output format +```sh +SIMDECK_SERVER_URL=http://127.0.0.1:4310 simdeck list +``` -Most subcommands print JSON. This makes the CLI easy to consume from scripts: +## Most-Used Commands ```sh -simdeck list | jq '.simulators[] | select(.isBooted)' +simdeck list +simdeck boot +simdeck install /path/to/App.app +simdeck launch com.example.App +simdeck open-url https://example.com +simdeck tap --label "Continue" --wait-timeout-ms 5000 +simdeck describe --format agent --max-depth 3 +simdeck screenshot --output screen.png +simdeck logs --seconds 30 --limit 200 ``` -Errors print a short human-readable message to stderr and a non-zero exit code. They do not print structured JSON for failure cases. +Most successful commands print JSON so they can be piped into tools such as `jq`. + +## Help + +```sh +simdeck --help +simdeck tap --help +simdeck daemon start --help +``` -## See also +## Next -- **[Command Reference](/cli/commands)** — every subcommand in detail. -- **[Flags & Options](/cli/flags)** — global flags and per-subcommand options. -- **[REST API](/api/rest)** — the HTTP equivalent of every CLI subcommand. +- [Commands](/cli/commands) +- [Flags](/cli/flags) +- [REST API](/api/rest) diff --git a/docs/contributing.md b/docs/contributing.md index b6ed66b0..89b4b900 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -1,126 +1,76 @@ # Contributing -SimDeck welcomes contributions. This page covers the toolchain, the layout, and the working rules to follow when proposing a change. +This page covers local setup, checks, and the boundaries to keep in mind when changing SimDeck. -## Toolchain +## Setup -You'll need: +Requirements: -- **macOS 13+** with the iOS Simulator runtimes installed. -- **Xcode command-line tools**: `xcode-select --install`. -- **Node.js ≥ 18** and npm. -- **Rust stable** via [rustup](https://rustup.rs/). - -Optional: - -- **`prettier`** for formatting (installed via `npm install`). -- **`cargo fmt`** and **`cargo clippy`** for Rust formatting and lints (ship with rustup). - -## First-time setup - -Clone, install dependencies, and build the CLI plus browser client: +- macOS with Xcode and iOS Simulator runtimes. +- Node.js 18 or newer. +- Rust stable. +- Optional Android SDK tools for Android emulator work. ```sh git clone https://github.com/NativeScript/SimDeck.git -cd simdeck +cd SimDeck npm install npm run build ``` -`npm install` installs JavaScript tooling only. `npm run build` rebuilds the Rust binary and React bundle. Use `npm run build:all` when you also need the NativeScript inspector, React Native inspector, and `simdeck/test` outputs. - -## Running locally - -```sh -npm run dev -``` - -This starts the Rust server in the background and runs the Vite dev server for the React client. The server log lands at `build/cli.log`. - -To run only the production server: +Run the built CLI: ```sh ./build/simdeck ``` -## Layout - -| Folder | What lives here | -| ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| `server/` | CLI entrypoint, project daemon, Rust HTTP server, stream transports, inspector hub, registry, and metrics. | -| `cli/` | Objective-C native bridge for private CoreSimulator and SimulatorKit APIs. | -| `client/` | React UI served at `/`. | -| `packages/nativescript-inspector/` | TypeScript runtime for the NativeScript inspector. | -| `packages/flutter-inspector/` | Flutter runtime plugin for publishing widget, render, and semantics hierarchy data. | -| `packages/inspector-agent/` | Swift Package for the Swift in-app inspector agent. | -| `packages/simdeck-test/` | JS/TS testing API for daemon-backed simulator automation. | -| `packages/vscode-extension/` | VS Code extension that opens the simulator inside an editor panel. | -| `scripts/` | Repeatable build entrypoints used by both local dev and CI. | -| `bin/` | Node launcher that locates and runs the compiled binary. | -| `docs/` | This documentation site (VitePress). | - -## Working rules - -If you contribute, keep these invariants in mind. They are also enforced by the `AGENTS.md` guide that lives at the repo root. - -- Simulator-native logic stays in Objective-C under `cli/`. -- Rust server logic stays under `server/`. -- Browser-only presentation logic stays in `client/`. -- NativeScript app runtime inspection logic stays in `packages/nativescript-inspector/`. -- React Native app runtime inspection logic stays in `packages/react-native-inspector/`. -- Flutter app runtime inspection logic stays in `packages/flutter-inspector/`. -- Prefer adding a server endpoint before adding client-only assumptions. -- Don't add a Node or Swift dependency to solve work that already fits in Foundation/AppKit. -- When touching private API usage, keep the adaptation small and explicit and document any simulator/runtime assumptions in `AGENTS.md`. -- Prefer stable CLI subcommands over hidden environment variables. -- The supported live video paths are the WebRTC H.264 offer endpoint plus the `/api/simulators/{udid}/h264` WebSocket fallback. Do not bring back legacy `/stream.h264` handling. -- If a feature depends on a booted simulator, fail with a clear JSON error instead of silently returning an empty asset. - -## Linting and formatting - -Format the entire repo: +Run the development server: ```sh -npm run format +npm run dev ``` -Check formatting in CI mode (no writes): +The server log is written to `build/cli.log`. -```sh -npm run format:check -``` +## Repository Map -Run all lints: - -```sh -npm run lint -``` +| Folder | Purpose | +| ----------- | ------------------------------------------------------ | +| `server/` | CLI entrypoint, daemon, API, stream transport, metrics | +| `cli/` | macOS simulator bridge | +| `client/` | Browser UI | +| `packages/` | Inspectors, VS Code extension, and `simdeck/test` | +| `scripts/` | Build, package, and test helpers | +| `docs/` | VitePress documentation | -This runs: +## Working Rules -- `prettier --check .` -- `cargo fmt --check` -- `cargo clippy --all-targets -- -D warnings` -- `tsc --noEmit` for the React client. +- Keep simulator-native logic in `cli/`. +- Keep server behavior in `server/`. +- Keep browser presentation in `client/`. +- Keep runtime inspector logic in the matching `packages/*-inspector` package. +- Prefer a stable CLI command or API route over hidden environment behavior. +- Update docs when changing CLI flags, API routes, stream behavior, or inspector methods. -## Tests +## Build And Check ```sh +npm run format +npm run lint npm run test +npm run ci ``` -This runs the Cargo test suite for the server and the Vitest suite for the client. +What `npm run ci` covers: -Build the JS/TS testing package with: +1. Formatting and lint checks. +2. Full build. +3. Rust and client tests. +4. VS Code extension package. -```sh -npm run build:simdeck-test -``` +## Integration Tests -The simulator-backed CLI integration suite is separate because it creates, -boots, drives, erases, and deletes a temporary iOS simulator. The suite also -builds and installs a tiny UIKit fixture app directly with `clang` so install, -uninstall, and opt-in launch checks use a deterministic local app bundle: +iOS: ```sh npm run build:cli @@ -128,78 +78,49 @@ npm run build:client npm run test:integration:cli ``` -For an interactive local run that opens Simulator.app and prints each CLI/HTTP -step with timings: +Verbose iOS run: ```sh npm run test:integration:cli:verbose ``` -The integration runner captures `describe` after each control step. If iOS -shows a known system URL-opening confirmation, the runner handles it and then -captures the UI again before continuing. - -Verbose mode prints CLI commands, command output, timings, and UI checkpoints. -Set `SIMDECK_INTEGRATION_TRACE_HTTP=1` if you also need raw HTTP request logs. - -Set `SIMDECK_INTEGRATION_KEEP_SIMULATOR=1` with the verbose command if you want -the temporary simulator left around for inspection after the suite exits. - -GitHub Actions runs the iOS suite on macOS after the normal build/test pipeline. -It also runs the Android integration suite on an Ubuntu runner with a real -Android emulator. Linux builds use a native iOS stub so the Android bridge, -daemon, CLI, and `simdeck/test` API can be exercised without macOS frameworks. -The Android CI job preboots the AVD with `android-emulator-runner` before -starting SimDeck; local Android integration runs skip cleanly unless an emulator -is already booted or `SIMDECK_INTEGRATION_BOOT_ANDROID=1` is set. -The integration suites do not require the live video display bridge; REST input -routes use the non-display input path, and the video stream is covered by -lower-level protocol tests. - -## Full CI pipeline +Android: ```sh -npm run ci +npm run build:cli +npm run build:simdeck-test +npm run test:integration:android ``` -This is the normal local CI script: - -1. `npm run lint` — formatting and lint checks. -2. `npm run build:all` — Rust + Objective-C, React client, NativeScript inspector, React Native inspector, and `simdeck/test`. -3. `npm run test` — Rust and TypeScript tests. -4. `npm run package:vscode-extension` — VS Code `.vsix`. - -GitHub Actions runs `npm run ci`, then `npm run test:integration:cli` for the -temp-simulator CLI and REST control sweep, plus -`npm run test:integration:android` on Ubuntu for Android emulator coverage. A -clean `npm run ci` and integration run are required for any PR that changes -simulator control behavior. +Useful variables: -## Documentation +| Variable | Purpose | +| ---------------------------------------- | ------------------------------- | +| `SIMDECK_INTEGRATION_VERBOSE=1` | Print more detail | +| `SIMDECK_INTEGRATION_SHOW_SIMULATOR=1` | Show Simulator.app during tests | +| `SIMDECK_INTEGRATION_KEEP_SIMULATOR=1` | Keep temporary iOS simulator | +| `SIMDECK_INTEGRATION_ANDROID_AVD=` | Pick an Android AVD | +| `SIMDECK_INTEGRATION_BOOT_ANDROID=1` | Boot Android locally | -This site is a VitePress project under `docs/`. To preview it: +## Docs ```sh npm run docs:dev -``` - -To build the static site: - -```sh npm run docs:build ``` -The build artefact lands at `docs/.vitepress/dist`. The docs deploy workflow (`.github/workflows/docs.yml`) publishes that directory to GitHub Pages on every push to `main`. +The docs site lives under `docs/` and deploys to GitHub Pages from `.github/workflows/docs.yml`. -When you change something in the repo that the docs already cover — a CLI flag, a route, a packet field, an inspector method — please update the matching docs page in the same PR. +## Issues And PRs -## Filing issues and PRs +For simulator or stream bugs, include: -- Open an issue for anything that requires discussion before code. -- For straightforward fixes, a PR is fine without a paired issue. -- Include reproduction steps and the macOS / Xcode version when filing simulator-related bugs. -- Include the server log (`build/cli.log` from `npm run dev`, or foreground daemon output from a reproduction) when filing video-stream bugs. +- Reproduction steps. +- `simdeck --version`. +- macOS and Xcode versions. +- Foreground daemon output or `build/cli.log`. +- Any relevant screenshots or CLI output. ## License -SimDeck is licensed under the Apache License 2.0. By contributing you agree to license your changes under the same terms. +SimDeck is licensed under Apache-2.0. Contributions are licensed under the same terms. diff --git a/docs/extensions/browser-client.md b/docs/extensions/browser-client.md index 04229894..16befb5e 100644 --- a/docs/extensions/browser-client.md +++ b/docs/extensions/browser-client.md @@ -1,104 +1,67 @@ # Browser Client -The default UI SimDeck serves at `/` is a React app built with Vite. It lives at `client/` in this repo and is bundled into `client/dist/` for production. The Rust server serves the bundle as static assets. +The browser client is the UI served by `simdeck` at `/`. It is the same surface used in normal browser sessions and inside the VS Code extension. -You almost certainly don't need to know about the internals — the client just works. This page is for contributors and for anyone who wants to embed the same surface somewhere else. +## Open It -## Tech stack - -- **React 19** — view layer. -- **TypeScript** — strict typing for everything in `client/src/`. -- **Vite** — dev server and production build. -- **Vitest** — unit tests. - -## Layout - -```text -client/ -├── index.html -├── vite.config.js -├── package.json -└── src/ - ├── main.tsx - ├── app/ - │ ├── App.tsx - │ └── AppShell.tsx - ├── api/ - │ ├── client.ts - │ ├── controls.ts - │ ├── simulators.ts - │ └── types.ts - ├── features/ - │ ├── simulators/ - │ ├── viewport/ - │ ├── stream/ - │ ├── input/ - │ ├── accessibility/ - │ └── toolbar/ - ├── shared/ - └── styles/ +```sh +simdeck ``` -| Folder | Responsibility | -| ------------------------- | ------------------------------------------------------------------------------------ | -| `api/` | Typed wrappers around the SimDeck REST API and shared TypeScript types. | -| `features/simulators/` | Sidebar list of simulators plus boot/shutdown affordances. | -| `features/viewport/` | Frame canvas, chrome compositing, hit testing. | -| `features/stream/` | WebRTC media/RGBA stream client, H.264 fallback, receiver stats, and frame plumbing. | -| `features/input/` | Touch / keyboard / hardware-button affordances. | -| `features/accessibility/` | Accessibility tree pane and source switcher. | -| `features/toolbar/` | Top toolbar (rotate, home, app switcher, dark mode toggle, refresh). | - -## Bootstrap flow +Then open the printed local URL. -1. The browser fetches `index.html` from the Rust server. -2. `main.tsx` mounts the React tree at `#root`. -3. `AppShell` calls `GET /api/health` to learn the active encoder mode. -4. The simulator sidebar fetches `GET /api/simulators` and renders the list. -5. Selecting a device posts an SDP offer to `/api/simulators//webrtc/offer`. -6. The browser renders the WebRTC stream. iOS uses an H.264 media track; Android loopback clients use a raw RGBA data channel, while non-loopback Android clients use VideoToolbox-encoded H.264. -7. Touch and key events round-trip through `POST /api/simulators//touch` and `/key`. +Detached flow: -## Dev workflow +```sh +simdeck ui --open +``` -The repo's `npm run dev` script runs the server and Vite together: +LAN flow: ```sh -npm run dev +simdeck ui --bind 0.0.0.0 --advertise-host 192.168.1.50 --open ``` -This: +## What The UI Provides -- Builds the Rust CLI if it isn't built. -- Stops any stale SimDeck server listening on `4310`. -- Starts the Rust server in the background, logging to `build/cli.log`. -- Runs `vite` from `client/` against the local server, with hot-module reload. +- Device list with boot and selection controls. +- Live video with stream quality controls. +- Pointer, keyboard, and hardware-button input. +- Rotation, home, app switcher, dark-mode toggle, and refresh actions. +- Accessibility and framework inspector panes. +- DevTools panel for supported WebKit, Metro, Chrome, and runtime targets. +- Stream diagnostics. -Vite serves on `http://127.0.0.1:5173` and proxies API calls to the Rust server on `4310`. +## Stream URL Options -## Tests and types +Force a stream transport while debugging: -```sh -npm run --prefix client typecheck -npm run --prefix client test +```text +http://127.0.0.1:4310?stream=webrtc +http://127.0.0.1:4310?stream=h264 ``` -`typecheck` runs `tsc --noEmit` against the strict client config. `test` runs the Vitest suite in `client/src/`. +Use the default URL for normal operation. -## Replacing the client +## Serve A Custom Client -The daemon takes a `--client-root ` flag. You can ship a completely different UI by pointing it at a directory of static files: +Point the daemon at another static bundle: ```sh -simdeck ui --port 4310 --client-root /path/to/your/dist --open +simdeck ui --client-root /path/to/dist --open ``` -As long as your client speaks the documented [REST API](/api/rest) and WebRTC offer endpoint, it will work end to end. +Your client should use the [REST API](/api/rest), WebRTC offer endpoint, and control WebSocket documented in the API reference. + +## Develop The Built-In Client -## Embedding in another app +```sh +npm run dev +``` -The browser client is designed to live inside any container that can host a webview. The bundled VS Code extension is one example; embedding it in an Electron app, a Tauri shell, or a custom dashboard works the same way: +This starts the local SimDeck server and the Vite dev server. Client checks: -1. Point the host at `http://:/`. -2. Allow the host to talk to the same HTTP origin exposed by the server. -3. For direct API calls outside the served origin, pass the SimDeck access token with `X-SimDeck-Token` or `Authorization: Bearer`. +```sh +npm run --prefix client typecheck +npm run --prefix client test +``` diff --git a/docs/extensions/vscode.md b/docs/extensions/vscode.md index 69e0cd75..2b915d23 100644 --- a/docs/extensions/vscode.md +++ b/docs/extensions/vscode.md @@ -1,76 +1,71 @@ # VS Code Extension -A bundled VS Code extension opens the SimDeck browser client inside an editor panel and can start or reuse the project daemon when needed. The extension lives in `packages/vscode-extension/` and ships pre-bundled with the npm package. +The VS Code extension opens the SimDeck browser UI in an editor panel and can start the project daemon for you. ## Install -The fastest path is to package the extension from this checkout and install it locally with the VS Code CLI: - -```sh -npm run package:vscode-extension -npm run install:vscode-extension -``` - -Short aliases are available too: +From a source checkout: ```sh npm run package:vscode npm run install:vscode ``` -This: +This builds and installs `build/vscode/simdeck-vscode.vsix`. -1. Builds a `.vsix` at `build/vscode/simdeck-vscode.vsix`. -2. Installs it via `code --install-extension build/vscode/simdeck-vscode.vsix --force`. +If the `code` command is missing, run **Shell Command: Install 'code' command in PATH** from the VS Code command palette. -If the `code` command isn't on your `PATH`, install it from VS Code via **Shell Command: Install 'code' command in PATH** in the command palette. +## Open SimDeck -## Use it - -After installing, open the command palette and run: +Run this command from the command palette: ```text SimDeck: Open Simulator View ``` -The extension opens a webview panel pointed at a SimDeck daemon URL. If the configured URL is not reachable, the extension runs `simdeck ui`, reads the returned daemon URL, and loads that URL in the webview. +The extension tries the configured server URL first. If it is not reachable and auto-start is enabled, it runs `simdeck ui` for the current workspace. -Two more commands round out the surface: +## Commands -- **SimDeck: Stop Project Daemon** — runs `simdeck daemon stop` for the current workspace. -- **SimDeck: Show Output** — opens the extension's output channel for debugging. +| Command | Purpose | +| ------------------------------ | ------------------------- | +| `SimDeck: Open Simulator View` | Open the webview panel | +| `SimDeck: Stop Project Daemon` | Run `simdeck daemon stop` | +| `SimDeck: Show Output` | Open extension logs | ## Settings -All settings live under the `simdeck.*` namespace in VS Code settings: - -| Setting | Default | Notes | -| ------------------------- | ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | -| `simdeck.serverUrl` | `http://127.0.0.1:4310` | Preferred URL to try first. If auto-start launches the daemon on a different port, the extension uses the URL returned by `simdeck ui`. | -| `simdeck.cliPath` | _empty_ | Optional explicit path to the `simdeck` CLI. Empty means: workspace `build/`, then `PATH`. | -| `simdeck.port` | `4310` | Preferred port passed to `simdeck ui` when auto-starting the project daemon. | -| `simdeck.bindAddress` | `127.0.0.1` | Bind address passed to `simdeck ui` when auto-starting the project daemon. | -| `simdeck.autoStartDaemon` | `true` | If `true`, the extension starts or reuses the project daemon when opening the simulator view if the preferred URL is not reachable. | -| `simdeck.autoStartServer` | `true` | Deprecated compatibility alias for `simdeck.autoStartDaemon`. | +| Setting | Default | Purpose | +| ------------------------- | ----------------------- | ---------------------------- | +| `simdeck.serverUrl` | `http://127.0.0.1:4310` | Preferred daemon URL | +| `simdeck.cliPath` | empty | Explicit path to the CLI | +| `simdeck.port` | `4310` | Port for auto-start | +| `simdeck.bindAddress` | `127.0.0.1` | Bind address for auto-start | +| `simdeck.autoStartDaemon` | `true` | Start the daemon when needed | -## Resolving the CLI +CLI resolution order: -When the extension needs to start or stop the project daemon it looks for the CLI in this order: +1. `simdeck.cliPath` +2. Workspace `build/simdeck` +3. `simdeck` on `PATH` -1. The explicit `simdeck.cliPath` setting. -2. The workspace's `build/simdeck` (handy when developing on this repo). -3. `simdeck` from `PATH`. +## Remote Or LAN Server -If none of those resolve, the extension surfaces an error in the output channel pointing you at this documentation. +Set `simdeck.serverUrl` to the remote SimDeck URL and disable auto-start: -## Talking to a remote server - -Set `simdeck.serverUrl` to any reachable SimDeck endpoint. The extension is purely a webview shell and doesn't ship its own version of the React client. Whatever the daemon serves is what you get. +```json +{ + "simdeck.serverUrl": "http://192.168.1.50:4310", + "simdeck.autoStartDaemon": false +} +``` -For [LAN-reachable daemons](/guide/lan-access), point the extension at `http://:` and disable `autoStartDaemon` so the extension does not start a local project daemon. +Pair in the webview if the server asks for the LAN code. ## Troubleshooting -- **The webview shows a blank panel.** Open the output channel; the extension logs the daemon URL returned by `simdeck ui` and any CLI stderr. -- **Auto-start doesn't work.** Ensure the resolved `cliPath` exists and is executable. The extension shows the resolved path in the output channel. -- **Stale daemon stays running.** Use **SimDeck: Stop Project Daemon** to run `simdeck daemon stop`, then reopen the simulator view. +| Symptom | Fix | +| ------------------------ | ------------------------------------------------------------------- | +| Blank panel | Open `SimDeck: Show Output` and check the daemon URL and CLI stderr | +| Auto-start fails | Set `simdeck.cliPath` to the real CLI path | +| Old server keeps loading | Run `SimDeck: Stop Project Daemon`, then reopen | diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index e81f6c2d..d2806a00 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -1,128 +1,60 @@ -# Architecture +# How It Works -SimDeck is intentionally split into a small number of clearly-scoped layers. Every layer has a single concern and a single owner directory in the repo. +This page is a short mental model for users and contributors. For daily usage, start with [Quick Start](/guide/quick-start) and [CLI commands](/cli/commands). -## High-level layout +## Pieces -SimDeck has three layers stacked between the browser and the target device: +| Piece | What it does | +| ------------------ | ------------------------------------------------------------------- | +| CLI | Starts SimDeck and exposes scriptable commands | +| Local daemon | Serves the browser UI, API, streams, metrics, and inspector routing | +| Browser client | Shows the live device, toolbar, inspector panes, and diagnostics | +| Native bridge | Handles simulator-specific work on macOS | +| Inspector runtimes | Optional app packages that publish richer UI trees | +| `simdeck/test` | JS/TS wrapper for automation | -1. **Browser / VS Code** runs the React client from `client/`. It speaks HTTP for control and WebRTC H.264 for live video, served by the Rust server. -2. **The Rust server** (`server/`, built on `axum` + `tokio`) owns the CLI entrypoint, project daemon lifecycle, REST routes (`api/`), the stream transports (`transport/`), the inspector WebSocket hub (`inspector.rs`), the per-UDID session registry (`simulators/`), metrics, and log streaming. -3. **Native device bridges** own platform-specific work. The Objective-C bridge (`cli/`) is reached through a narrow C ABI in `cli/native/XCWNativeBridge.*` for iOS. The Rust Android bridge (`server/src/android.rs`) shells out to the Android SDK for AVD discovery, emulator lifecycle, ADB input, screenshots, UIAutomator, and logcat. +## Request Flow -Underneath all of that are the iOS Simulator (`CoreSimulator` and `SimulatorKit`) and the Android emulator (`emulator` and `adb`). +Most user actions follow the same path: -## Layer responsibilities +1. Browser, CLI, or test sends a command to the daemon. +2. The daemon checks the selected device and starts a warm session when needed. +3. SimDeck performs the requested simulator or emulator action. +4. The command returns JSON, a screenshot, logs, or updated stream state. -### `server/` — Rust HTTP and WebRTC transport +This is why a long-lived daemon feels faster than repeatedly calling lower-level simulator tools. -Owns the public CLI shape (`simdeck`, `simdeck ui`, `daemon`, `boot`, `shutdown`, …), daemon metadata, the HTTP API, WebRTC streaming, the inspector hub, log streaming, and metrics. +## Video Flow -Key modules: +The browser opens a live stream for the selected device. SimDeck sends fresh frames, drops stale ones when a client falls behind, and lets the browser request refreshes. The UI can use WebRTC or H.264-over-WebSocket fallback depending on browser support and network behavior. -| Module | Responsibility | -| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------- | -| `server/src/main.rs` | CLI entrypoint, project daemon management, AppKit main-thread shim, tokio runtime bootstrap. | -| `server/src/api/routes.rs` | Every `/api/*` route, including simulator control, accessibility, and inspector proxy. | -| `server/src/android.rs` | Android AVD discovery, emulator lifecycle, ADB input, emulator gRPC video, screenshots, UIAutomator, and logcat. | -| `server/src/transport/webrtc.rs` | WebRTC offer/answer endpoint for H.264 browser video. | -| `server/src/transport/packet.rs` | Shared encoded frame type used between simulator sessions and transports. | -| `server/src/inspector.rs` | WebSocket hub for the NativeScript runtime inspector. | -| `server/src/simulators/registry.rs` | Per-UDID session registry with lazy attachment to the native bridge. | -| `server/src/simulators/session.rs` | Frame broadcast channel, keyframe gating, refresh requests. | -| `server/src/metrics/counters.rs` | Atomic counters and per-client stream stats accepted from stream transports or `/api/client-stream-stats`. | -| `server/src/logs.rs` | `os_log` log streaming and filtering. | +Tune this from the user-facing controls or with: -The Rust server runs the tokio runtime on a worker thread while the AppKit main loop spins on the main thread. The native bridge needs the main loop to deliver display callbacks and HID events. +```sh +simdeck daemon restart --video-codec software --stream-quality low +``` -### `cli/` — Objective-C native bridge +## Inspector Flow -Anything that depends on macOS frameworks, `xcrun simctl`, or private `CoreSimulator` / `SimulatorKit` APIs lives here. The Rust side talks to it through a narrow C ABI: +`simdeck describe` and the browser inspector use the best available source: -- `cli/native/XCWNativeBridge.{h,m}` — exported C functions for simulator control, chrome rendering, and frame callbacks. -- `cli/native/XCWNativeSession.{h,m}` — wraps one Objective-C private simulator session handle for the Rust registry. +1. Framework runtime inspector, such as NativeScript, React Native, or Flutter. +2. Swift in-app agent for UIKit or SwiftUI apps. +3. Native accessibility snapshot as the universal fallback. -Inside the bridge: +The response tells you which source was used and why a requested source fell back. -- **`XCWSimctl.{h,m}`** wraps `xcrun simctl` for discovery, lifecycle management, app launching, URL opening, and screenshot capture. -- **`XCWPrivateSimulatorBooter.{h,m}`** uses private `CoreSimulator` APIs for direct simulator boot without launching Simulator.app. -- **`DFPrivateSimulatorDisplayBridge.{h,m}`** owns headless private display frames plus HID-based touch and keyboard injection. It resolves the active Xcode developer directory explicitly, prefers direct CoreSimulator screen IOSurface callbacks, and activates the older SimulatorKit offscreen renderable view only when direct frame callbacks are unavailable. -- **`XCWPrivateSimulatorSession.{h,m}`** owns one private display bridge per booted simulator plus a selectable hardware or software H.264 encoder. -- **`XCWPrivateSimulatorChromeBridge.{h,m}`** is an experimental private `SimulatorKit` chrome bridge kept nearby as a reference. -- **`XCWChromeRenderer.{h,m}`** renders Apple's CoreSimulator device-type PDF chrome assets into PNGs for the browser. -- **`XCWH264Encoder.{h,m}`** software / hardware H.264 encode. +## Repository Layout -### `client/` — React browser UI +| Folder | Purpose | +| ----------- | ------------------------------------------------- | +| `server/` | CLI entrypoint, daemon, API, streaming, metrics | +| `cli/` | macOS simulator bridge | +| `client/` | Browser UI | +| `packages/` | Inspectors, VS Code extension, and `simdeck/test` | +| `scripts/` | Build, packaging, and integration helpers | +| `docs/` | Documentation site | -The React app served at `/` is a thin shell that calls the REST API. It consumes -live device video over WebRTC H.264. iOS frames come from the native simulator -display bridge; Android frames come from emulator gRPC `streamScreenshot` and -are encoded through VideoToolbox on the server. +## Contributor Boundary -Layout under `client/src/`: - -- `app/AppShell.tsx` — top-level shell. -- `api/` — typed wrappers around `/api/*` (`client.ts`, `controls.ts`, `simulators.ts`, `types.ts`). -- `features/stream/` — WebRTC client, receiver stats, and frame plumbing. -- `features/viewport/` — frame canvas, hit testing, chrome compositing. -- `features/input/` — touch/keyboard/hardware button affordances. -- `features/accessibility/` — accessibility tree pane and source switcher. -- `features/simulators/` — simulator list, boot/shutdown affordances. -- `features/toolbar/` — top toolbar (rotate, home, app switcher, dark mode toggle). - -The client never depends on private APIs and never assumes anything not exposed by the HTTP API. - -### `packages/` — companion packages - -- **`packages/nativescript-inspector/`** ships `@nativescript/simdeck-inspector`, a TypeScript runtime that connects from a NativeScript app to the server's WebSocket inspector hub. See [NativeScript Runtime](/inspector/nativescript). -- **`packages/react-native-inspector/`** ships `react-native-simdeck`, a React Native runtime that connects from an app to the server's WebSocket inspector hub and publishes React Fiber hierarchy data. See [React Native Runtime](/inspector/react-native). -- **`packages/flutter-inspector/`** ships `simdeck_flutter_inspector`, a Flutter runtime plugin that connects from an app to the server's WebSocket inspector hub and publishes widget, render, and semantics hierarchy data. See [Flutter Runtime](/inspector/flutter). -- **`packages/inspector-agent/`** ships `SimDeckInspectorAgent`, a Swift Package you can link from a debug iOS app to expose its UIKit hierarchy. See [Swift In-App Agent](/inspector/swift). -- **`packages/vscode-extension/`** is the VS Code extension that opens the browser client inside a webview panel and auto-starts the server. -- **`packages/simdeck-test/`** ships `simdeck/test`, a small JS/TS wrapper around daemon startup and the REST control API. See [Testing](/guide/testing). - -## Data flow - -### Simulator control - -Most control endpoints follow the same path: a typed Rust handler in `server/src/api/routes.rs` calls `SessionRegistry::bridge()`, which dispatches into `cli/native/XCWNativeBridge.*` over the C ABI. From there the call lands in the matching Objective-C unit. For example, `POST /api/simulators/{udid}/boot` ends up in `XCWPrivateSimulatorBooter`, which uses private `CoreSimulator` APIs for direct boot and returns a clear error if that private path fails. - -### Live video - -The browser posts an SDP offer to `/api/simulators/{udid}/webrtc/offer`. The handler in `transport::webrtc` starts the selected frame source, waits for the first H.264 keyframe, returns an SDP answer, and writes H.264 samples to a WebRTC video track. iOS frames come from the private display bridge, which first waits for direct CoreSimulator screen IOSurface callbacks and then falls back to the SimulatorKit offscreen renderable view. For Android, the source is emulator gRPC raw pixels passed through the shared VideoToolbox encoder path. - -### Input - -Touch and keyboard events POST to `/api/simulators/{udid}/touch` and `/key`. The handler resolves the active session and replays the event through the private display bridge using HID. - -### Inspectors - -The accessibility tree endpoint blends three sources, in priority order: - -1. **NativeScript runtime inspector** — preferred when the foreground app has connected to `/api/inspector/connect` over WebSocket. -2. **Swift in-app inspector agent** — used when the foreground app links the `SimDeckInspectorAgent` Swift Package and listens on a TCP port discovered between `47370` and `47402`. -3. **Accessibility snapshot** — a final fallback that shells out to the accessibility snapshot - -The server discovers which inspectors are reachable for a given Simulator and surfaces the available list in the `availableSources` field on every accessibility-tree response. - -## Process model - -SimDeck stays in one OS process. The Rust binary: - -1. Calls `xcw_native_initialize_app()` so AppKit creates an `NSApplication` on the main thread. -2. Spawns a tokio runtime on a worker thread that owns the HTTP server, WebRTC transport, inspector hub, and registry. -3. Spins the AppKit main loop in 50 ms slices on the main thread to dispatch display and HID callbacks. - -Normal CLI commands may spawn `simdeck daemon run` in the background for the current project. The daemon writes metadata under the system temp directory, and later commands reuse it while `/api/health` stays healthy. - -## Working rules - -If you contribute, keep the following invariants in mind: - -- Simulator-native logic stays in Objective-C under `cli/`. -- Rust server logic stays under `server/`. -- Browser-only presentation logic stays in `client/`. -- NativeScript app runtime inspection logic stays in `packages/nativescript-inspector/`. -- Flutter app runtime inspection logic stays in `packages/flutter-inspector/`. -- Add a server endpoint before adding client-only assumptions. -- The supported live video paths are the WebRTC H.264 offer endpoint plus the `/api/simulators/{udid}/h264` WebSocket fallback. Do not bring back legacy `/stream.h264` handling. +Keep platform-specific simulator work in the native layer, server behavior in `server/`, and browser presentation in `client/`. Add API support before adding UI assumptions that cannot be scripted. diff --git a/docs/guide/daemon.md b/docs/guide/daemon.md index 9bb94502..8d364b75 100644 --- a/docs/guide/daemon.md +++ b/docs/guide/daemon.md @@ -1,152 +1,86 @@ -# Project Daemon +# Daemon -SimDeck runs one warm native host per project. The daemon owns the HTTP API, the WebRTC video endpoint, inspector routing, and lazy native simulator sessions. +SimDeck runs a local server for the current project. The server owns the browser UI, REST API, live stream, inspector connections, and warm device sessions. -Normal CLI commands start the daemon automatically when they need it. Use `simdeck daemon` only when you want to manage it explicitly. +Most commands start or reuse the project daemon automatically. Manage it directly only when you need a specific lifecycle. -Running `simdeck` with no subcommand starts a foreground workspace daemon, prints local and LAN HTTP URLs, prints a six-digit pairing code for LAN browsers, and stops when the command exits, when you press `q`, or when you press Ctrl-C. Pass a simulator name or UDID as the only argument to select it by default: +## Foreground ```sh simdeck -simdeck "iPhone 17 Pro Max" -simdeck -d -simdeck -k -simdeck -r ``` -The shorthand flags map to detached start, kill, and restart respectively. `simdeck -k` reports the PID that was killed, and `simdeck -r` returns the same JSON shape as `simdeck daemon start`. - -`simdeck daemon` is project-scoped. `simdeck service` is the optional macOS -LaunchAgent wrapper for users who want an always-on daemon after login. +Use this for normal interactive work. It prints browser URLs and stops when you press `q` or Ctrl-C. -## Start +## Detached ```sh +simdeck -d simdeck daemon start ``` -The command starts the daemon for the current project root and prints JSON: - -```json -{ - "ok": true, - "projectRoot": "/path/to/app", - "pid": 12345, - "pairingCode": "123456", - "url": "http://127.0.0.1:4310", - "started": true -} -``` - -If a healthy daemon is already running for that project, `started` is `false` and the same daemon is reused. - -## Open The UI - -For day-to-day use, `ui` is the shortest path: - -```sh -simdeck ui --open -``` - -This starts or reuses the project daemon, serves the bundled browser client, and opens the authenticated local URL. - -## Options - -`daemon start`, `daemon restart`, and `ui` accept the same server options: - -| Flag | Default | Notes | -| -------------------- | --------------------- | ------------------------------------------------------------------------------------ | -| `--port ` | `4310` | HTTP port for the REST API, browser UI, and WebRTC offer endpoint. | -| `--bind ` | `127.0.0.1` | Bind address. Use `0.0.0.0` for [LAN access](/guide/lan-access). | -| `--advertise-host` | matches local host | Hostname or IP advertised to browser clients. | -| `--client-root` | bundled `client/dist` | Override the static browser client directory. | -| `--video-codec` | `auto` | One of `auto`, `hardware`, or `software`. See [Video](/guide/video). | -| `--low-latency` | `false` | Software H.264 profile for slower runners; caps at 15 fps and drops stale frames. | -| `--stream-quality` | `full` | Realtime stream quality profile, including `full`, `low`, `tiny`, and `ci-software`. | -| `--local-stream-fps` | `60` | Local quality stream frame target, from 15 to 240 fps. | -| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | - -Example: - -```sh -simdeck ui --bind 0.0.0.0 --advertise-host 192.168.1.50 --open -``` - -## Status +Both start or reuse a background daemon for the current project. Detached mode is useful for tests, editor integrations, and scripts. ```sh simdeck daemon status +simdeck daemon stop +simdeck daemon restart +simdeck daemon killall ``` -The status output includes the daemon URL, supervisor PID, project root, access -token, and detached daemon log path. Local same-origin browser use does not -require copying the token; direct remote API callers should send it as -`X-SimDeck-Token` or `Authorization: Bearer `. +`daemon killall` stops SimDeck project daemons from every workspace. -## Restart +## Open The Browser UI ```sh -simdeck daemon restart +simdeck ui --open ``` -This stops the daemon for the current project, starts a fresh one, and returns the same JSON shape as `daemon start`. +This starts or reuses the daemon, then opens the authenticated local URL. -## Stop +## Common Server Options -```sh -simdeck daemon stop -``` +`simdeck ui`, `daemon start`, `daemon restart`, and `service restart` use the same core options: -This terminates the daemon for the current project and removes its metadata file from the system temp directory. The next CLI command that needs the daemon starts a fresh one. +| Flag | Default | Use it when | +| ---------------------------- | -------------- | ------------------------------------------ | ------ | ---------------------------------- | +| `--port ` | `4310` | The default port is busy | +| `--bind ` | `127.0.0.1` | You need LAN access with `0.0.0.0` or `::` | +| `--advertise-host ` | detected | Remote browsers need a specific host or IP | +| `--video-codec auto | hardware | software` | `auto` | You need to force encoder behavior | +| `--stream-quality ` | `full` | You want lower CPU or bandwidth use | +| `--local-stream-fps ` | `60` | You want a different local stream target | +| `--client-root ` | bundled client | You are serving a custom static client | -## Kill All Project Daemons +Example: ```sh -simdeck daemon killall +simdeck daemon start --port 4320 --video-codec software --stream-quality low ``` -This scans SimDeck daemon metadata across all workspaces, terminates live daemon PIDs, and removes stale metadata files. - ## Always-On Service -For agents and editor integrations that should be able to reach SimDeck at any -time after login, use `simdeck service` to install the macOS user service: +Use the macOS user service when SimDeck should be reachable after login without starting a project daemon first: ```sh simdeck service on -``` - -This writes `~/Library/LaunchAgents/dev.nativescript.simdeck.plist`, starts the -server with `launchctl`, and keeps it alive. It binds to `127.0.0.1:4310` by -default and serves the bundled browser client. - -Restart it after changing options: - -```sh -simdeck service restart --port 4310 --video-codec software --low-latency -``` - -Disable it when you do not want a persistent daemon: - -```sh +simdeck service restart --port 4310 simdeck service off ``` -Prefer the project daemon for project-scoped metadata and automatic lifecycle. -Use the service when the priority is easy access from Codex, VS Code, or a -browser at any time. +Prefer the project daemon for normal repository work. Use the service for long-lived agent or editor setups. -## CoreSimulator Service Layer +## Restart CoreSimulator -The project daemon is different from Apple's CoreSimulator service. If `simctl` reports stale service state or the live display never produces a first frame, restart Apple's service layer: +If `simctl` hangs, reports a stale service version, or devices never attach: ```sh simdeck core-simulator restart ``` -You can also manage it explicitly: +Then retry: ```sh -simdeck core-simulator start -simdeck core-simulator shutdown +simdeck list +simdeck boot ``` diff --git a/docs/guide/github-actions.md b/docs/guide/github-actions.md index f7bd6220..d371f619 100644 --- a/docs/guide/github-actions.md +++ b/docs/guide/github-actions.md @@ -1,24 +1,20 @@ # GitHub Actions -SimDeck can run an iOS Simulator session from a pull request comment. A repository -needs two pieces: +SimDeck can post a temporary hosted simulator session from a pull request comment. -1. A build workflow that uploads a zipped iOS Simulator `.app` artifact. -2. A comment workflow that calls SimDeck's session action when someone - comments `simdeck run ios` on a pull request. +You need: -## Build Workflow +1. A build workflow that uploads a zipped iOS Simulator `.app`. +2. A comment workflow that starts the SimDeck session when someone comments `simdeck run ios`. -Build your app however your project normally builds a simulator target, then use -the upload action to package and publish the `.app` artifact: +## Build Workflow ```yaml name: Build iOS Simulator on: push: - branches: - - main + branches: [main] pull_request: permissions: @@ -43,20 +39,16 @@ jobs: app-glob: platforms/ios/build/**/*-iphonesimulator/*.app ``` -The uploaded artifact name defaults to `ios-simulator-app-`. For pull -requests, the SHA is the PR head commit. +Pin the action to a release tag when you want a stable integration point. ## Comment Workflow -Add a second workflow that delegates the hosted simulator session to SimDeck: - ```yaml name: SimDeck iOS Comment on: issue_comment: - types: - - created + types: [created] permissions: actions: read @@ -86,22 +78,13 @@ jobs: bundle_id: com.example.app ``` -When triggered, the session action: +## Pull Request Command -- reacts to the command comment immediately; -- installs `simdeck` and `cloudflared` on a single macOS runner; -- starts SimDeck with software encoding and the `tiny` stream profile by default; -- prefers `iPhone 17 Pro`, then falls back to the newest available iPhone simulator; -- restores the CoreSimulator device cache when available; -- posts the first bot comment only after a simulator UDID has been selected, - mentioning the requester when `command_comment_author` is set; -- downloads the simulator app artifact for the PR head commit, installs it, and - launches it; -- stops after 30 minutes by default, or earlier if the simulator shuts down. - -## Comment Flags +```text +simdeck run ios +``` -The comment can include lightweight flags: +Optional flags: ```text simdeck run ios no-cache-sim @@ -111,26 +94,26 @@ simdeck run ios quality=low simdeck run ios public-health ``` -Supported quality values are `tiny`, `low`, `economy`, `fast`, `smooth`, -`balanced`, `full`, `quality`, and `ci-software`. - -## Inputs - -The most common session action inputs are: - -| Input | Default | Purpose | -| ------------------------ | ------------------------- | --------------------------------------------- | -| `bundle_id` | empty | Fallback app bundle id to launch. | -| `build_workflow` | `build-ios-simulator.yml` | Workflow file that uploads the app artifact. | -| `command_comment_id` | empty | Comment id to react to immediately. | -| `command_comment_author` | empty | GitHub user to mention when the URL is ready. | -| `artifact_prefix` | `ios-simulator-app` | Prefix used for `-` artifacts. | -| `simdeck_version` | `latest` | npm version or dist-tag to install. | -| `stream_profile` | `tiny` | Default stream quality profile. | -| `simulator_name` | `iPhone 17 Pro` | Preferred simulator device name. | -| `keepalive_seconds` | `1800` | Session lifetime after app launch. | -| `simulator_cache` | `true` | Restore and save CoreSimulator device cache. | - -The caller workflow owns job-level settings such as `runs-on`, -`timeout-minutes`, permissions, and concurrency. Pin `NativeScript/SimDeck` to a -release tag instead of `@main` when you want a stable integration point. +Supported quality values include `tiny`, `low`, `economy`, `fast`, `smooth`, `balanced`, `full`, `quality`, and `ci-software`. + +## Common Inputs + +| Input | Default | Purpose | +| ------------------- | ------------------------- | ------------------------------------------- | +| `bundle_id` | empty | Bundle ID to launch | +| `build_workflow` | `build-ios-simulator.yml` | Workflow file that uploads the app artifact | +| `artifact_prefix` | `ios-simulator-app` | Artifact prefix | +| `simdeck_version` | `latest` | npm version or dist-tag | +| `stream_profile` | `tiny` | Default stream quality | +| `simulator_name` | `iPhone 17 Pro` | Preferred simulator | +| `keepalive_seconds` | `1800` | Session lifetime after launch | +| `simulator_cache` | `true` | Restore and save simulator cache | + +## What The Session Does + +- Installs SimDeck and tunnel tooling on a macOS runner. +- Picks or creates an iOS Simulator. +- Downloads the app artifact for the PR head commit. +- Installs and launches the app. +- Posts a browser URL back to the pull request. +- Stops after the configured keepalive window. diff --git a/docs/guide/index.md b/docs/guide/index.md index b7cafac1..9114e771 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -1,55 +1,45 @@ -# Introduction +# Overview -SimDeck is a local-first control plane for the iOS Simulator. It bundles a Rust HTTP server, a native Objective-C bridge for CoreSimulator and SimulatorKit APIs, a React browser UI, and a JS/TS test API into one project-local CLI. +SimDeck is a local tool for viewing, controlling, inspecting, and automating mobile simulators. -The goal is simple: turn a booted Simulator into a streamable, scriptable surface that any tool — a browser, VS Code, a NativeScript runtime, or an automated test — can drive through one local daemon. +Run `simdeck` from your project. It starts a local server, serves a browser UI, and exposes the same controls through the CLI and HTTP API. -## Why SimDeck? +## What You Can Do -The default Simulator is great when it sits in front of you. It is much less great when: +- View a live iOS Simulator or Android emulator in a browser. +- Tap, swipe, type, press hardware buttons, rotate, and open URLs. +- Install, launch, uninstall, boot, shut down, and erase devices. +- Capture screenshots, logs, pasteboard text, and accessibility trees. +- Inspect app UI through built-in accessibility or optional in-app inspectors. +- Write JS/TS automation with `simdeck/test`. +- Share a paired browser session over your LAN. +- Open the simulator view inside VS Code. -- You want to see your app running while writing code in another window. -- You want to drive a Simulator from a remote machine on your LAN. -- You want to build automation around `simctl` without stitching together shell pipelines. -- You want to inspect a NativeScript or Swift app's view hierarchy without linking the Xcode debugger. -- You want a project daemon that stays warm instead of cold-starting native simulator control for every command. +## Daily Workflow -SimDeck addresses all of those with one CLI, one HTTP API, one WebRTC stream path, and one daemon per project. +```sh +simdeck +``` -## What's in the box +Open the local URL, pick a device, and use the toolbar or CLI commands: -SimDeck ships as a single npm package (`simdeck`) that installs: +```sh +simdeck list +simdeck boot +simdeck install /path/to/App.app +simdeck launch com.example.App +simdeck tap --label "Continue" --wait-timeout-ms 5000 +simdeck describe --format agent --max-depth 3 +``` -1. **A native CLI and project daemon.** Rust + Objective-C, compiled on install. It serves the HTTP API and WebRTC H.264 video. -2. **A bundled React client.** `simdeck` starts a foreground daemon and prints browser URLs; `simdeck ui --open` starts or reuses a background daemon. -3. **A JS/TS testing package.** `simdeck/test` gives app tests a small API for launching, tapping, querying accessibility state, batching actions, and taking screenshots. +Use `simdeck -d` for a detached background daemon, `simdeck -k` to stop it, and `simdeck -r` to restart it. -Optional companion packages: +## Pick A Page -- [`@nativescript/simdeck-inspector`](/inspector/nativescript) — a debug-only NativeScript inspector runtime. -- [`react-native-simdeck`](/inspector/react-native) — a debug-only React Native inspector runtime. -- [`simdeck_flutter_inspector`](/inspector/flutter) — a debug-only Flutter inspector runtime. -- [`packages/inspector-agent`](/inspector/swift) — a Swift Package you can link from your iOS app to expose its UIKit hierarchy. -- [`packages/vscode-extension`](/extensions/vscode) — opens the simulator inside a VS Code panel. - -## High-level architecture - -The repository splits cleanly along the layers SimDeck talks to: - -- **`server/`** holds the CLI entrypoint, project daemon, Rust HTTP server, WebRTC transport, inspector hub, and metrics. It serves the REST API at `/api/*`, live video at `/api/simulators/{udid}/webrtc/offer`, and the inspector WebSocket at `/api/inspector/connect`. -- **`cli/`** holds the Objective-C native bridge that links private `CoreSimulator` and `SimulatorKit` APIs. The Rust server calls into it through a narrow C ABI for boot, frame capture, encode, and HID input. -- **`client/`** holds the React UI that renders the streamed simulator and the inspector tools. -- **`packages/`** holds companion packages: NativeScript inspector, React Native inspector, Swift inspector agent, VS Code extension, and `simdeck/test`. -- **`scripts/`** holds repeatable build entrypoints used both locally and by CI. - -For a full breakdown, see [Architecture](/guide/architecture). - -## Where to next - -- **[Installation](/guide/installation)** — how to install the CLI from npm or build it from source. -- **[Quick Start](/guide/quick-start)** — boot a simulator and stream it to your browser in under a minute. -- **[Project Daemon](/guide/daemon)** — how `ui`, `daemon start`, and automatic daemon reuse work. -- **[Testing](/guide/testing)** — use `simdeck/test` and run the simulator-backed integration suite. -- **[Architecture](/guide/architecture)** — the layout, data flow, and private-API surface. -- **[CLI Reference](/cli/commands)** — every command with its flags. -- **[HTTP API](/api/rest)** — every REST endpoint, with response shapes. +- [Install](/guide/installation): requirements and setup. +- [Quick Start](/guide/quick-start): first browser session. +- [Daemon](/guide/daemon): foreground, detached, and always-on modes. +- [Video & Streaming](/guide/video): stream quality and codec choices. +- [LAN Access](/guide/lan-access): pairing and remote browser access. +- [Testing](/guide/testing): `simdeck/test` and integration tests. +- [Troubleshooting](/guide/troubleshooting): practical fixes for common failures. diff --git a/docs/guide/installation.md b/docs/guide/installation.md index c3de7417..a7c7e52f 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -1,57 +1,39 @@ -# Installation +# Install -SimDeck ships as a single npm package that contains the launcher, the client bundle, and the native CLI binary. +## Requirements -## Prerequisites +- macOS on Apple Silicon. +- Xcode with the simulator runtimes you want to use. +- Node.js 18 or newer. +- Optional Android SDK tools for Android emulator support. +- Optional Rust stable when building from source. -SimDeck only runs on macOS. The native bridge links private `CoreSimulator` and `SimulatorKit` frameworks, so it cannot run on Linux or Windows. - -| Requirement | Why | -| ---------------------------------- | ---------------------------------------------------------------------------------------------------------------- | -| **macOS 13+** | Required for current `CoreSimulator` and Apple's VideoToolbox H.264 encoder. | -| **Xcode + iOS Simulator runtimes** | The native bridge loads Xcode's private simulator frameworks and invokes `xcrun simctl` for non-boot operations. | -| **Android SDK tools** | Optional. Required for Android emulator support (`emulator`, `adb`, and AVD images). | -| **Node.js ≥ 18** | The launcher (`bin/simdeck.mjs`) and the bundled client tooling. | -| **Rust (stable)** | Required only when building from source. Installed via [rustup](https://rustup.rs/). | - -The package contains a macOS native binary. Non-macOS installs are unsupported. - -## Install from npm - -The fastest path is the published CLI: +Check Xcode selection if you have multiple installs: ```sh -npm install -g simdeck +xcode-select -p ``` -This installs the launcher and bundled native binary to your global `node_modules`. After it finishes: +## Install From npm + +Use `npx` for a quick try: ```sh -simdeck +npx simdeck ``` -The foreground command prints local and LAN HTTP URLs and stops its workspace daemon when you press `q` or Ctrl-C. You can select a simulator by name or UDID with `simdeck "iPhone 17 Pro Max"`. - -The global install prints the next setup steps: +Install globally for regular use: ```sh +npm install -g simdeck@latest simdeck -npx skills add NativeScript/SimDeck --skill simdeck -a codex -g -simdeck service on ``` -Install the `nativescript.simdeck` VS Code extension if you want the simulator -view inside VS Code. - -`simdeck service on` is recommended when agents or editor integrations should be -able to reach SimDeck any time after login. It installs a localhost macOS -LaunchAgent and can be removed with `simdeck service off`. +The package installs the launcher, browser client, and native binary. -## Install the Codex skill +## Install The Codex Skill -SimDeck includes an agent skill at `skills/simdeck/SKILL.md`. Install it with -[skills.sh](https://skills.sh/) so Codex can choose the right commands and -inspection loops automatically: +For Codex workflows, install the SimDeck skill so agents use the stable commands: ```sh npx skills add NativeScript/SimDeck --skill simdeck -a codex -g @@ -59,95 +41,45 @@ npx skills add NativeScript/SimDeck --skill simdeck -a codex -g Restart Codex after installing the skill. -## Install from source - -Clone the repo and install dependencies: - -```sh -git clone https://github.com/NativeScript/SimDeck.git -cd simdeck -npm install -``` +## VS Code -Build the source checkout before running it directly or installing it globally: +Install the VS Code extension if you want the simulator inside an editor panel: ```sh -npm run build +npm run package:vscode +npm run install:vscode ``` -The native CLI build writes: +From the marketplace, use `nativescript.simdeck-vscode` when available. -```text -build/simdeck -build/simdeck-bin -``` - -You can then run the local checkout directly: - -```sh -./build/simdeck -``` - -Or install the local checkout globally: - -```sh -npm install -g . -``` - -After a global install you can call `simdeck` from anywhere. - -## Build the React client - -The client bundle ships pre-built when installed from npm. When working from source, build it explicitly: - -```sh -./scripts/build-client.sh -``` - -This calls `npm install` and `npm run build` inside the `client/` workspace and writes the production bundle to `client/dist`. The Rust server serves that bundle at the HTTP root. - -## Build from source - -The root `package.json` exposes a one-shot app build for the native CLI and browser client: +## Build From Source ```sh +git clone https://github.com/NativeScript/SimDeck.git +cd SimDeck +npm install npm run build +./build/simdeck ``` -This runs: - -- `npm run build:cli` — Rust server + Objective-C bridge → `build/simdeck` -- `npm run build:client` — Vite production build → `client/dist` - -Use `npm run build:all` when you also need the companion package outputs: - -```sh -npm run build:all -``` - -That adds: +Useful build scripts: -- `npm run build:nativescript-inspector` — TypeScript build of the NativeScript inspector -- `npm run build:react-native-inspector` — TypeScript build of the React Native inspector -- `npm run build:simdeck-test` — TypeScript build of `simdeck/test` +| Script | What it builds | +| ---------------------- | ----------------------------------------- | +| `npm run build` | Native CLI and browser client | +| `npm run build:cli` | Native CLI only | +| `npm run build:client` | Browser client only | +| `npm run build:all` | CLI, client, inspectors, and test package | +| `npm run build:docs` | Documentation site | -Other convenience scripts include `npm run build:docs`, `npm run build:vscode-extension`, `npm run package:vscode`, and `npm run package:all`. You can also run any one of the component scripts on its own. - -## Update or uninstall - -To update from npm: +## Update Or Remove ```sh npm install -g simdeck@latest -``` - -To remove the global install: - -```sh npm uninstall -g simdeck ``` -If a project daemon is running, stop it before uninstalling so your shell does not keep talking to the old binary: +Stop a running daemon before uninstalling: ```sh simdeck daemon stop diff --git a/docs/guide/lan-access.md b/docs/guide/lan-access.md index 90acb796..d3167781 100644 --- a/docs/guide/lan-access.md +++ b/docs/guide/lan-access.md @@ -1,105 +1,66 @@ # LAN Access -SimDeck binds to `127.0.0.1` by default. You can move it to a LAN-reachable interface so other devices on your network — another Mac, an iPad, a phone — can stream the simulator. +SimDeck binds to `127.0.0.1` by default. Bind to a LAN address when another device needs to open the browser UI. -## Bind to all interfaces - -Use `--bind` to listen on a non-loopback address: - -```sh -simdeck ui --port 4310 --bind 0.0.0.0 --open -``` - -The HTTP server binds to the requested address. It serves the REST API, browser UI, inspector WebSocket, and WebRTC offer endpoint, so any browser on the LAN can reach SimDeck through `http://:4310`. LAN browsers must enter the pairing code printed by the foreground command or returned by `daemon start` before the API cookie is issued. - -## Advertise the right host - -By default SimDeck advertises `localhost`, which only works for browsers running on the same Mac. Tell the server what host to print for remote clients: +## Start A LAN Session ```sh simdeck ui \ - --port 4310 \ --bind 0.0.0.0 \ --advertise-host 192.168.1.50 \ --open ``` -The advertised host is used in the printed Network URL and in `daemon start` JSON output. +Open the printed network URL from the remote browser: -If you skip `--advertise-host` while binding to `0.0.0.0`, the server prints a warning at startup because it will still tell remote clients to dial `localhost`. +```text +http://192.168.1.50:4310 +``` + +Enter the pairing code printed by the CLI. After pairing, the browser receives the API cookie. -## Pick a hostname or IP +## Pick The Right Host -You can advertise either a DNS name (preferred when you have a stable mDNS or DHCP entry) or an IP literal. Examples: +Use an IP address or hostname that the remote device can resolve: ```sh -simdeck ui --bind 0.0.0.0 --advertise-host my-mac.lan --open +simdeck ui --bind 0.0.0.0 --advertise-host my-mac.local --open simdeck ui --bind 0.0.0.0 --advertise-host 192.168.1.50 --open -simdeck ui --bind :: --advertise-host my-mac.local --open ``` -Whatever you advertise must be resolvable from the remote client. +If you bind to `0.0.0.0` but advertise `localhost`, remote browsers will try to connect to themselves. -## Health response +## Direct API Access -`GET /api/health` reports the HTTP port and active video encoder mode: +Loopback browser sessions are authenticated automatically. Direct API callers should send the token from: -```json -{ - "ok": true, - "httpPort": 4310, - "videoCodec": "auto", - "lowLatency": false, - "webRtc": { - "iceServers": [{ "urls": ["stun:stun.l.google.com:19302"] }], - "iceTransportPolicy": "all" - } -} +```sh +simdeck daemon status ``` -Restarting the server rotates the generated API access token. Open clients reconnect automatically after pairing or receiving the loopback cookie again. - -## Authentication and security - -SimDeck generates an API access token when it starts the project daemon. Loopback browser UI loads receive the token automatically through a strict same-site cookie, so opening `http://127.0.0.1:` remains seamless. Non-loopback LAN browsers do not receive that cookie from the static page; they must submit the six-digit pairing code shown by the CLI before the server sets the cookie. - -Direct API callers must send one of: +Use either header: ```text X-SimDeck-Token: Authorization: Bearer ``` -The foreground `simdeck` command prints an HTTP network URL and a pairing code: - -```text -🚀 SimDeck is ready - - Local: http://127.0.0.1:4310 - Network: http://192.168.1.50:4310 - Pair: 123 456 - -q or ^C to stop server -``` - -Get the token for scripts with: - -```sh -simdeck daemon status -``` - -Recommended practice for shared networks: +## Security Checklist -- Run SimDeck only on networks you control. -- Treat the token from `daemon status` as a secret for scripted LAN access. -- Combine with macOS Application Firewall to restrict inbound access to known peers. -- For shared NativeScript inspectors, set an `authToken` when starting the [Swift in-app agent](/inspector/swift#auth-token) so app-side requests must include the token. +- Use LAN mode only on networks you trust. +- Treat the daemon token as a secret. +- Restrict the port with macOS Firewall when needed. +- Stop the daemon when the shared session is done: -## Quick checklist + ```sh + simdeck daemon stop + ``` -To make a SimDeck server reachable from another device: +## Troubleshooting -1. `--bind 0.0.0.0` (or `--bind ::`). -2. `--advertise-host `. -3. Allow the chosen port through any firewalls. -4. Visit `http://:` from the remote device. +| Symptom | Fix | +| ----------------------------- | ----------------------------------------------------------------- | +| Remote browser cannot connect | Confirm `--bind 0.0.0.0`, firewall rules, and the advertised host | +| Pairing code is rejected | Restart the daemon and use the newly printed code | +| Stream connects but stutters | Use `--stream-quality low` or `--video-codec software` | +| API script gets `401` | Send `X-SimDeck-Token` or `Authorization: Bearer ` | diff --git a/docs/guide/quick-start.md b/docs/guide/quick-start.md index dc0a4378..ee9d4537 100644 --- a/docs/guide/quick-start.md +++ b/docs/guide/quick-start.md @@ -1,95 +1,86 @@ # Quick Start -This guide walks you from a fresh install to a Simulator streaming in your browser and controllable from the CLI. - -## 1. Open The UI - -After [installing](/guide/installation), start a foreground SimDeck daemon and open one of the printed browser URLs: +## 1. Start SimDeck ```sh simdeck ``` -The command prints local and LAN URLs: +SimDeck prints a local browser URL, a LAN URL when one is available, and a pairing code for LAN browsers. ```text -🚀 SimDeck is ready - - Local: http://127.0.0.1:4310 - Network: http://192.168.1.50:4310 - Pair: 123 456 +SimDeck is ready -q or ^C to stop server +Local: http://127.0.0.1:4310 +Network: http://192.168.1.50:4310 +Pair: 123 456 ``` -This foreground daemon is scoped to the current workspace and exits when the command exits, when you press `q`, or when you press Ctrl-C. Local browsers open without a prompt. LAN browsers pair once with the printed code, then receive the API cookie. Use `simdeck ui --open` or `simdeck daemon start` when you want a reusable background daemon. +Open the local URL. Press `q` or Ctrl-C in the terminal to stop the foreground server. -For shorthand background lifecycle commands: +To open a specific simulator by name or UDID: ```sh -simdeck -d # detached start -simdeck -k # kill background daemon -simdeck -r # restart background daemon +simdeck "iPhone 17 Pro Max" +simdeck 9750DF52-0471-48FF-B49A-B184C4BD3A3D ``` -One HTTP listener runs inside the daemon: +## 2. Pick Or Boot A Device -- **HTTP** on `--port` (default `4310`) for the REST API, static React client, inspector WebSocket, and WebRTC offer endpoint. - -## 2. Pick A Simulator - -The opened UI lists every simulator. You can also list and boot from the CLI: +The UI lists available iOS Simulators and Android emulators. You can also use the CLI: ```sh simdeck list simdeck boot ``` -To focus a specific simulator by name or UDID at launch: +Android emulator IDs are prefixed with `android:`. + +## 3. Install And Launch An App ```sh -simdeck "iPhone 17 Pro Max" -simdeck 9750DF52-0471-48FF-B49A-B184C4BD3A3D +simdeck install /path/to/App.app +simdeck launch com.example.App +simdeck open-url myapp://debug ``` -::: tip First-frame delay -On a cold boot the daemon has to boot the device through private CoreSimulator APIs, attach the private display bridge, and wait for a keyframe before video flows. The first frame typically shows up within a second; subsequent reloads of the same Simulator are near-instant. -::: - -## 3. Drive It - -The same daemon backs browser controls, CLI commands, and tests: +For Android: ```sh -simdeck open-url https://example.com -simdeck launch com.apple.Preferences -simdeck tap --label "Continue" --wait-timeout-ms 5000 -simdeck describe --format agent --max-depth 2 +simdeck install android: /path/to/app.apk +simdeck launch android: com.example.app ``` -Coordinate commands use screen points by default. Pass `--normalized` when sending `0.0..1.0` coordinates: +## 4. Drive The UI + +Use coordinates when you know them: ```sh -simdeck tap 0.5 0.5 --normalized +simdeck tap 120 240 +simdeck swipe 200 700 200 200 +simdeck type "hello" ``` -See the full [CLI Reference](/cli/commands) for every command and flag. +Use selectors when you want automation to wait for UI state: -## What just happened? +```sh +simdeck tap --label "Continue" --wait-timeout-ms 5000 +simdeck describe --format agent --max-depth 3 +``` -When you opened the UI: +## 5. Keep It Running In The Background -1. The browser fetched `/api/health` and confirmed the active H.264 encoder mode. -2. It posted a WebRTC SDP offer to `/api/simulators//webrtc/offer`. -3. The Rust transport hub asked the native bridge for a fresh keyframe and started writing H.264 samples to the WebRTC video track. -4. Touch and keyboard events round-trip through `POST /api/simulators//touch` and `/key`, which the native bridge replays through HID. +```sh +simdeck -d +simdeck -k +simdeck -r +``` -You can read more about the layers involved in [Architecture](/guide/architecture) and [Video Pipeline](/guide/video). +These are shortcuts for detached start, stop, and restart. See [Daemon](/guide/daemon) for details. -## Common follow-ups +## Next -- **Run the server on a LAN-reachable port.** See [LAN Access](/guide/lan-access). -- **Manage the warm native host explicitly.** See [Project Daemon](/guide/daemon). -- **Write JS/TS app tests.** See [Testing](/guide/testing). -- **Stream is choppy or stuck on a black frame.** See [Video Pipeline](/guide/video) and [Troubleshooting](/guide/troubleshooting). -- **Embed the client in VS Code.** See the [VS Code extension](/extensions/vscode). +- [CLI commands](/cli/commands) +- [Video & streaming](/guide/video) +- [Inspectors](/inspector/) +- [Troubleshooting](/guide/troubleshooting) diff --git a/docs/guide/testing.md b/docs/guide/testing.md index 0fe8c2a2..8cb0fdbe 100644 --- a/docs/guide/testing.md +++ b/docs/guide/testing.md @@ -1,10 +1,11 @@ # Testing -SimDeck supports two test layers: a small JS/TS client package for app tests, and a simulator-backed integration suite for the CLI and REST API. +SimDeck supports two testing workflows: -## App Tests With `simdeck/test` +- App-level JS/TS automation through `simdeck/test`. +- Repository integration tests that drive real simulators and emulators. -`simdeck/test` starts or reuses the project daemon and gives tests a typed API for simulator control: +## App Tests With `simdeck/test` ```ts import { connect } from "simdeck/test"; @@ -12,55 +13,46 @@ import { connect } from "simdeck/test"; const sim = await connect(); try { - const devices = await sim.list(); - await sim.launch("", "com.example.App"); - await sim.tap("", 0.5, 0.5); - await sim.waitFor("", { label: "Continue" }); - const png = await sim.screenshot(""); + const { simulators } = await sim.list(); + const udid = simulators.find((device) => device.isBooted)?.udid; + + await sim.launch(udid, "com.example.App"); + await sim.tap(udid, 0.5, 0.5); + await sim.waitFor(udid, { label: "Continue" }); + await sim.screenshot(udid); } finally { sim.close(); } ``` -`connect()` starts the daemon when needed, reuses it when healthy, and only stops daemons it started itself unless `keepDaemon` is set. - -## Session API - -The current session object exposes: - -| Method | Purpose | -| -------------------------------------- | ----------------------------------------------------------------- | -| `list()` | Fetch simulator inventory from `GET /api/simulators`. | -| `boot()`, `shutdown()`, `erase()` | Manage simulator or Android emulator lifecycle. | -| `install()`, `uninstall()` | Install or remove an app. | -| `launch()` | Launch an installed bundle ID or Android package. | -| `openUrl()` | Open a URL or deep link. | -| `tap()`, `tapElement()` | Tap normalized coordinates or a matching accessibility element. | -| `touch()`, `swipe()`, `gesture()` | Send normalized pointer gestures. | -| `typeText()`, `key()`, `keySequence()` | Send text or HID keyboard input. | -| `button()` | Press a hardware button. | -| `home()`, `dismissKeyboard()` | Trigger common system controls. | -| `appSwitcher()` | Open the app switcher. | -| `rotateLeft()`, `rotateRight()` | Rotate the simulator display. | -| `toggleAppearance()` | Toggle light/dark appearance. | -| `pasteboardSet()`, `pasteboardGet()` | Read or write pasteboard text. | -| `chromeProfile()` | Fetch screen/chrome geometry. | -| `logs()` | Fetch recent simulator or Android log entries. | -| `tree()` | Fetch an accessibility hierarchy. | -| `query()` | Return compact matches for a selector. | -| `waitFor()` | Poll until a selector appears. | -| `assert()` | Assert a selector is present. | -| `batch()` | Run multiple REST actions through `/api/simulators/{udid}/batch`. | -| `screenshot()` | Return a PNG buffer. | -| `close()` | Stop the daemon if this session started it. | - -Selectors can match `id`, `label`, `value`, or `type`. Query options accept `source`, `maxDepth`, and `includeHidden`. - -## Repository Integration Suite - -The repo includes simulator-backed integration runners. The iOS runner is -macOS-only; it creates a temporary simulator, builds and installs a small UIKit -fixture app, then sweeps the CLI and REST control surface. +`connect()` starts the project daemon if needed, reuses a healthy daemon, and only stops daemons it started itself. + +## Useful Test Methods + +| Method | Purpose | +| ----------------------------------------------- | ------------------------------ | +| `list()` | Device inventory | +| `boot()`, `shutdown()`, `erase()` | Device lifecycle | +| `install()`, `uninstall()`, `launch()` | App lifecycle | +| `openUrl()` | Universal links and deep links | +| `tap()`, `tapElement()`, `swipe()`, `gesture()` | UI input | +| `typeText()`, `key()`, `keySequence()` | Text and keyboard input | +| `button()`, `home()`, `appSwitcher()` | System controls | +| `tree()`, `query()`, `waitFor()`, `assert()` | UI state checks | +| `screenshot()`, `logs()` | Evidence capture | +| `batch()` | Multi-step actions | + +Selectors can match `id`, `label`, `value`, or `type`. + +## Repository Tests + +Normal unit and client tests: + +```sh +npm run test +``` + +iOS integration test: ```sh npm run build:cli @@ -69,41 +61,13 @@ npm run test:integration:fixture npm run test:integration:cli ``` -Verbose mode opens Simulator.app and prints each step with timing: +Verbose iOS run: ```sh npm run test:integration:cli:verbose ``` -Useful environment variables: - -| Variable | Purpose | -| --------------------------------------- | ----------------------------------------------------- | -| `SIMDECK_INTEGRATION_VERBOSE=1` | Print commands, outputs, timings, and UI checkpoints. | -| `SIMDECK_INTEGRATION_TRACE_HTTP=1` | Print raw HTTP request logs. | -| `SIMDECK_INTEGRATION_SHOW_SIMULATOR=1` | Open Simulator.app during the run. | -| `SIMDECK_INTEGRATION_KEEP_SIMULATOR=1` | Leave the temporary simulator after exit. | -| `SIMDECK_INTEGRATION_SIMCTL_TIMEOUT_MS` | Override the cold CoreSimulator command timeout. | -| `SIMDECK_INTEGRATION_IOS_RUNTIME` | Force a runtime by version, name, or identifier. | -| `SIMDECK_INTEGRATION_DEVICE_TYPE` | Force an iPhone device type by name or identifier. | - -The integration suite is separate from `npm run test` because it boots and drives a real iOS simulator. -The UIKit fixture app is cached under `.cache/simdeck/fixture` using a hash -of its generated source, plist, simulator SDK, Clang version, and host -architecture. - -By default, the integration runner selects the newest available iOS runtime that -does not exceed the active `iphonesimulator` SDK version, falling back to the -same major version when needed. This keeps CI off newer installed runtimes that -do not match the selected Xcode toolchain. - -Android coverage is opt-in because it requires a locally installed Android SDK -and at least one existing AVD. It runs on macOS or Linux. On Linux, SimDeck -builds the daemon with a native iOS stub and leaves the Android bridge active. -The runner starts an isolated SimDeck daemon and sweeps the Android CLI and -`simdeck/test` surface for lifecycle, tree, screenshot, pasteboard behavior, -app launch, URL opening, touch/swipe/gesture, keyboard, system buttons, -rotation, appearance, logs, and batch controls: +Android integration test: ```sh npm run build:cli @@ -111,32 +75,28 @@ npm run build:simdeck-test npm run test:integration:android ``` -Set `SIMDECK_INTEGRATION_ANDROID_AVD=` to pick a specific AVD. The -runner expects that emulator to already be booted, which is how the Linux CI job -uses `reactivecircus/android-emulator-runner`. If no AVD is configured, or a -local AVD exists but is not running, the Android runner prints a skip message -and exits successfully. Set `SIMDECK_INTEGRATION_REQUIRE_RUNNING_ANDROID=1` to -turn that skip into a failure. Set `SIMDECK_INTEGRATION_BOOT_ANDROID=1` to let -SimDeck cold-boot a local AVD before the suite. Set -`SIMDECK_INTEGRATION_KEEP_ANDROID=1` to leave an emulator booted when the runner -started it. Set `SIMDECK_INTEGRATION_STEP_TIMEOUT_MS` to override the per-step -timeout. +Android tests require the Android SDK and a running or bootable AVD. + +## Helpful Environment Variables -## Stress and Leak Checks +| Variable | Purpose | +| ----------------------------------------------- | ---------------------------------------------------- | +| `SIMDECK_INTEGRATION_VERBOSE=1` | Print commands, outputs, and timings | +| `SIMDECK_INTEGRATION_SHOW_SIMULATOR=1` | Open Simulator.app during iOS tests | +| `SIMDECK_INTEGRATION_KEEP_SIMULATOR=1` | Keep the temporary iOS simulator | +| `SIMDECK_INTEGRATION_TRACE_HTTP=1` | Print HTTP request logs | +| `SIMDECK_INTEGRATION_ANDROID_AVD=` | Pick an Android AVD | +| `SIMDECK_INTEGRATION_BOOT_ANDROID=1` | Let SimDeck boot the Android emulator | +| `SIMDECK_INTEGRATION_REQUIRE_RUNNING_ANDROID=1` | Fail instead of skipping when Android is unavailable | -Use the stress runner against an already-running daemon when you want to shake out -high-usage reliability issues without adding minutes to every PR: +## Stress Test A Running Daemon ```sh npm run test:stress -- --server-url http://127.0.0.1:4310 --iterations 1000 --concurrency 12 ``` -To include simulator-specific refresh traffic and RSS growth checks: +Include simulator refresh traffic: ```sh -npm run test:stress -- --udid --iterations 2000 --concurrency 16 --max-rss-growth-mb 256 +npm run test:stress -- --udid --iterations 2000 --concurrency 16 ``` - -The runner repeatedly calls health, metrics, simulator listing, stream-quality, -and optional simulator refresh endpoints. It samples the daemon process RSS with -`ps` and fails if the peak or growth limits are exceeded. diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index 0961b0c3..06633d58 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -1,133 +1,210 @@ # Troubleshooting -Most SimDeck issues fall into one of three buckets: simulator boot, video stream, or accessibility/inspector. This page lists the symptoms and fixes for the ones we hit most often. +Use this page when SimDeck does not start, cannot see a device, shows a bad stream, or falls back to the wrong inspector. -## Server won't start +## First Checks -### `bind HTTP listener on 127.0.0.1:4310` +```sh +simdeck --version +xcode-select -p +simdeck daemon status +simdeck list +``` + +If a background daemon may be stale: + +```sh +simdeck daemon stop +simdeck +``` + +## Server Will Not Start -Another process already owns the HTTP port. Pick a different one: +### Port is already in use + +```text +bind HTTP listener on 127.0.0.1:4310 +``` + +Use another port: ```sh -simdeck daemon start --port 4320 +simdeck ui --port 4320 --open ``` -Or find what's holding it: +Or find the listener: ```sh lsof -nP -iTCP:4310 -sTCP:LISTEN ``` -If the holder is an old SimDeck daemon for the current project, stop it: +If it is an old project daemon: ```sh simdeck daemon stop ``` -### `simdeck native binary is missing` +### Native binary is missing + +Reinstall from npm: + +```sh +npm install -g simdeck@latest +``` -The launcher script could not find the native binary. Reinstall the package or run the local build: +From a source checkout: ```sh -npm install -g simdeck -# or, from a checkout: -./scripts/build-cli.sh +npm run build:cli ``` -### Native build fails from source +### Source build fails -`npm run build:cli` runs `cargo build --release` and Apple's Clang against the Objective-C bridge. The most common failures are: +Check the common prerequisites: -- **Rust missing.** Install via [rustup](https://rustup.rs/), then reinstall. -- **Xcode command-line tools missing.** Run `xcode-select --install`. -- **Sandboxed CI without macOS frameworks.** Build the npm package on macOS so the published tarball contains the native binary. +```sh +xcode-select --install +rustc --version +node --version +``` -## Simulator never boots +Builds must run on macOS because SimDeck links macOS simulator frameworks. -### Private CoreSimulator boot errors +## Device Does Not Boot Or List -SimDeck boots devices through private CoreSimulator APIs so it does not launch Simulator.app. If booting fails, the CLI returns that private CoreSimulator error directly and does not fall back to `xcrun simctl boot`. Confirm Xcode selection first: +### `simdeck list` hangs or returns stale data + +Restart Apple's simulator service: ```sh -xcode-select -p -DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer simdeck boot +simdeck core-simulator restart +simdeck list ``` -If Xcode selection is correct but SimDeck still fails, capture the server log and file an issue. Running `xcrun simctl boot ` can still be useful as a manual comparison, but it may launch Simulator.app and is not SimDeck's fallback path. +### Wrong Xcode is selected -### CoreSimulator service unhealthy +```sh +sudo xcode-select -s /Applications/Xcode.app +simdeck list +``` -If `simctl list` itself hangs or returns garbage, the macOS `com.apple.CoreSimulator.CoreSimulatorService` is wedged. Restart it: +Or run one command with an explicit developer directory: ```sh -simdeck core-simulator restart +DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer simdeck list ``` -Re-run `simdeck list` to confirm before retrying. +### Android emulator is missing -### Multiple Xcode installs +Confirm Android SDK tools are on `PATH`: + +```sh +adb devices +emulator -list-avds +``` -When more than one Xcode is installed, `xcrun simctl` uses whichever Xcode is selected by `xcode-select`. Pick the one whose runtimes you care about: +Android IDs in SimDeck use `android:`. + +## Stream Is Black Or Stuck + +### Timed out waiting for the first frame + +Try software encoding: ```sh -sudo xcode-select -s /Applications/Xcode.app +simdeck daemon restart --video-codec software +``` + +For CI or virtualized Macs: + +```sh +simdeck daemon restart --video-codec software --stream-quality ci-software ``` -## Stream is black or stuck +### Stream stutters or refreshes repeatedly -### "Timed out waiting for initial simulator keyframe" +Lower the quality: -The encoder did not produce a keyframe within 3 seconds. The most common causes: +```sh +simdeck daemon restart --stream-quality low +``` -- **VideoToolbox is busy.** macOS screen recording can starve the hardware H.264 encoder. Auto mode detects sustained hardware encode overload and temporarily falls back to software H.264. For a fully software-only run, start with: +Check metrics: - ```sh - simdeck daemon stop - simdeck daemon start --video-codec software - ``` +```sh +curl http://127.0.0.1:4310/api/metrics +``` + +If `frames_dropped_server` keeps climbing, the client or network cannot keep up. Move closer to the host, reduce quality, or switch to software encoding. - On virtualized CI Macs where hardware H.264 is unavailable, use - `--video-codec software --stream-quality ci-software`. That profile - targets a 960-pixel longest edge at 24 fps and lowers bitrate/CPU pressure - before backlog turns into visible stream delay. +### Browser cannot establish WebRTC -- **The Simulator window is minimised or off-screen.** The private display bridge captures from a headless context, so this is rare, but if you see it after waking from sleep, shut the simulator down and boot it again. -- **The simulator is mid-shutdown.** Wait for `simdeck list` to report `isBooted: true`. +Force the H.264 WebSocket fallback while testing: -### Frequent stutter or "Refresh stream" loops +```text +http://127.0.0.1:4310?stream=h264 +``` -The transport hub forces a keyframe whenever a client falls behind. If `frames_dropped_server` on `/api/metrics` climbs steadily, the bottleneck is between the encoder and the decoder. +For routed remote sessions, configure TURN as described in [Video & Streaming](/guide/video#remote-browsers). -- Bring the client closer (LAN with low latency vs Wi-Fi mesh hops). -- Check `client_streams` in `/api/metrics`. If `decodedFps` is much lower than `packetFps`, the client decoder is the bottleneck. +## Inspector Looks Wrong -## Inspector returns AX instead of NativeScript / UIKit +### `describe` returns accessibility instead of framework data -The accessibility tree endpoint blends three inspector sources and falls back to AX snapshot when none of the others are reachable. The response includes both a `source` field and a `fallbackReason` field that explains what happened. +The fallback is expected when no in-app inspector is available. Check: -Common reasons: +- The app with the inspector is foregrounded. +- The app was built in debug mode. +- The inspector package starts before the app UI boots. +- The app is pointing at the active SimDeck port. -| `fallbackReason` | Fix | -| --------------------------------------------------------- | ---------------------------------------------------------------------------------------- | -| `The in-app inspector process is not the foreground app.` | Bring the inspector-enabled app to the foreground. | -| `NativeScript hierarchy is not published by the app.` | Make sure the app calls `startSimDeckInspector(...)` before bootstrapping. | -| `No connected NativeScript inspector ...` | The NativeScript inspector hasn't completed its WebSocket handshake yet. Reload the app. | -| `No in-app inspector found ... on ports 47370-47402` | The Swift agent isn't listening; confirm the app links and starts the agent in DEBUG. | +Use a forced source to see the failure reason: -For more on the inspector matrix, see the [Inspector Overview](/inspector/). +```sh +simdeck describe --source nativescript +simdeck describe --source react-native +simdeck describe --source flutter +simdeck describe --source uikit +``` + +### NativeScript inspector does not connect + +- Call `startSimDeckInspector({ port: 4310 })` before bootstrap. +- For Angular, call it before `runNativeScriptAngularApp(...)`. +- Confirm the simulator app can reach `http://127.0.0.1:4310/api/health`. + +### React Native source locations are missing + +Use a development build. Production bundles usually strip React debug source metadata. + +### Flutter source locations are missing + +Run a debug build with widget creation tracking. Flutter enables this by default for normal debug runs. + +## LAN Browser Cannot Connect + +Start SimDeck with a LAN bind and reachable advertised host: + +```sh +simdeck ui --bind 0.0.0.0 --advertise-host 192.168.1.50 --open +``` -## NativeScript inspector won't connect +Then check: -- Confirm `startSimDeckInspector({ port: 4310 })` runs in the simulator app's main thread before bootstrap. -- Confirm the simulator can reach the host: from inside the app, `fetch('http://127.0.0.1:4310/api/health')` should succeed. -- For Angular apps, make sure `startSimDeckInspector(...)` runs **before** `runNativeScriptAngularApp(...)`. -- Watch the server log for messages such as `Registered NativeScript inspector for process …`. If you don't see one, the WebSocket never completed. +- The remote browser opens `http://192.168.1.50:4310`. +- macOS Firewall allows the port. +- The pairing code matches the current daemon. +- API scripts send the daemon token. -## Logs +See [LAN Access](/guide/lan-access). -When all else fails, capture the server log: +## Logs To Include In Issues -- Dev server: read `build/cli.log` when using `npm run dev`. -- Project daemon: stop and restart it from a terminal when you need foreground logs for a reproduction. +Include: -Include both files when filing an issue, along with `simdeck --version`, the macOS version, and the Xcode version. +- `simdeck --version` +- macOS version +- Xcode version +- The command you ran +- Foreground daemon output, or `build/cli.log` when using `npm run dev` +- `simdeck daemon status` without sharing the token publicly diff --git a/docs/guide/video.md b/docs/guide/video.md index a2a0548d..4b2a96d5 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -1,207 +1,123 @@ -# Video Pipeline +# Video & Streaming -SimDeck streams the iOS Simulator over WebRTC using browser-native H.264 video playout, with H.264 over WebSocket as the fallback for Safari and networks where peer media negotiation fails. This page walks through the encoder choices, fallback transport, keyframe handshake, and metrics you can use to tune them. +SimDeck streams live device video to the browser. Local sessions default to high quality. Remote or constrained sessions can trade detail for lower CPU and latency. -## Codec selection +## Pick A Stream Quality -The server can encode the simulator display in three modes, picked at startup with `--video-codec`: +Start with the default: -| Value | Encoder | When to use it | -| ------------------ | ------------------------------------ | ------------------------------------------------------------------------------------------- | -| `auto` _(default)_ | VideoToolbox chooses the encoder | Normal local and remote preview. Falls back to software when hardware encode is overloaded. | -| `hardware` | Required hardware H.264 | Use only when the hardware encoder is known to be available. | -| `software` | Software-only H.264 via VideoToolbox | Use when hardware encode stalls, is unavailable, or must be avoided. | +```sh +simdeck +``` -Restart the daemon to change encoder mode: +Lower quality when the stream stutters, the machine is under load, or you are using a remote browser: ```sh -simdeck daemon restart --video-codec software +simdeck daemon restart --stream-quality low +simdeck daemon restart --stream-quality tiny +simdeck daemon restart --stream-quality ci-software ``` -For slower runners, add `--low-latency` with software H.264: +Common profiles: + +| Profile | Use it for | +| ------------- | --------------------------------------- | +| `full` | Local browser on a fast Mac | +| `balanced` | Good local quality with less bandwidth | +| `economy` | Remote browser or busy machine | +| `low` | Slower Wi-Fi or shared hosts | +| `tiny` | Pull request previews and low bandwidth | +| `ci-software` | Virtualized CI Macs | + +The browser also has stream controls for transport, resolution, FPS, and refresh. + +## Pick A Codec ```sh -simdeck daemon start --video-codec software --low-latency +simdeck daemon restart --video-codec auto +simdeck daemon restart --video-codec hardware +simdeck daemon restart --video-codec software ``` -Low-latency mode caps software H.264 at 15 fps, keeps a single in-flight frame, -scales the longest edge to 1170 pixels, and backs off FPS more aggressively when -encode pressure rises. WebRTC refresh pacing uses the same 15 fps floor so the -server does not keep waking capture/encode faster than the stream can consume. -It is CLI-only because it is meant for less capable machines where freshness -matters more than maximum smoothness. - -The requested encoder mode is reported to clients in the JSON `videoCodec` field on `GET /api/health`. -In `auto`, active simulator encoder metrics also report `activeEncoderMode`. -When the hardware encoder is overloaded, `auto` rebuilds the active compression -session as software H.264, then periodically retries hardware and stays there -when the measured encode latency returns to budget. -If all current browser viewers for a simulator are backgrounded or unfocused, -the active session also switches to software H.264 until at least one viewer is -foreground again. -The browser UI exposes stream controls for encoder, FPS, transport, and resolution. H264 resolution choices are `full` (4096 px at 60 fps), `balanced` (1280 px at 60 fps), `economy` (1080 px at 30 fps), `low` (720 px at 30 fps), and `tiny` (540 px at 30 fps). Local H264 WebSocket sessions default to full resolution at 60 fps. Remote browser sessions default to software H.264, 30 fps, and adaptive quality. - -## Remote WebRTC ICE - -By default SimDeck advertises Google's public STUN server to both the Rust -WebRTC peer and the browser. For remote browsers, especially Safari connecting -to a GitHub Actions runner, provide a TURN server and force relay candidates: +| Codec | Use it for | +| ---------- | --------------------------------------------------------------------- | +| `auto` | Normal use. SimDeck can move between hardware and software as needed. | +| `hardware` | Dedicated local machines where hardware H.264 is reliable. | +| `software` | CI, screen recording conflicts, or hardware encoder stalls. | + +For very constrained software sessions: ```sh -SIMDECK_WEBRTC_ICE_SERVERS=turns:turn.example.com:5349?transport=tcp \ -SIMDECK_WEBRTC_ICE_USERNAME=simdeck \ -SIMDECK_WEBRTC_ICE_CREDENTIAL=secret \ -SIMDECK_WEBRTC_ICE_TRANSPORT_POLICY=relay \ -simdeck daemon start --video-codec software --low-latency +simdeck daemon restart --video-codec software --low-latency ``` -The browser reads these settings from `GET /api/health` before creating its -peer connection, so the local and remote peers use the same ICE configuration. -Use `SIMDECK_WEBRTC_ICE_TRANSPORT_POLICY=all` or leave it unset for local LAN -and localhost sessions. +## WebRTC And Fallback -## H264 WebSocket fallback +The browser tries WebRTC first. If WebRTC cannot render a frame, the UI can fall back to H.264 over WebSocket when the browser supports WebCodecs. -The browser UI defaults to `?stream=auto`: it tries WebRTC first and falls back -to H264 over WebSocket if a decoded frame still does not render. -For remote browser sessions, SimDeck also falls back immediately when the -browser's WebRTC offer contains no local `host` ICE candidates, which covers -Safari privacy/network settings that suppress direct candidates. The stream -settings menu includes a transport picker for Auto, WebRTC, and H264 WS. You -can also force a mode while testing: +Force a mode while debugging: ```text http://127.0.0.1:4310?stream=webrtc http://127.0.0.1:4310?stream=h264 ``` -H264 WS uses the same native H.264 encoder as WebRTC, but sends each encoded -sample on a binary WebSocket at: - -```http -GET /api/simulators/{udid}/h264?profile=full&fps=60&videoCodec=auto -``` - -Each message starts with a compact SimDeck header, followed by optional AVC -decoder config and the encoded sample bytes. The browser decodes with -WebCodecs, keeps only the latest decoded frame, and paints on -`requestAnimationFrame` so stale frames do not build latency. Input stays on -the separate `/api/simulators/{udid}/input` WebSocket so large video frames do -not block touch and keyboard messages. Stream-quality updates and client stats -use the WebRTC data channel or the H264 WS video socket instead of repeatedly -posting REST requests. H264 WS defaults to the `full` profile on loopback and -`auto` quality for remote sessions. H264 `Auto` starts at `full` on loopback; -remote `Auto` starts lower but can climb through the internal `smooth` step -(1170 px), `balanced`, and `full` after sustained low decode/render pressure. - -```http -GET /api/simulators/{udid}/input -``` - -That WebSocket accepts the same normalized control JSON used by the WebRTC data -channel and coalesces high-frequency touch `moved` events. - -## Keyframe handshake +## Remote Browsers -When a browser connects through `/api/simulators/{udid}/webrtc/offer`: +For another browser on the same network, see [LAN Access](/guide/lan-access). -1. The server ensures the `SimulatorSession` is started and asks the encoder for an immediate refresh. -2. It waits up to 3 seconds for the next keyframe. -3. As soon as a keyframe arrives, it answers the browser's SDP offer and starts writing H.264 samples to a WebRTC video track. -4. Subsequent frames stream until the peer connection closes. +For routed remote access, use a tunnel or relay you trust. If your network requires TURN for WebRTC, set these before starting SimDeck: -If the encoder cannot deliver a keyframe within 3 seconds, the server tears the session down with a clear error so the client can retry. This usually happens only when CoreSimulator is itself stuck. - -## Drop and lag handling +```sh +SIMDECK_WEBRTC_ICE_SERVERS=turns:turn.example.com:5349?transport=tcp \ +SIMDECK_WEBRTC_ICE_USERNAME=simdeck \ +SIMDECK_WEBRTC_ICE_CREDENTIAL=secret \ +SIMDECK_WEBRTC_ICE_TRANSPORT_POLICY=relay \ +simdeck daemon start --video-codec software --stream-quality low +``` -The transport hub uses a tokio broadcast channel to fan out frames. If a slow client misses frames the hub: +## Stream Diagnostics -1. Increments `frames_dropped_server` on the metrics counter. -2. Sets a "waiting for keyframe" flag and skips non-keyframes until a fresh one arrives. -3. Calls `request_refresh()` on the session so the encoder forces a keyframe. +Check health: -The WebRTC path favors freshness: stale frames are dropped and the sender requests a new keyframe after discontinuities. +```sh +curl http://127.0.0.1:4310/api/health +``` -## Picking a codec +Check counters: -A few practical guidelines: +```sh +curl http://127.0.0.1:4310/api/metrics +``` -- **Start on the default for local preview.** Browser realtime mode uses VideoToolbox H.264 with full resolution at 60 fps. Auto mode moves active sessions to software when hardware H.264 is overloaded, then retries hardware after a cooldown. -- **Backgrounded web clients use software.** The browser sends page visibility/focus over the stream control channel. If every active viewer for a simulator is hidden or unfocused, the native session uses software H.264 until a viewer returns foreground. -- **Use `--local-stream-fps` above 60 only for local high-refresh testing.** The local quality stream defaults to 60 fps; higher targets pace both capture refresh and hardware encode submission so the stream does not build delay by pushing unbounded frames. -- **Switch to `software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency. -- **Studio providers default to software H.264 plus `--stream-quality smooth`.** `smooth` is an internal/provider profile, not a browser picker item. It uses a 1170-pixel longest edge, allows up to 60 fps, raises the bitrate budget to reduce compression artifacts, and lets multiple provider sessions share CPU cores without depending on one hardware encoder. -- **The remote browser renders WebRTC as a native `