Add auto software fallback when the SimDeck client loses foreground

cli/XCWH264Encoder.h

@@ -20,6 +20,7 @@ typedef void (^XCWH264EncoderOutputHandler)(NSData *sampleData,
 - (void)encodePixelBuffer:(CVPixelBufferRef)pixelBuffer;
 - (void)requestKeyFrame;
 - (void)reconfigureForStreamQualityChange;
+- (void)setClientForeground:(BOOL)foreground;
 - (NSDictionary *)statsRepresentation;
 - (void)invalidate;
 

cli/XCWPrivateSimulatorSession.h

@@ -31,6 +31,7 @@ typedef void (^XCWPrivateSimulatorEncodedFrameHandler)(NSData *sampleData,
 - (void)requestKeyFrameRefresh;
 - (void)requestFrameRefresh;
 - (void)reconfigureVideoEncoder;
+- (void)setClientForeground:(BOOL)foreground;
 - (NSDictionary *)videoEncoderStats;
 - (id)addEncodedFrameListener:(XCWPrivateSimulatorEncodedFrameHandler)handler;
 - (void)removeEncodedFrameListener:(id)token;

cli/XCWPrivateSimulatorSession.m

@@ -177,6 +177,14 @@ - (void)reconfigureVideoEncoder {
     [self refreshCurrentFrame];
 }
 
+- (void)setClientForeground:(BOOL)foreground {
+    [_videoEncoder setClientForeground:foreground];
+    if (foreground) {
+        [_videoEncoder requestKeyFrame];
+        [self refreshCurrentFrame];
+    }
+}
+
 - (NSDictionary *)videoEncoderStats {
     return [_videoEncoder statsRepresentation];
 }

cli/native/XCWNativeBridge.h

@@ -76,6 +76,7 @@ bool xcw_native_session_start(void * _Nonnull handle, char * _Nullable * _Nullab
 void xcw_native_session_request_refresh(void * _Nonnull handle);
 void xcw_native_session_request_keyframe(void * _Nonnull handle);
 void xcw_native_session_reconfigure_video_encoder(void * _Nonnull handle);
+void xcw_native_session_set_client_foreground(void * _Nonnull handle, bool foreground);
 char * _Nullable xcw_native_session_video_encoder_stats(void * _Nonnull handle, char * _Nullable * _Nullable error_message);
 int32_t xcw_native_session_rotation_quarter_turns(void * _Nonnull handle);
 bool xcw_native_session_send_touch(void * _Nonnull handle, double x, double y, const char * _Nonnull phase, char * _Nullable * _Nullable error_message);

cli/native/XCWNativeBridge.m

@@ -937,6 +937,12 @@ void xcw_native_session_reconfigure_video_encoder(void *handle) {
     }
 }
 
+void xcw_native_session_set_client_foreground(void *handle, bool foreground) {
+    @autoreleasepool {
+        [XCWNativeSessionFromHandle(handle) setClientForeground:foreground];
+    }
+}
+
 char *xcw_native_session_video_encoder_stats(void *handle, char **error_message) {
     @autoreleasepool {
         return XCWJSONStringFromObject([XCWNativeSessionFromHandle(handle) videoEncoderStats] ?: @{}, error_message);

cli/native/XCWNativeSession.h

@@ -15,6 +15,7 @@ NS_ASSUME_NONNULL_BEGIN
 - (void)requestRefresh;
 - (void)requestKeyFrame;
 - (void)reconfigureVideoEncoder;
+- (void)setClientForeground:(BOOL)foreground;
 - (NSDictionary *)videoEncoderStats;
 - (NSInteger)rotationQuarterTurns;
 - (BOOL)sendTouchAtX:(double)x

cli/native/XCWNativeSession.m

@@ -102,6 +102,10 @@ - (void)reconfigureVideoEncoder {
     [self.session reconfigureVideoEncoder];
 }
 
+- (void)setClientForeground:(BOOL)foreground {
+    [self.session setClientForeground:foreground];
+}
+
 - (NSDictionary *)videoEncoderStats {
     return [self.session videoEncoderStats];
 }

client/src/api/types.ts

@@ -1,6 +1,12 @@
 export interface EncoderStats {
+  activeEncoderMode?: string;
   averageEncodeLatencyUs?: number;
   averageEncoderLoadPercent?: number;
+  autoHardwareRetries?: number;
+  autoSoftwareFallbackActive?: boolean;
+  autoSoftwareFallbackRemainingUs?: number;
+  autoSoftwareFallbacks?: number;
+  clientForeground?: boolean;
   consecutiveOverBudgetFrames?: number;
   encoderBudgetUs?: number;
   encoderLoadPercent?: number;

client/src/features/stream/streamWorkerClient.ts

@@ -86,7 +86,9 @@ export function sendStreamClientStats(stats: unknown): boolean {
 }
 
 export function sendWebRtcStreamControl(options: {
+  clientId?: string;
   forceKeyframe?: boolean;
+  foreground?: boolean;
   snapshot?: boolean;
 }): boolean {
   return sendDataChannelMessage(
@@ -3018,7 +3020,12 @@ export class StreamWorkerClient {
     return (await this.backend?.collectVisualArtifactSample?.(udid)) ?? null;
   }
 
-  sendStreamControl(options: { forceKeyframe?: boolean; snapshot?: boolean }) {
+  sendStreamControl(options: {
+    clientId?: string;
+    forceKeyframe?: boolean;
+    foreground?: boolean;
+    snapshot?: boolean;
+  }) {
     return Boolean(
       this.backend?.sendControl?.({ ...options, type: "streamControl" }),
     );

client/src/features/stream/useLiveStream.ts

@@ -99,6 +99,10 @@ function currentClientBundle(): string {
   );
 }
 
+function isDocumentForeground(): boolean {
+  return document.visibilityState === "visible" && document.hasFocus();
+}
+
 export function useLiveStream({
   canvasElement,
   paused = false,
@@ -346,6 +350,43 @@ export function useLiveStream({
     streamTransport,
   ]);
 
+  useEffect(() => {
+    if (!simulator?.udid || paused) {
+      return;
+    }
+
+    const sendForegroundState = (foreground = isDocumentForeground()) => {
+      workerClientRef.current?.sendStreamControl({
+        clientId: clientTelemetryIdRef.current,
+        foreground,
+      });
+    };
+
+    const sendCurrentForegroundState = () => {
+      sendForegroundState();
+    };
+    const sendBackgroundState = () => {
+      sendForegroundState(false);
+    };
+
+    sendCurrentForegroundState();
+    document.addEventListener("visibilitychange", sendCurrentForegroundState);
+    window.addEventListener("focus", sendCurrentForegroundState);
+    window.addEventListener("blur", sendCurrentForegroundState);
+    window.addEventListener("pageshow", sendCurrentForegroundState);
+    window.addEventListener("pagehide", sendBackgroundState);
+    return () => {
+      document.removeEventListener(
+        "visibilitychange",
+        sendCurrentForegroundState,
+      );
+      window.removeEventListener("focus", sendCurrentForegroundState);
+      window.removeEventListener("blur", sendCurrentForegroundState);
+      window.removeEventListener("pageshow", sendCurrentForegroundState);
+      window.removeEventListener("pagehide", sendBackgroundState);
+    };
+  }, [paused, simulator?.udid, streamTransport]);
+
   useEffect(() => {
     if (
       streamConfigApplyKey <= 0 ||
@@ -410,6 +451,10 @@ export function useLiveStream({
         visualSampleCount: latestVisualArtifactSampleCountRef.current,
         visibilityState: document.visibilityState,
       };
+      workerClientRef.current?.sendStreamControl({
+        clientId: clientTelemetryIdRef.current,
+        foreground: isDocumentForeground(),
+      });
       if (
         sendStreamClientStats(payload) ||
         remote ||

client/src/features/toolbar/DebugPanel.tsx

@@ -80,6 +80,16 @@ export function DebugPanel({
   if (encoder) {
     rows.push(
       { label: "Encoder", value: encoder.encoderMode ?? "—" },
+      { label: "Active Encoder", value: encoder.activeEncoderMode ?? "—" },
+      {
+        label: "Client Foreground",
+        value:
+          typeof encoder.clientForeground === "boolean"
+            ? encoder.clientForeground
+              ? "yes"
+              : "no"
+            : "—",
+      },
       { label: "Encoder State", value: encoder.overloadState ?? "—" },
       {
         label: "Encoder Load",

docs/api/health.md

@@ -56,7 +56,10 @@ Returns a snapshot of every server-side counter and the rolling buffer of client
     {
       "udid": "9D7E5BB7-...",
       "encoder": {
-        "encoderMode": "hardware",
+        "encoderMode": "auto",
+        "activeEncoderMode": "hardware",
+        "clientForeground": true,
+        "autoSoftwareFallbackActive": false,
         "hardwareAccelerated": true,
         "overloadState": "nominal",
         "averageEncoderLoadPercent": 42.1,
@@ -113,7 +116,14 @@ is derived from native VideoToolbox submit-to-output latency:
 
 This is an inferred pressure signal rather than a private macOS hardware queue
 counter. It is useful for deciding when to lower stream resolution/FPS or switch
-from hardware to software encoding.
+from hardware to software encoding. When `encoderMode` is `auto`, SimDeck uses
+this signal to temporarily rebuild the active session with software H.264 if the
+hardware encoder is overloaded, then retries hardware after a cooldown. The
+`activeEncoderMode`, `autoSoftwareFallbackActive`, `autoSoftwareFallbacks`, and
+`autoHardwareRetries` fields expose that state. `clientForeground` is `false`
+when all current browser viewers for the simulator are hidden or unfocused; in
+that state the native session uses software H.264 until a viewer returns
+foreground.
 
 ### Client stream stats
 

docs/api/rest.md

@@ -220,6 +220,14 @@ stream attached to that peer:
 { "type": "streamControl", "forceKeyframe": true }
 ```
 
+Clients can also report page focus/visibility through stream control. When all
+current viewers for a simulator report `foreground: false`, the native session
+uses software H.264 until a viewer returns foreground:
+
+```json
+{ "type": "streamControl", "clientId": "browser", "foreground": false }
+```
+
 ```json
 { "type": "streamQuality", "config": { "profile": "low", "fps": 30 } }
 ```
@@ -258,6 +266,10 @@ socket:
 { "type": "streamControl", "forceKeyframe": true }
 ```
 
+```json
+{ "type": "streamControl", "clientId": "browser", "foreground": false }
+```
+
 ```json
 { "type": "streamQuality", "config": { "profile": "low", "fps": 30 } }
 ```

docs/guide/troubleshooting.md

@@ -79,7 +79,7 @@ sudo xcode-select -s /Applications/Xcode.app
 
 The encoder did not produce a keyframe within 3 seconds. The most common causes:
 
-- **VideoToolbox is busy.** macOS screen recording can starve the hardware H.264 encoder. Switch to software H.264:
+- **VideoToolbox is busy.** macOS screen recording can starve the hardware H.264 encoder. Auto mode detects sustained hardware encode overload and temporarily falls back to software H.264. For a fully software-only run, start with:
 
   ```sh
   simdeck daemon stop

docs/guide/video.md

@@ -6,11 +6,11 @@ SimDeck streams the iOS Simulator over WebRTC using browser-native H.264 video p
 
 The server can encode the simulator display in three modes, picked at startup with `--video-codec`:
 
-| Value              | Encoder                              | When to use it                                                       |
-| ------------------ | ------------------------------------ | -------------------------------------------------------------------- |
-| `auto` _(default)_ | VideoToolbox chooses the encoder     | Normal local and remote preview. Does not require hardware encoding. |
-| `hardware`         | Required hardware H.264              | Use only when the hardware encoder is known to be available.         |
-| `software`         | Software-only H.264 via VideoToolbox | Use when hardware encode stalls, is unavailable, or must be avoided. |
+| Value              | Encoder                              | When to use it                                                                              |
+| ------------------ | ------------------------------------ | ------------------------------------------------------------------------------------------- |
+| `auto` _(default)_ | VideoToolbox chooses the encoder     | Normal local and remote preview. Falls back to software when hardware encode is overloaded. |
+| `hardware`         | Required hardware H.264              | Use only when the hardware encoder is known to be available.                                |
+| `software`         | Software-only H.264 via VideoToolbox | Use when hardware encode stalls, is unavailable, or must be avoided.                        |
 
 Restart the daemon to change encoder mode:
 
@@ -32,6 +32,13 @@ It is CLI-only because it is meant for less capable machines where freshness
 matters more than maximum smoothness.
 
 The requested encoder mode is reported to clients in the JSON `videoCodec` field on `GET /api/health`.
+In `auto`, active simulator encoder metrics also report `activeEncoderMode`.
+When the hardware encoder is overloaded, `auto` rebuilds the active compression
+session as software H.264, then periodically retries hardware and stays there
+when the measured encode latency returns to budget.
+If all current browser viewers for a simulator are backgrounded or unfocused,
+the active session also switches to software H.264 until at least one viewer is
+foreground again.
 The browser UI exposes stream controls for encoder, FPS, transport, and resolution. H264 resolution choices are `full` (4096 px at 60 fps), `balanced` (1280 px at 60 fps), `economy` (1080 px at 30 fps), `low` (720 px at 30 fps), and `tiny` (540 px at 30 fps). Local H264 WebSocket sessions default to full resolution at 60 fps. Remote browser sessions default to software H.264, 30 fps, and adaptive quality.
 
 ## Remote WebRTC ICE
@@ -119,7 +126,8 @@ The WebRTC path favors freshness: stale frames are dropped and the sender reques
 
 A few practical guidelines:
 
-- **Start on the default for local preview.** Browser realtime mode uses VideoToolbox H.264 with full resolution at 60 fps. Pass `--video-codec software` only when the shared hardware encoder is unavailable or performs worse on that host.
+- **Start on the default for local preview.** Browser realtime mode uses VideoToolbox H.264 with full resolution at 60 fps. Auto mode moves active sessions to software when hardware H.264 is overloaded, then retries hardware after a cooldown.
+- **Backgrounded web clients use software.** The browser sends page visibility/focus over the stream control channel. If every active viewer for a simulator is hidden or unfocused, the native session uses software H.264 until a viewer returns foreground.
 - **Use `--local-stream-fps` above 60 only for local high-refresh testing.** The local quality stream defaults to 60 fps; higher targets pace both capture refresh and hardware encode submission so the stream does not build delay by pushing unbounded frames.
 - **Switch to `software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency.
 - **Studio providers default to software H.264 plus `--stream-quality smooth`.** `smooth` is an internal/provider profile, not a browser picker item. It uses a 1170-pixel longest edge, allows up to 60 fps, raises the bitrate budget to reduce compression artifacts, and lets multiple provider sessions share CPU cores without depending on one hardware encoder.
@@ -161,7 +169,12 @@ active simulator session. `strained` means encode latency is approaching the
 active frame budget; `overloaded` means smoothed latency is over budget or
 multiple frames in a row exceeded the budget. For hardware H.264 this usually
 means the shared VideoToolbox encoder is saturated; lower resolution/FPS or
-switch to software H.264.
+switch to software H.264. In `auto`, `activeEncoderMode`,
+`autoSoftwareFallbackActive`, `autoSoftwareFallbacks`, and
+`autoHardwareRetries` show whether the active session is temporarily using
+software H.264 while the hardware path cools down. `clientForeground` shows
+whether any current browser viewer is visible and focused; when it is `false`,
+the active encoder is software regardless of the requested mode.
 
 Clients can also push their decoder/renderer stats back to the server. Browser
 clients normally send these over the WebRTC telemetry data channel or the H264

server/native_stubs.c

@@ -340,6 +340,11 @@ void xcw_native_session_reconfigure_video_encoder(void *handle) {
   (void)handle;
 }
 
+void xcw_native_session_set_client_foreground(void *handle, bool foreground) {
+  (void)handle;
+  (void)foreground;
+}
+
 char *xcw_native_session_video_encoder_stats(void *handle,
                                              char **error_message) {
   (void)handle;