From 652bcfb199d3fe4ace5855a6efdf1ece9b8ce195 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 12 May 2026 18:15:44 -0400 Subject: [PATCH 1/3] Add auto encoder fallback to software under load --- cli/XCWH264Encoder.m | 125 +++++++++++++++++++++++++++------- docs/api/health.md | 10 ++- docs/guide/troubleshooting.md | 2 +- docs/guide/video.md | 13 +++- 4 files changed, 121 insertions(+), 29 deletions(-) diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index f5cfbfa9..f1ab3878 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -42,6 +42,7 @@ static const double XCWEncoderStrainedLoadPercent = 85.0; static const double XCWEncoderOverloadedLoadPercent = 105.0; static const NSUInteger XCWEncoderConsecutiveOverBudgetFrameThreshold = 3; +static const uint64_t XCWAutoHardwareRetryIntervalUs = 10000000; typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) { XCWVideoEncoderModeAuto, @@ -507,6 +508,7 @@ @implementation XCWH264Encoder { OSType _scaledPixelFormat; BOOL _scalingActive; XCWVideoEncoderMode _encoderMode; + XCWVideoEncoderMode _activeEncoderMode; BOOL _lowLatencyMode; BOOL _realtimeStreamMode; CMVideoCodecType _codecType; @@ -533,6 +535,9 @@ @implementation XCWH264Encoder { uint64_t _hardwareFrameIntervalUs; uint64_t _lastHardwareSubmissionUs; NSUInteger _hardwarePacedFrameCount; + uint64_t _autoSoftwareFallbackUntilUs; + NSUInteger _autoSoftwareFallbackCount; + NSUInteger _autoHardwareRetryCount; NSString *_selectedEncoderID; NSInteger _lastSessionStatus; NSInteger _lastPrepareStatus; @@ -553,9 +558,10 @@ - (instancetype)initWithOutputHandler:(XCWH264EncoderOutputHandler)outputHandler _pendingLock = OS_UNFAIR_LOCK_INIT; _needsKeyFrame = YES; _encoderMode = XCWVideoEncoderModeFromEnvironment(); + _activeEncoderMode = _encoderMode; _lowLatencyMode = (_encoderMode == XCWVideoEncoderModeH264Software) && XCWLowLatencyModeFromEnvironment(); _realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode; - _codecType = XCWVideoCodecTypeForMode(_encoderMode); + _codecType = XCWVideoCodecTypeForMode(_activeEncoderMode); _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; _hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked]; return self; @@ -604,15 +610,17 @@ - (void)reconfigureForStreamQualityChange { dispatch_async(_queue, ^{ [self invalidateCompressionSessionLocked]; self->_encoderMode = XCWVideoEncoderModeFromEnvironment(); + self->_activeEncoderMode = self->_encoderMode; self->_lowLatencyMode = (self->_encoderMode == XCWVideoEncoderModeH264Software) && XCWLowLatencyModeFromEnvironment(); self->_realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || self->_lowLatencyMode; - self->_codecType = XCWVideoCodecTypeForMode(self->_encoderMode); + self->_codecType = XCWVideoCodecTypeForMode(self->_activeEncoderMode); self->_needsKeyFrame = YES; self->_softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; self->_softwarePacedFrameCount = 0; self->_softwareHealthyFrameCount = 0; self->_hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked]; self->_hardwarePacedFrameCount = 0; + self->_autoSoftwareFallbackUntilUs = 0; }); } @@ -653,6 +661,12 @@ - (NSDictionary *)statsRepresentation { ? @"average-latency-near-budget" : @"consecutive-frames-near-budget"; } + uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); + BOOL autoSoftwareFallbackActive = [self isAutoSoftwareFallbackActiveLocked]; + uint64_t autoSoftwareFallbackRemainingUs = autoSoftwareFallbackActive && + self->_autoSoftwareFallbackUntilUs > nowUs + ? self->_autoSoftwareFallbackUntilUs - nowUs + : 0; stats = @{ @"inputFrames": @(inputFrameCount), @"pendingReplacements": @(pendingReplacementCount), @@ -684,9 +698,14 @@ - (NSDictionary *)statsRepresentation { @"hardwarePacedFrames": @(self->_hardwarePacedFrameCount), @"transportCodec": XCWCodecName(self->_codecType), @"encoderMode": XCWVideoEncoderModeName(self->_encoderMode), + @"activeEncoderMode": XCWVideoEncoderModeName(self->_activeEncoderMode), + @"autoSoftwareFallbackActive": @(autoSoftwareFallbackActive), + @"autoSoftwareFallbackRemainingUs": @(autoSoftwareFallbackRemainingUs), + @"autoSoftwareFallbacks": @(self->_autoSoftwareFallbackCount), + @"autoHardwareRetries": @(self->_autoHardwareRetryCount), @"lowLatencyMode": @(self->_lowLatencyMode), @"realtimeStreamMode": @(self->_realtimeStreamMode), - @"encoderId": XCWVideoEncoderIDForMode(self->_encoderMode) ?: @"automatic", + @"encoderId": XCWVideoEncoderIDForMode(self->_activeEncoderMode) ?: @"automatic", @"selectedEncoderId": self->_selectedEncoderID ?: NSNull.null, @"hardwareAccelerated": @(self->_hardwareAccelerated), @"lastSessionStatus": @(self->_lastSessionStatus), @@ -760,11 +779,59 @@ - (uint64_t)initialHardwareFrameIntervalUsLocked { return _realtimeStreamMode ? XCWRealtimeFrameIntervalUs() : XCWLocalStreamFrameIntervalUs(); } +- (BOOL)isAutoSoftwareFallbackActiveLocked { + return _encoderMode == XCWVideoEncoderModeAuto && + _activeEncoderMode == XCWVideoEncoderModeH264Software; +} + +- (void)resetAutoFallbackLatencyStateLocked { + _latestEncodeLatencyUs = 0; + _averageEncodeLatencyUs = 0; + _peakEncodeLatencyUs = 0; + _consecutiveOverBudgetFrameCount = 0; + _consecutiveStrainedFrameCount = 0; + _wasOverloaded = NO; +} + +- (void)enterAutoSoftwareFallbackLockedAtTimeUs:(uint64_t)nowUs { + if (_encoderMode != XCWVideoEncoderModeAuto || + _activeEncoderMode == XCWVideoEncoderModeH264Software) { + return; + } + _activeEncoderMode = XCWVideoEncoderModeH264Software; + _codecType = XCWVideoCodecTypeForMode(_activeEncoderMode); + _autoSoftwareFallbackUntilUs = nowUs + XCWAutoHardwareRetryIntervalUs; + _autoSoftwareFallbackCount += 1; + _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; + _softwareHealthyFrameCount = 0; + _softwarePacedFrameCount = 0; + [self invalidateCompressionSessionLocked]; + [self resetAutoFallbackLatencyStateLocked]; + _needsKeyFrame = YES; +} + +- (void)retryAutoHardwareIfNeededLockedAtTimeUs:(uint64_t)nowUs { + if (![self isAutoSoftwareFallbackActiveLocked] || + _autoSoftwareFallbackUntilUs == 0 || + nowUs < _autoSoftwareFallbackUntilUs) { + return; + } + _activeEncoderMode = XCWVideoEncoderModeAuto; + _codecType = XCWVideoCodecTypeForMode(_activeEncoderMode); + _autoSoftwareFallbackUntilUs = 0; + _autoHardwareRetryCount += 1; + _hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked]; + _hardwarePacedFrameCount = 0; + [self invalidateCompressionSessionLocked]; + [self resetAutoFallbackLatencyStateLocked]; + _needsKeyFrame = YES; +} + - (uint64_t)activeFrameIntervalUsLocked { - if (_encoderMode == XCWVideoEncoderModeH264Software) { + if (_activeEncoderMode == XCWVideoEncoderModeH264Software) { return _softwareFrameIntervalUs > 0 ? _softwareFrameIntervalUs : [self initialSoftwareFrameIntervalUsLocked]; } - if (_encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware) { + if (_activeEncoderMode == XCWVideoEncoderModeAuto || _activeEncoderMode == XCWVideoEncoderModeH264Hardware) { return _hardwareFrameIntervalUs > 0 ? _hardwareFrameIntervalUs : [self initialHardwareFrameIntervalUsLocked]; } int32_t expectedFrameRate = MAX(1, [self expectedFrameRateLocked]); @@ -772,7 +839,7 @@ - (uint64_t)activeFrameIntervalUsLocked { } - (int32_t)expectedFrameRateLocked { - if (_encoderMode == XCWVideoEncoderModeH264Software) { + if (_activeEncoderMode == XCWVideoEncoderModeH264Software) { if (_lowLatencyMode) { return XCWTargetLowLatencySoftwareFrameRate; } @@ -785,7 +852,7 @@ - (int32_t)expectedFrameRateLocked { } - (BOOL)shouldPaceHardwareFrameAtTimeUs:(uint64_t)nowUs { - if ((_encoderMode != XCWVideoEncoderModeAuto && _encoderMode != XCWVideoEncoderModeH264Hardware) || _needsKeyFrame) { + if ((_activeEncoderMode != XCWVideoEncoderModeAuto && _activeEncoderMode != XCWVideoEncoderModeH264Hardware) || _needsKeyFrame) { return NO; } if (_realtimeStreamMode) { @@ -806,7 +873,7 @@ - (BOOL)shouldPaceHardwareFrameAtTimeUs:(uint64_t)nowUs { } - (BOOL)shouldPaceSoftwareFrameAtTimeUs:(uint64_t)nowUs { - if (_encoderMode != XCWVideoEncoderModeH264Software || _needsKeyFrame) { + if (_activeEncoderMode != XCWVideoEncoderModeH264Software || _needsKeyFrame) { return NO; } if (_softwareFrameIntervalUs == 0) { @@ -824,7 +891,7 @@ - (BOOL)shouldPaceSoftwareFrameAtTimeUs:(uint64_t)nowUs { } - (void)adaptSoftwarePacingForLatencyUs:(uint64_t)latencyUs { - if (_encoderMode != XCWVideoEncoderModeH264Software || !_lowLatencyMode || latencyUs == 0) { + if (_activeEncoderMode != XCWVideoEncoderModeH264Software || !_lowLatencyMode || latencyUs == 0) { return; } if (_softwareFrameIntervalUs == 0) { @@ -892,7 +959,10 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer { return NO; } - CGSize targetSize = XCWScaledDimensionsForSourceSize(sourceWidth, sourceHeight, _encoderMode, _lowLatencyMode, _realtimeStreamMode); + uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); + [self retryAutoHardwareIfNeededLockedAtTimeUs:nowUs]; + + CGSize targetSize = XCWScaledDimensionsForSourceSize(sourceWidth, sourceHeight, _activeEncoderMode, _lowLatencyMode, _realtimeStreamMode); int32_t targetWidth = (int32_t)targetSize.width; int32_t targetHeight = (int32_t)targetSize.height; if (targetWidth <= 0 || targetHeight <= 0) { @@ -900,7 +970,6 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer { } _scalingActive = sourceWidth != targetWidth || sourceHeight != targetHeight; - uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); if ([self shouldPaceSoftwareFrameAtTimeUs:nowUs] || [self shouldPaceHardwareFrameAtTimeUs:nowUs]) { return YES; } @@ -946,13 +1015,13 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer { _inFlightFrameCount += 1; _submittedFrameCount += 1; - if (_encoderMode == XCWVideoEncoderModeH264Software) { + if (_activeEncoderMode == XCWVideoEncoderModeH264Software) { _lastSoftwareSubmissionUs = nowUs; - } else if (_encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware) { + } else if (_activeEncoderMode == XCWVideoEncoderModeAuto || _activeEncoderMode == XCWVideoEncoderModeH264Hardware) { _lastHardwareSubmissionUs = nowUs; } _maxInFlightFrameCount = MAX(_maxInFlightFrameCount, _inFlightFrameCount); - if (_encoderMode == XCWVideoEncoderModeH264Software || !_realtimeStreamMode) { + if (_activeEncoderMode == XCWVideoEncoderModeH264Software || !_realtimeStreamMode) { VTCompressionSessionCompleteFrames(_compressionSession, presentationTime); } return YES; @@ -966,20 +1035,20 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height [self invalidateCompressionSessionLocked]; NSMutableDictionary *encoderSpecification = [NSMutableDictionary dictionary]; - NSString *encoderID = XCWVideoEncoderIDForMode(_encoderMode); + NSString *encoderID = XCWVideoEncoderIDForMode(_activeEncoderMode); if (encoderID.length > 0) { encoderSpecification[(__bridge NSString *)kVTVideoEncoderSpecification_EncoderID] = encoderID; } - if (_encoderMode != XCWVideoEncoderModeH264Software && (_lowLatencyMode || _realtimeStreamMode)) { + if (_activeEncoderMode != XCWVideoEncoderModeH264Software && (_lowLatencyMode || _realtimeStreamMode)) { if (@available(macOS 11.3, *)) { encoderSpecification[(__bridge NSString *)kVTVideoEncoderSpecification_EnableLowLatencyRateControl] = @YES; } } - if (_encoderMode == XCWVideoEncoderModeH264Software) { + if (_activeEncoderMode == XCWVideoEncoderModeH264Software) { encoderSpecification[(__bridge NSString *)kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder] = @NO; - } else if (_encoderMode == XCWVideoEncoderModeH264Hardware) { + } else if (_activeEncoderMode == XCWVideoEncoderModeH264Hardware) { encoderSpecification[(__bridge NSString *)kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder] = @YES; - } else if (_encoderMode == XCWVideoEncoderModeAuto && _realtimeStreamMode) { + } else if (_activeEncoderMode == XCWVideoEncoderModeAuto && _realtimeStreamMode) { encoderSpecification[(__bridge NSString *)kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder] = @YES; } @@ -1006,7 +1075,7 @@ - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height _needsKeyFrame = YES; int expectedFrameRate = [self expectedFrameRateLocked]; - int averageBitRate = XCWAverageBitRateForDimensions(width, height, _encoderMode, _lowLatencyMode, _realtimeStreamMode); + int averageBitRate = XCWAverageBitRateForDimensions(width, height, _activeEncoderMode, _lowLatencyMode, _realtimeStreamMode); VTSessionSetProperty(session, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue); if (@available(macOS 10.14, *)) { @@ -1123,7 +1192,7 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix if (_pixelTransferSession == NULL) { OSStatus sessionStatus = VTPixelTransferSessionCreate(kCFAllocatorDefault, &_pixelTransferSession); if (sessionStatus != noErr || _pixelTransferSession == NULL) { - if (_encoderMode == XCWVideoEncoderModeH264Software) { + if (_activeEncoderMode == XCWVideoEncoderModeH264Software) { return [self copySoftwareScaledPixelBuffer:pixelBuffer targetWidth:targetWidth targetHeight:targetHeight]; @@ -1155,7 +1224,7 @@ - (nullable CVPixelBufferRef)copyScaledPixelBufferIfNeeded:(CVPixelBufferRef)pix _lastScaleStatus = transferStatus; if (transferStatus != noErr) { CVPixelBufferRelease(scaledPixelBuffer); - if (_encoderMode == XCWVideoEncoderModeH264Software) { + if (_activeEncoderMode == XCWVideoEncoderModeH264Software) { return [self copySoftwareScaledPixelBuffer:pixelBuffer targetWidth:targetWidth targetHeight:targetHeight]; @@ -1177,7 +1246,7 @@ - (BOOL)shouldUseSoftwareScalerForSourceWidth:(int32_t)sourceWidth if (sourceWidth == targetWidth && sourceHeight == targetHeight) { return NO; } - return _encoderMode == XCWVideoEncoderModeAuto || _encoderMode == XCWVideoEncoderModeH264Hardware; + return _activeEncoderMode == XCWVideoEncoderModeAuto || _activeEncoderMode == XCWVideoEncoderModeH264Hardware; } - (nullable CVPixelBufferRef)copySoftwareScaledPixelBuffer:(CVPixelBufferRef)pixelBuffer @@ -1315,8 +1384,11 @@ - (void)handleEncodedSampleBuffer:(CMSampleBufferRef)sampleBuffer if (isKeyFrame) { _keyFrameOutputCount += 1; } + BOOL shouldEnterAutoSoftwareFallback = NO; + uint64_t measurementTimeUs = 0; if (submittedAtUs > 0) { uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); + measurementTimeUs = nowUs; _latestEncodeLatencyUs = nowUs >= submittedAtUs ? nowUs - submittedAtUs : 0; _peakEncodeLatencyUs = MAX(_peakEncodeLatencyUs, _latestEncodeLatencyUs); _averageEncodeLatencyUs = _averageEncodeLatencyUs <= 0.0 @@ -1346,6 +1418,10 @@ - (void)handleEncodedSampleBuffer:(CMSampleBufferRef)sampleBuffer _overloadEventCount += 1; } _wasOverloaded = overloaded; + shouldEnterAutoSoftwareFallback = overloaded && + _encoderMode == XCWVideoEncoderModeAuto && + _activeEncoderMode != XCWVideoEncoderModeH264Software && + _hardwareAccelerated; [self adaptSoftwarePacingForLatencyUs:_latestEncodeLatencyUs]; } NSString *codec = nil; @@ -1400,6 +1476,9 @@ - (void)handleEncodedSampleBuffer:(CMSampleBufferRef)sampleBuffer CGSize dimensions = CGSizeMake(_width, _height); self.outputHandler(sampleData, timestampUs, isKeyFrame, codec, decoderConfig, dimensions); + if (shouldEnterAutoSoftwareFallback) { + [self enterAutoSoftwareFallbackLockedAtTimeUs:measurementTimeUs]; + } } - (void)completeInFlightFrame { diff --git a/docs/api/health.md b/docs/api/health.md index aa8eaf84..1ff4d8fe 100644 --- a/docs/api/health.md +++ b/docs/api/health.md @@ -56,7 +56,9 @@ Returns a snapshot of every server-side counter and the rolling buffer of client { "udid": "9D7E5BB7-...", "encoder": { - "encoderMode": "hardware", + "encoderMode": "auto", + "activeEncoderMode": "hardware", + "autoSoftwareFallbackActive": false, "hardwareAccelerated": true, "overloadState": "nominal", "averageEncoderLoadPercent": 42.1, @@ -113,7 +115,11 @@ is derived from native VideoToolbox submit-to-output latency: This is an inferred pressure signal rather than a private macOS hardware queue counter. It is useful for deciding when to lower stream resolution/FPS or switch -from hardware to software encoding. +from hardware to software encoding. When `encoderMode` is `auto`, SimDeck uses +this signal to temporarily rebuild the active session with software H.264 if the +hardware encoder is overloaded, then retries hardware after a cooldown. The +`activeEncoderMode`, `autoSoftwareFallbackActive`, `autoSoftwareFallbacks`, and +`autoHardwareRetries` fields expose that state. ### Client stream stats diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index b885accf..0961b0c3 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -79,7 +79,7 @@ sudo xcode-select -s /Applications/Xcode.app The encoder did not produce a keyframe within 3 seconds. The most common causes: -- **VideoToolbox is busy.** macOS screen recording can starve the hardware H.264 encoder. Switch to software H.264: +- **VideoToolbox is busy.** macOS screen recording can starve the hardware H.264 encoder. Auto mode detects sustained hardware encode overload and temporarily falls back to software H.264. For a fully software-only run, start with: ```sh simdeck daemon stop diff --git a/docs/guide/video.md b/docs/guide/video.md index a20989f5..cba383f8 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -8,7 +8,7 @@ The server can encode the simulator display in three modes, picked at startup wi | Value | Encoder | When to use it | | ------------------ | ------------------------------------ | -------------------------------------------------------------------- | -| `auto` _(default)_ | VideoToolbox chooses the encoder | Normal local and remote preview. Does not require hardware encoding. | +| `auto` _(default)_ | VideoToolbox chooses the encoder | Normal local and remote preview. Falls back to software when hardware encode is overloaded. | | `hardware` | Required hardware H.264 | Use only when the hardware encoder is known to be available. | | `software` | Software-only H.264 via VideoToolbox | Use when hardware encode stalls, is unavailable, or must be avoided. | @@ -32,6 +32,10 @@ It is CLI-only because it is meant for less capable machines where freshness matters more than maximum smoothness. The requested encoder mode is reported to clients in the JSON `videoCodec` field on `GET /api/health`. +In `auto`, active simulator encoder metrics also report `activeEncoderMode`. +When the hardware encoder is overloaded, `auto` rebuilds the active compression +session as software H.264, then periodically retries hardware and stays there +when the measured encode latency returns to budget. The browser UI exposes stream controls for encoder, FPS, transport, and resolution. H264 resolution choices are `full` (4096 px at 60 fps), `balanced` (1280 px at 60 fps), `economy` (1080 px at 30 fps), `low` (720 px at 30 fps), and `tiny` (540 px at 30 fps). Local H264 WebSocket sessions default to full resolution at 60 fps. Remote browser sessions default to software H.264, 30 fps, and adaptive quality. ## Remote WebRTC ICE @@ -119,7 +123,7 @@ The WebRTC path favors freshness: stale frames are dropped and the sender reques A few practical guidelines: -- **Start on the default for local preview.** Browser realtime mode uses VideoToolbox H.264 with full resolution at 60 fps. Pass `--video-codec software` only when the shared hardware encoder is unavailable or performs worse on that host. +- **Start on the default for local preview.** Browser realtime mode uses VideoToolbox H.264 with full resolution at 60 fps. Auto mode moves active sessions to software when hardware H.264 is overloaded, then retries hardware after a cooldown. - **Use `--local-stream-fps` above 60 only for local high-refresh testing.** The local quality stream defaults to 60 fps; higher targets pace both capture refresh and hardware encode submission so the stream does not build delay by pushing unbounded frames. - **Switch to `software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency. - **Studio providers default to software H.264 plus `--stream-quality smooth`.** `smooth` is an internal/provider profile, not a browser picker item. It uses a 1170-pixel longest edge, allows up to 60 fps, raises the bitrate budget to reduce compression artifacts, and lets multiple provider sessions share CPU cores without depending on one hardware encoder. @@ -161,7 +165,10 @@ active simulator session. `strained` means encode latency is approaching the active frame budget; `overloaded` means smoothed latency is over budget or multiple frames in a row exceeded the budget. For hardware H.264 this usually means the shared VideoToolbox encoder is saturated; lower resolution/FPS or -switch to software H.264. +switch to software H.264. In `auto`, `activeEncoderMode`, +`autoSoftwareFallbackActive`, `autoSoftwareFallbacks`, and +`autoHardwareRetries` show whether the active session is temporarily using +software H.264 while the hardware path cools down. Clients can also push their decoder/renderer stats back to the server. Browser clients normally send these over the WebRTC telemetry data channel or the H264 From 979a37016fc3a8cdfb2386081c859f04494af890 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 12 May 2026 18:27:16 -0400 Subject: [PATCH 2/3] Fallback to software encoder when web client is backgrounded --- cli/XCWH264Encoder.h | 1 + cli/XCWH264Encoder.m | 70 +++++++++++---- cli/XCWPrivateSimulatorSession.h | 1 + cli/XCWPrivateSimulatorSession.m | 8 ++ cli/native/XCWNativeBridge.h | 1 + cli/native/XCWNativeBridge.m | 6 ++ cli/native/XCWNativeSession.h | 1 + cli/native/XCWNativeSession.m | 4 + client/src/api/types.ts | 6 ++ .../src/features/stream/streamWorkerClient.ts | 9 +- client/src/features/stream/useLiveStream.ts | 42 +++++++++ client/src/features/toolbar/DebugPanel.tsx | 10 +++ docs/api/health.md | 6 +- docs/api/rest.md | 12 +++ docs/guide/video.md | 8 +- server/native_stubs.c | 5 ++ server/src/api/routes.rs | 86 ++++++++++++++++++- server/src/main.rs | 1 + server/src/native/bridge.rs | 6 ++ server/src/native/ffi.rs | 1 + server/src/simulators/session.rs | 7 ++ server/src/transport/webrtc.rs | 33 +++++++ 22 files changed, 305 insertions(+), 19 deletions(-) diff --git a/cli/XCWH264Encoder.h b/cli/XCWH264Encoder.h index 87076836..135d80e9 100644 --- a/cli/XCWH264Encoder.h +++ b/cli/XCWH264Encoder.h @@ -20,6 +20,7 @@ typedef void (^XCWH264EncoderOutputHandler)(NSData *sampleData, - (void)encodePixelBuffer:(CVPixelBufferRef)pixelBuffer; - (void)requestKeyFrame; - (void)reconfigureForStreamQualityChange; +- (void)setClientForeground:(BOOL)foreground; - (NSDictionary *)statsRepresentation; - (void)invalidate; diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index f1ab3878..f0a90442 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -509,6 +509,7 @@ @implementation XCWH264Encoder { BOOL _scalingActive; XCWVideoEncoderMode _encoderMode; XCWVideoEncoderMode _activeEncoderMode; + BOOL _clientForeground; BOOL _lowLatencyMode; BOOL _realtimeStreamMode; CMVideoCodecType _codecType; @@ -559,6 +560,7 @@ - (instancetype)initWithOutputHandler:(XCWH264EncoderOutputHandler)outputHandler _needsKeyFrame = YES; _encoderMode = XCWVideoEncoderModeFromEnvironment(); _activeEncoderMode = _encoderMode; + _clientForeground = YES; _lowLatencyMode = (_encoderMode == XCWVideoEncoderModeH264Software) && XCWLowLatencyModeFromEnvironment(); _realtimeStreamMode = XCWRealtimeStreamModeFromEnvironment() || _lowLatencyMode; _codecType = XCWVideoCodecTypeForMode(_activeEncoderMode); @@ -621,6 +623,17 @@ - (void)reconfigureForStreamQualityChange { self->_hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked]; self->_hardwarePacedFrameCount = 0; self->_autoSoftwareFallbackUntilUs = 0; + [self updateActiveEncoderModeForClientForegroundLockedAtTimeUs:(uint64_t)(CACurrentMediaTime() * 1000000.0)]; + }); +} + +- (void)setClientForeground:(BOOL)foreground { + dispatch_async(_queue, ^{ + if (self->_clientForeground == foreground) { + return; + } + self->_clientForeground = foreground; + [self updateActiveEncoderModeForClientForegroundLockedAtTimeUs:(uint64_t)(CACurrentMediaTime() * 1000000.0)]; }); } @@ -699,6 +712,7 @@ - (NSDictionary *)statsRepresentation { @"transportCodec": XCWCodecName(self->_codecType), @"encoderMode": XCWVideoEncoderModeName(self->_encoderMode), @"activeEncoderMode": XCWVideoEncoderModeName(self->_activeEncoderMode), + @"clientForeground": @(self->_clientForeground), @"autoSoftwareFallbackActive": @(autoSoftwareFallbackActive), @"autoSoftwareFallbackRemainingUs": @(autoSoftwareFallbackRemainingUs), @"autoSoftwareFallbacks": @(self->_autoSoftwareFallbackCount), @@ -781,6 +795,7 @@ - (uint64_t)initialHardwareFrameIntervalUsLocked { - (BOOL)isAutoSoftwareFallbackActiveLocked { return _encoderMode == XCWVideoEncoderModeAuto && + _autoSoftwareFallbackUntilUs != 0 && _activeEncoderMode == XCWVideoEncoderModeH264Software; } @@ -793,38 +808,63 @@ - (void)resetAutoFallbackLatencyStateLocked { _wasOverloaded = NO; } +- (void)switchActiveEncoderModeLocked:(XCWVideoEncoderMode)mode { + if (_activeEncoderMode == mode) { + return; + } + _activeEncoderMode = mode; + _codecType = XCWVideoCodecTypeForMode(_activeEncoderMode); + if (_activeEncoderMode == XCWVideoEncoderModeH264Software) { + _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; + _softwareHealthyFrameCount = 0; + _softwarePacedFrameCount = 0; + } else { + _hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked]; + _hardwarePacedFrameCount = 0; + } + [self invalidateCompressionSessionLocked]; + [self resetAutoFallbackLatencyStateLocked]; + _needsKeyFrame = YES; +} + +- (void)updateActiveEncoderModeForClientForegroundLockedAtTimeUs:(uint64_t)nowUs { + if (!_clientForeground) { + [self switchActiveEncoderModeLocked:XCWVideoEncoderModeH264Software]; + return; + } + if (_encoderMode == XCWVideoEncoderModeAuto && + _autoSoftwareFallbackUntilUs != 0 && + nowUs < _autoSoftwareFallbackUntilUs) { + [self switchActiveEncoderModeLocked:XCWVideoEncoderModeH264Software]; + return; + } + if (_encoderMode == XCWVideoEncoderModeAuto && _autoSoftwareFallbackUntilUs != 0) { + _autoSoftwareFallbackUntilUs = 0; + _autoHardwareRetryCount += 1; + } + [self switchActiveEncoderModeLocked:_encoderMode]; +} + - (void)enterAutoSoftwareFallbackLockedAtTimeUs:(uint64_t)nowUs { if (_encoderMode != XCWVideoEncoderModeAuto || _activeEncoderMode == XCWVideoEncoderModeH264Software) { return; } - _activeEncoderMode = XCWVideoEncoderModeH264Software; - _codecType = XCWVideoCodecTypeForMode(_activeEncoderMode); _autoSoftwareFallbackUntilUs = nowUs + XCWAutoHardwareRetryIntervalUs; _autoSoftwareFallbackCount += 1; - _softwareFrameIntervalUs = [self initialSoftwareFrameIntervalUsLocked]; - _softwareHealthyFrameCount = 0; - _softwarePacedFrameCount = 0; - [self invalidateCompressionSessionLocked]; - [self resetAutoFallbackLatencyStateLocked]; - _needsKeyFrame = YES; + [self switchActiveEncoderModeLocked:XCWVideoEncoderModeH264Software]; } - (void)retryAutoHardwareIfNeededLockedAtTimeUs:(uint64_t)nowUs { if (![self isAutoSoftwareFallbackActiveLocked] || + !_clientForeground || _autoSoftwareFallbackUntilUs == 0 || nowUs < _autoSoftwareFallbackUntilUs) { return; } - _activeEncoderMode = XCWVideoEncoderModeAuto; - _codecType = XCWVideoCodecTypeForMode(_activeEncoderMode); _autoSoftwareFallbackUntilUs = 0; _autoHardwareRetryCount += 1; - _hardwareFrameIntervalUs = [self initialHardwareFrameIntervalUsLocked]; - _hardwarePacedFrameCount = 0; - [self invalidateCompressionSessionLocked]; - [self resetAutoFallbackLatencyStateLocked]; - _needsKeyFrame = YES; + [self switchActiveEncoderModeLocked:XCWVideoEncoderModeAuto]; } - (uint64_t)activeFrameIntervalUsLocked { diff --git a/cli/XCWPrivateSimulatorSession.h b/cli/XCWPrivateSimulatorSession.h index 346531c6..8a6e51b0 100644 --- a/cli/XCWPrivateSimulatorSession.h +++ b/cli/XCWPrivateSimulatorSession.h @@ -31,6 +31,7 @@ typedef void (^XCWPrivateSimulatorEncodedFrameHandler)(NSData *sampleData, - (void)requestKeyFrameRefresh; - (void)requestFrameRefresh; - (void)reconfigureVideoEncoder; +- (void)setClientForeground:(BOOL)foreground; - (NSDictionary *)videoEncoderStats; - (id)addEncodedFrameListener:(XCWPrivateSimulatorEncodedFrameHandler)handler; - (void)removeEncodedFrameListener:(id)token; diff --git a/cli/XCWPrivateSimulatorSession.m b/cli/XCWPrivateSimulatorSession.m index fe177c1a..701299cf 100644 --- a/cli/XCWPrivateSimulatorSession.m +++ b/cli/XCWPrivateSimulatorSession.m @@ -177,6 +177,14 @@ - (void)reconfigureVideoEncoder { [self refreshCurrentFrame]; } +- (void)setClientForeground:(BOOL)foreground { + [_videoEncoder setClientForeground:foreground]; + if (foreground) { + [_videoEncoder requestKeyFrame]; + [self refreshCurrentFrame]; + } +} + - (NSDictionary *)videoEncoderStats { return [_videoEncoder statsRepresentation]; } diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h index 526dbd38..c3123775 100644 --- a/cli/native/XCWNativeBridge.h +++ b/cli/native/XCWNativeBridge.h @@ -76,6 +76,7 @@ bool xcw_native_session_start(void * _Nonnull handle, char * _Nullable * _Nullab void xcw_native_session_request_refresh(void * _Nonnull handle); void xcw_native_session_request_keyframe(void * _Nonnull handle); void xcw_native_session_reconfigure_video_encoder(void * _Nonnull handle); +void xcw_native_session_set_client_foreground(void * _Nonnull handle, bool foreground); char * _Nullable xcw_native_session_video_encoder_stats(void * _Nonnull handle, char * _Nullable * _Nullable error_message); int32_t xcw_native_session_rotation_quarter_turns(void * _Nonnull handle); bool xcw_native_session_send_touch(void * _Nonnull handle, double x, double y, const char * _Nonnull phase, char * _Nullable * _Nullable error_message); diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m index d36630d7..5694e187 100644 --- a/cli/native/XCWNativeBridge.m +++ b/cli/native/XCWNativeBridge.m @@ -937,6 +937,12 @@ void xcw_native_session_reconfigure_video_encoder(void *handle) { } } +void xcw_native_session_set_client_foreground(void *handle, bool foreground) { + @autoreleasepool { + [XCWNativeSessionFromHandle(handle) setClientForeground:foreground]; + } +} + char *xcw_native_session_video_encoder_stats(void *handle, char **error_message) { @autoreleasepool { return XCWJSONStringFromObject([XCWNativeSessionFromHandle(handle) videoEncoderStats] ?: @{}, error_message); diff --git a/cli/native/XCWNativeSession.h b/cli/native/XCWNativeSession.h index 76550fd2..387f904f 100644 --- a/cli/native/XCWNativeSession.h +++ b/cli/native/XCWNativeSession.h @@ -15,6 +15,7 @@ NS_ASSUME_NONNULL_BEGIN - (void)requestRefresh; - (void)requestKeyFrame; - (void)reconfigureVideoEncoder; +- (void)setClientForeground:(BOOL)foreground; - (NSDictionary *)videoEncoderStats; - (NSInteger)rotationQuarterTurns; - (BOOL)sendTouchAtX:(double)x diff --git a/cli/native/XCWNativeSession.m b/cli/native/XCWNativeSession.m index f99ff5e4..c956f9d3 100644 --- a/cli/native/XCWNativeSession.m +++ b/cli/native/XCWNativeSession.m @@ -102,6 +102,10 @@ - (void)reconfigureVideoEncoder { [self.session reconfigureVideoEncoder]; } +- (void)setClientForeground:(BOOL)foreground { + [self.session setClientForeground:foreground]; +} + - (NSDictionary *)videoEncoderStats { return [self.session videoEncoderStats]; } diff --git a/client/src/api/types.ts b/client/src/api/types.ts index 05684a78..25cd56b9 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -1,6 +1,12 @@ export interface EncoderStats { + activeEncoderMode?: string; averageEncodeLatencyUs?: number; averageEncoderLoadPercent?: number; + autoHardwareRetries?: number; + autoSoftwareFallbackActive?: boolean; + autoSoftwareFallbackRemainingUs?: number; + autoSoftwareFallbacks?: number; + clientForeground?: boolean; consecutiveOverBudgetFrames?: number; encoderBudgetUs?: number; encoderLoadPercent?: number; diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 00ea6d36..a7e15e51 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -86,7 +86,9 @@ export function sendStreamClientStats(stats: unknown): boolean { } export function sendWebRtcStreamControl(options: { + clientId?: string; forceKeyframe?: boolean; + foreground?: boolean; snapshot?: boolean; }): boolean { return sendDataChannelMessage( @@ -3018,7 +3020,12 @@ export class StreamWorkerClient { return (await this.backend?.collectVisualArtifactSample?.(udid)) ?? null; } - sendStreamControl(options: { forceKeyframe?: boolean; snapshot?: boolean }) { + sendStreamControl(options: { + clientId?: string; + forceKeyframe?: boolean; + foreground?: boolean; + snapshot?: boolean; + }) { return Boolean( this.backend?.sendControl?.({ ...options, type: "streamControl" }), ); diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index 5cf02fbe..96e0023b 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -99,6 +99,10 @@ function currentClientBundle(): string { ); } +function isDocumentForeground(): boolean { + return document.visibilityState === "visible" && document.hasFocus(); +} + export function useLiveStream({ canvasElement, paused = false, @@ -346,6 +350,40 @@ export function useLiveStream({ streamTransport, ]); + useEffect(() => { + if (!simulator?.udid || paused) { + return; + } + + const sendForegroundState = (foreground = isDocumentForeground()) => { + workerClientRef.current?.sendStreamControl({ + clientId: clientTelemetryIdRef.current, + foreground, + }); + }; + + const sendCurrentForegroundState = () => { + sendForegroundState(); + }; + const sendBackgroundState = () => { + sendForegroundState(false); + }; + + sendCurrentForegroundState(); + document.addEventListener("visibilitychange", sendCurrentForegroundState); + window.addEventListener("focus", sendCurrentForegroundState); + window.addEventListener("blur", sendCurrentForegroundState); + window.addEventListener("pageshow", sendCurrentForegroundState); + window.addEventListener("pagehide", sendBackgroundState); + return () => { + document.removeEventListener("visibilitychange", sendCurrentForegroundState); + window.removeEventListener("focus", sendCurrentForegroundState); + window.removeEventListener("blur", sendCurrentForegroundState); + window.removeEventListener("pageshow", sendCurrentForegroundState); + window.removeEventListener("pagehide", sendBackgroundState); + }; + }, [paused, simulator?.udid, streamTransport]); + useEffect(() => { if ( streamConfigApplyKey <= 0 || @@ -410,6 +448,10 @@ export function useLiveStream({ visualSampleCount: latestVisualArtifactSampleCountRef.current, visibilityState: document.visibilityState, }; + workerClientRef.current?.sendStreamControl({ + clientId: clientTelemetryIdRef.current, + foreground: isDocumentForeground(), + }); if ( sendStreamClientStats(payload) || remote || diff --git a/client/src/features/toolbar/DebugPanel.tsx b/client/src/features/toolbar/DebugPanel.tsx index 65ea520c..dbbd3551 100644 --- a/client/src/features/toolbar/DebugPanel.tsx +++ b/client/src/features/toolbar/DebugPanel.tsx @@ -80,6 +80,16 @@ export function DebugPanel({ if (encoder) { rows.push( { label: "Encoder", value: encoder.encoderMode ?? "—" }, + { label: "Active Encoder", value: encoder.activeEncoderMode ?? "—" }, + { + label: "Client Foreground", + value: + typeof encoder.clientForeground === "boolean" + ? encoder.clientForeground + ? "yes" + : "no" + : "—", + }, { label: "Encoder State", value: encoder.overloadState ?? "—" }, { label: "Encoder Load", diff --git a/docs/api/health.md b/docs/api/health.md index 1ff4d8fe..d0dfb7e6 100644 --- a/docs/api/health.md +++ b/docs/api/health.md @@ -58,6 +58,7 @@ Returns a snapshot of every server-side counter and the rolling buffer of client "encoder": { "encoderMode": "auto", "activeEncoderMode": "hardware", + "clientForeground": true, "autoSoftwareFallbackActive": false, "hardwareAccelerated": true, "overloadState": "nominal", @@ -119,7 +120,10 @@ from hardware to software encoding. When `encoderMode` is `auto`, SimDeck uses this signal to temporarily rebuild the active session with software H.264 if the hardware encoder is overloaded, then retries hardware after a cooldown. The `activeEncoderMode`, `autoSoftwareFallbackActive`, `autoSoftwareFallbacks`, and -`autoHardwareRetries` fields expose that state. +`autoHardwareRetries` fields expose that state. `clientForeground` is `false` +when all current browser viewers for the simulator are hidden or unfocused; in +that state the native session uses software H.264 until a viewer returns +foreground. ### Client stream stats diff --git a/docs/api/rest.md b/docs/api/rest.md index c5c0a428..5d16c27b 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -220,6 +220,14 @@ stream attached to that peer: { "type": "streamControl", "forceKeyframe": true } ``` +Clients can also report page focus/visibility through stream control. When all +current viewers for a simulator report `foreground: false`, the native session +uses software H.264 until a viewer returns foreground: + +```json +{ "type": "streamControl", "clientId": "browser", "foreground": false } +``` + ```json { "type": "streamQuality", "config": { "profile": "low", "fps": 30 } } ``` @@ -258,6 +266,10 @@ socket: { "type": "streamControl", "forceKeyframe": true } ``` +```json +{ "type": "streamControl", "clientId": "browser", "foreground": false } +``` + ```json { "type": "streamQuality", "config": { "profile": "low", "fps": 30 } } ``` diff --git a/docs/guide/video.md b/docs/guide/video.md index cba383f8..1e1d7490 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -36,6 +36,9 @@ In `auto`, active simulator encoder metrics also report `activeEncoderMode`. When the hardware encoder is overloaded, `auto` rebuilds the active compression session as software H.264, then periodically retries hardware and stays there when the measured encode latency returns to budget. +If all current browser viewers for a simulator are backgrounded or unfocused, +the active session also switches to software H.264 until at least one viewer is +foreground again. The browser UI exposes stream controls for encoder, FPS, transport, and resolution. H264 resolution choices are `full` (4096 px at 60 fps), `balanced` (1280 px at 60 fps), `economy` (1080 px at 30 fps), `low` (720 px at 30 fps), and `tiny` (540 px at 30 fps). Local H264 WebSocket sessions default to full resolution at 60 fps. Remote browser sessions default to software H.264, 30 fps, and adaptive quality. ## Remote WebRTC ICE @@ -124,6 +127,7 @@ The WebRTC path favors freshness: stale frames are dropped and the sender reques A few practical guidelines: - **Start on the default for local preview.** Browser realtime mode uses VideoToolbox H.264 with full resolution at 60 fps. Auto mode moves active sessions to software when hardware H.264 is overloaded, then retries hardware after a cooldown. +- **Backgrounded web clients use software.** The browser sends page visibility/focus over the stream control channel. If every active viewer for a simulator is hidden or unfocused, the native session uses software H.264 until a viewer returns foreground. - **Use `--local-stream-fps` above 60 only for local high-refresh testing.** The local quality stream defaults to 60 fps; higher targets pace both capture refresh and hardware encode submission so the stream does not build delay by pushing unbounded frames. - **Switch to `software` when the hardware encoder stalls or is unavailable.** The encoder scales the longest edge to 1600 pixels, can climb toward 60 fps, and backs off dynamically under encode latency. - **Studio providers default to software H.264 plus `--stream-quality smooth`.** `smooth` is an internal/provider profile, not a browser picker item. It uses a 1170-pixel longest edge, allows up to 60 fps, raises the bitrate budget to reduce compression artifacts, and lets multiple provider sessions share CPU cores without depending on one hardware encoder. @@ -168,7 +172,9 @@ means the shared VideoToolbox encoder is saturated; lower resolution/FPS or switch to software H.264. In `auto`, `activeEncoderMode`, `autoSoftwareFallbackActive`, `autoSoftwareFallbacks`, and `autoHardwareRetries` show whether the active session is temporarily using -software H.264 while the hardware path cools down. +software H.264 while the hardware path cools down. `clientForeground` shows +whether any current browser viewer is visible and focused; when it is `false`, +the active encoder is software regardless of the requested mode. Clients can also push their decoder/renderer stats back to the server. Browser clients normally send these over the WebRTC telemetry data channel or the H264 diff --git a/server/native_stubs.c b/server/native_stubs.c index 03b239d8..148540b5 100644 --- a/server/native_stubs.c +++ b/server/native_stubs.c @@ -340,6 +340,11 @@ void xcw_native_session_reconfigure_video_encoder(void *handle) { (void)handle; } +void xcw_native_session_set_client_foreground(void *handle, bool foreground) { + (void)handle; + (void)foreground; +} + char *xcw_native_session_video_encoder_stats(void *handle, char **error_message) { (void)handle; diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 721e4d01..ce9ec813 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -27,7 +27,7 @@ use futures::{SinkExt, StreamExt}; use serde::Deserialize; use serde_json::Map; use serde_json::{json as json_value, Value}; -use std::collections::VecDeque; +use std::collections::{HashMap, VecDeque}; use std::env; use std::net::SocketAddr; use std::sync::{Arc, Mutex as StdMutex, OnceLock}; @@ -50,6 +50,7 @@ const H264_WS_FLAG_KEYFRAME: u8 = 1 << 0; const H264_WS_FLAG_CONFIG: u8 = 1 << 1; const H264_WS_SEND_TIMEOUT: Duration = Duration::from_secs(2); const H264_WS_KEYFRAME_WAIT_TIMEOUT: Duration = Duration::from_secs(3); +const STREAM_CLIENT_FOREGROUND_TTL: Duration = Duration::from_secs(30); #[derive(Clone)] pub struct AppState { @@ -58,10 +59,60 @@ pub struct AppState { pub logs: LogRegistry, pub inspectors: InspectorHub, pub metrics: Arc, + pub stream_clients: StreamClientForegroundRegistry, pub simulator_inventory: SimulatorInventoryCache, pub android: AndroidBridge, } +#[derive(Clone, Default)] +pub struct StreamClientForegroundRegistry { + inner: Arc>>, +} + +#[derive(Clone, Copy)] +struct StreamClientForegroundState { + foreground: bool, + updated_at: Instant, +} + +impl StreamClientForegroundRegistry { + pub fn record(&self, udid: &str, client_id: &str, foreground: bool) -> (bool, bool) { + let now = Instant::now(); + let mut clients = self + .inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + clients.retain(|_, state| { + now.duration_since(state.updated_at) <= STREAM_CLIENT_FOREGROUND_TTL + }); + let previous = any_foreground_client_for_udid(&clients, udid); + clients.insert( + (udid.to_owned(), client_id.to_owned()), + StreamClientForegroundState { + foreground, + updated_at: now, + }, + ); + let next = any_foreground_client_for_udid(&clients, udid).unwrap_or(true); + (next, previous != Some(next)) + } +} + +fn any_foreground_client_for_udid( + clients: &HashMap<(String, String), StreamClientForegroundState>, + udid: &str, +) -> Option { + let mut saw_client = false; + let mut saw_foreground = false; + for ((client_udid, _), state) in clients { + if client_udid == udid { + saw_client = true; + saw_foreground |= state.foreground; + } + } + saw_client.then_some(saw_foreground) +} + #[derive(Clone, Default)] pub struct SimulatorInventoryCache { inner: Arc>, @@ -2189,9 +2240,12 @@ fn handle_h264_socket_message( } } H264SocketMessage::StreamControl { + client_id, force_keyframe, + foreground, snapshot, } => { + apply_stream_client_foreground(state, session, &client_id, foreground); if force_keyframe.unwrap_or(false) { session.request_keyframe(); } @@ -2210,6 +2264,31 @@ fn handle_h264_socket_message( true } +fn apply_stream_client_foreground( + state: &AppState, + session: &SimulatorSession, + client_id: &Option, + foreground: Option, +) { + let Some(foreground) = foreground else { + return; + }; + let Some(client_id) = client_id + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + else { + return; + }; + let (any_foreground, changed) = + state + .stream_clients + .record(session.udid(), client_id, foreground); + if changed { + session.set_client_foreground(any_foreground); + } +} + fn handle_android_h264_socket_message( state: &AppState, source: &AndroidWebRtcSource, @@ -2234,7 +2313,9 @@ fn handle_android_h264_socket_message( } } H264SocketMessage::StreamControl { + client_id: _, force_keyframe, + foreground: _, snapshot, } => { if force_keyframe.unwrap_or(false) { @@ -2258,8 +2339,11 @@ enum H264SocketMessage { stats: Box, }, StreamControl { + #[serde(rename = "clientId")] + client_id: Option, #[serde(rename = "forceKeyframe")] force_keyframe: Option, + foreground: Option, snapshot: Option, }, StreamQuality { diff --git a/server/src/main.rs b/server/src/main.rs index b8e68582..4a727884 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -5395,6 +5395,7 @@ async fn serve( logs, inspectors, metrics, + stream_clients: Default::default(), simulator_inventory: Default::default(), android: Default::default(), }; diff --git a/server/src/native/bridge.rs b/server/src/native/bridge.rs index 0de5d2db..5f7340ae 100644 --- a/server/src/native/bridge.rs +++ b/server/src/native/bridge.rs @@ -773,6 +773,12 @@ impl NativeSession { } } + pub fn set_client_foreground(&self, foreground: bool) { + unsafe { + ffi::xcw_native_session_set_client_foreground(self.handle, foreground); + } + } + pub fn video_encoder_stats(&self) -> serde_json::Value { unsafe { let mut error = ptr::null_mut(); diff --git a/server/src/native/ffi.rs b/server/src/native/ffi.rs index 803b0e26..0011881d 100644 --- a/server/src/native/ffi.rs +++ b/server/src/native/ffi.rs @@ -201,6 +201,7 @@ unsafe extern "C" { pub fn xcw_native_session_request_refresh(handle: *mut c_void); pub fn xcw_native_session_request_keyframe(handle: *mut c_void); pub fn xcw_native_session_reconfigure_video_encoder(handle: *mut c_void); + pub fn xcw_native_session_set_client_foreground(handle: *mut c_void, foreground: bool); pub fn xcw_native_session_video_encoder_stats( handle: *mut c_void, error_message: *mut *mut c_char, diff --git a/server/src/simulators/session.rs b/server/src/simulators/session.rs index 4748fb63..a86f8429 100644 --- a/server/src/simulators/session.rs +++ b/server/src/simulators/session.rs @@ -236,6 +236,13 @@ impl SimulatorSession { self.inner.native.reconfigure_video_encoder(); } + pub fn set_client_foreground(&self, foreground: bool) { + self.inner.native.set_client_foreground(foreground); + if foreground { + self.request_keyframe(); + } + } + pub fn send_touch(&self, x: f64, y: f64, phase: &str) -> Result<(), AppError> { self.inner.native.send_touch(x, y, phase) } diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 0858d8c3..9b5ebae5 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -642,9 +642,12 @@ fn attach_control_data_channel( } } WebRtcDataChannelMessage::StreamControl { + client_id, force_keyframe, + foreground, snapshot, } => { + apply_stream_client_foreground(&state, &session, &client_id, foreground); let command = WebRtcStreamCommand { force_keyframe: force_keyframe.unwrap_or(false), snapshot: snapshot.unwrap_or(false), @@ -678,6 +681,31 @@ fn attach_control_data_channel( })); } +fn apply_stream_client_foreground( + state: &AppState, + session: &crate::simulators::session::SimulatorSession, + client_id: &Option, + foreground: Option, +) { + let Some(foreground) = foreground else { + return; + }; + let Some(client_id) = client_id + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + else { + return; + }; + let (any_foreground, changed) = + state + .stream_clients + .record(session.udid(), client_id, foreground); + if changed { + session.set_client_foreground(any_foreground); + } +} + fn register_android_data_channel( peer_connection: &Arc, source: AndroidWebRtcSource, @@ -732,7 +760,9 @@ fn attach_android_data_channel( } } WebRtcDataChannelMessage::StreamControl { + client_id: _, force_keyframe, + foreground: _, snapshot, } => { let command = WebRtcStreamCommand { @@ -974,8 +1004,11 @@ enum WebRtcDataChannelMessage { stats: Box, }, StreamControl { + #[serde(rename = "clientId")] + client_id: Option, #[serde(rename = "forceKeyframe")] force_keyframe: Option, + foreground: Option, snapshot: Option, }, StreamQuality { From 4bfca372e33f8b49a672793714024adf88ac2f0a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 12 May 2026 18:32:00 -0400 Subject: [PATCH 3/3] Format foreground encoder changes --- client/src/features/stream/useLiveStream.ts | 5 ++++- docs/guide/video.md | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index 96e0023b..c675dd28 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -376,7 +376,10 @@ export function useLiveStream({ window.addEventListener("pageshow", sendCurrentForegroundState); window.addEventListener("pagehide", sendBackgroundState); return () => { - document.removeEventListener("visibilitychange", sendCurrentForegroundState); + document.removeEventListener( + "visibilitychange", + sendCurrentForegroundState, + ); window.removeEventListener("focus", sendCurrentForegroundState); window.removeEventListener("blur", sendCurrentForegroundState); window.removeEventListener("pageshow", sendCurrentForegroundState); diff --git a/docs/guide/video.md b/docs/guide/video.md index 1e1d7490..a2a0548d 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -6,11 +6,11 @@ SimDeck streams the iOS Simulator over WebRTC using browser-native H.264 video p The server can encode the simulator display in three modes, picked at startup with `--video-codec`: -| Value | Encoder | When to use it | -| ------------------ | ------------------------------------ | -------------------------------------------------------------------- | +| Value | Encoder | When to use it | +| ------------------ | ------------------------------------ | ------------------------------------------------------------------------------------------- | | `auto` _(default)_ | VideoToolbox chooses the encoder | Normal local and remote preview. Falls back to software when hardware encode is overloaded. | -| `hardware` | Required hardware H.264 | Use only when the hardware encoder is known to be available. | -| `software` | Software-only H.264 via VideoToolbox | Use when hardware encode stalls, is unavailable, or must be avoided. | +| `hardware` | Required hardware H.264 | Use only when the hardware encoder is known to be available. | +| `software` | Software-only H.264 via VideoToolbox | Use when hardware encode stalls, is unavailable, or must be avoided. | Restart the daemon to change encoder mode: