feat(resize/apple): resize overlay — blur + spinner during mid-stream resize

Make a Match-window resize deliberate instead of a stutter: blur the live stream and show a spinner while the host rebuilds its virtual display + encoder and VideoToolbox re-inits on the new-mode IDR. No new protocol — driven entirely by existing client signals. - ResizeIndicator (pure core, unit-tested): START = follower steering, END = a decoded frame at the target size, TIMEOUT = 2.5s safety net for a rejected/capped switch that never yields a new-size frame; re-arms only on a CHANGED target, not a repeated same-size drag. - MatchWindowFollower.onResizeTarget fires the instant the window differs from the live mode (deduped via lastSteered); a new onDecodedSize callback threads each new-mode IDR's coded dims through StreamPump/Stage2Pipeline → SessionPresenter → both stream views. - SessionModel gains @Published resizing (+ resizeTargeted/resizeDecoded, a tick on the 1 Hz stats timer, reset on disconnect); ContentView blurs the stream 16px and overlays ResizeIndicatorView while resizing (the 32px trust-prompt blur is unchanged and takes precedence). tvOS declares the props but never fires the follower (it drives modes via AVDisplayManager), so the overlay stays dormant there. Pure core verified on the Linux toolchain; full AppKit/UIKit build pending on a Mac. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-11 15:58:14 +02:00
parent d294b3923c
commit f01c5e210c
11 changed files with 313 additions and 10 deletions
@@ -57,6 +57,16 @@ public final class MatchWindowFollower {
    private var pendingSize: (width: Int, height: Int)?
    private var lastRequested: (width: UInt32, height: UInt32)?
    private var lastRequestAt: Date?
+    /// The last size we reported via [`onResizeTarget`] — dedups the per-layout stream of a drag so
+    /// the UI is notified once per distinct target, and reset to `nil` when the window is back in
+    /// sync with the live mode (so a later resize re-reports).
+    private var lastSteered: (width: UInt32, height: UInt32)?
+
+    /// Fired (on the main actor) the instant the window starts differing from the live mode — i.e.
+    /// a resize is under way and a `Reconfigure` for `(width, height)` is imminent. Drives the
+    /// resize overlay's INSTANT feedback (blur + spinner) BEFORE the debounced request leaves; the
+    /// overlay clears when a decoded frame reaches this size (or on a timeout). Deduped per target.
+    public var onResizeTarget: ((_ width: UInt32, _ height: UInt32) -> Void)?

    /// `debounce` = quiet time after the last size event before requesting (Win32 gets
    /// `WM_EXITSIZEMOVE` for free; we debounce). `minSpacing` = floor between accepted requests
@@ -80,6 +90,7 @@ public final class MatchWindowFollower {
            work?.cancel()
            work = nil
            pendingSize = nil
+            lastSteered = nil
        }
    }

@@ -90,6 +101,24 @@ public final class MatchWindowFollower {
        guard enabled else { return }
        pendingSize = (widthPx, heightPx)
        schedule()
+        reportSteering(widthPx: widthPx, heightPx: heightPx)
+    }
+
+    /// Report the resize overlay's START signal (deduped): the moment the normalized window size
+    /// differs from the live mode we're steering toward a new size. No connection / no negotiated
+    /// mode yet → nothing to compare against, skip.
+    private func reportSteering(widthPx: Int, heightPx: Int) {
+        guard let connection else { return }
+        let target = MatchWindow.normalize(widthPx: widthPx, heightPx: heightPx)
+        let mode = connection.currentMode()
+        guard mode.width > 0, mode.height > 0 else { return }
+        if target.width == mode.width, target.height == mode.height {
+            lastSteered = nil // back in sync — a later change re-reports
+            return
+        }
+        if lastSteered?.width == target.width, lastSteered?.height == target.height { return }
+        lastSteered = target
+        onResizeTarget?(target.width, target.height)
    }

    private func schedule() {
@@ -0,0 +1,63 @@
+// Resize-in-progress indicator state (design/midstream-resolution-resize.md — client UX).
+//
+// A mid-stream resize takes the host 0.3–2 s to rebuild its virtual display + encoder, and the
+// first new-mode frame is an IDR that the decoder re-inits on. Rather than let the stream scale
+// (stretch/blur) to the changing window during that gap, the client EMBRACES the delay: it shows a
+// deliberate blur + spinner the instant a resize starts and clears it the instant the sharp
+// new-resolution frame is on screen — so the wait reads as intentional, not as lag.
+//
+// This is driven ENTIRELY by signals the client already has (no new protocol):
+//   * START — the Match-window follower reports the size it is steering toward (instant, on the
+//     first resize layout, before the debounced request even leaves).
+//   * END — the decode pipeline reports each new-mode IDR's dimensions; when they reach the target
+//     the new picture is here.
+//   * TIMEOUT — the safety net for a switch that never delivers the exact target: the host rejected
+//     it (gamescope), capped it to an advertised mode, or a corrective ack landed a different size.
+//
+// Pure + side-effect-free so the transition logic is unit-tested without a live session or UI
+// (`ResizeIndicatorTests`); `SessionModel` owns an instance and mirrors `active` into a @Published.
+
+import Foundation
+
+/// The pure state of the resize overlay. `now` is a monotonic time in seconds (the caller passes
+/// `ProcessInfo.processInfo.systemUptime` or a test clock).
+public struct ResizeIndicator {
+    /// Whether the blur + spinner should be shown.
+    public private(set) var active = false
+    /// The size the follower is steering toward — cleared once a decoded frame reaches it.
+    private var target: (width: UInt32, height: UInt32)?
+    /// When the current `active` span began — the timeout is measured from here.
+    private var since: TimeInterval?
+    /// How long to keep the overlay up if the target frame never arrives (rejected / capped switch).
+    public var timeout: TimeInterval
+
+    public init(timeout: TimeInterval = 2.5) { self.timeout = timeout }
+
+    /// The follower is steering toward `width`×`height` — a resize is under way. Show the overlay now
+    /// (instant feedback). Called only for a genuine change (the follower skips a target equal to the
+    /// live mode), possibly many times as a drag moves through sizes; the timeout re-arms whenever the
+    /// target actually changes so a slow drag never trips it mid-gesture.
+    public mutating func steering(width: UInt32, height: UInt32, now: TimeInterval) {
+        if !active || target?.width != width || target?.height != height {
+            since = now
+        }
+        target = (width, height)
+        active = true
+    }
+
+    /// A decoded frame arrived at `width`×`height` (a new-mode IDR). Clears the overlay once it
+    /// matches the steered target — the sharp new-resolution picture is on glass.
+    public mutating func decoded(width: UInt32, height: UInt32) {
+        guard active, let t = target, t.width == width, t.height == height else { return }
+        active = false
+        since = nil
+    }
+
+    /// Timeout safety net: stop showing the overlay once `timeout` has elapsed with no matching frame
+    /// (a rejected or host-capped switch never delivers the exact target).
+    public mutating func tick(now: TimeInterval) {
+        guard active, let s = since, now - s >= timeout else { return }
+        active = false
+        since = nil
+    }
+}
@@ -85,7 +85,8 @@ final class SessionPresenter {
        displayMeter: LatencyMeter? = nil,
        makeDisplayLink: (AnyObject, Selector) -> CADisplayLink,
        onFrame: (@Sendable (AccessUnit) -> Void)?,
-        onSessionEnd: (@Sendable () -> Void)?
+        onSessionEnd: (@Sendable () -> Void)?,
+        onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil
    ) {
        stop()
        self.connection = connection
@@ -128,12 +129,14 @@ final class SessionPresenter {
            link.add(to: .main, forMode: .common)
            stage2Link = link
            syncFrameRate(hz: connection.currentMode().refreshHz)
-            pipeline.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
+            pipeline.start(
+                connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd,
+                onDecodedSize: onDecodedSize)
        } else {
            let pump = StreamPump()
            pump.start(
                connection: connection, layer: baseLayer,
-                onFrame: onFrame, onSessionEnd: onSessionEnd)
+                onFrame: onFrame, onSessionEnd: onSessionEnd, onDecodedSize: onDecodedSize)
            self.pump = pump
        }
    }
@@ -329,7 +329,8 @@ public final class Stage2Pipeline {
    public func start(
        connection: PunktfunkConnection,
        onFrame: (@Sendable (AccessUnit) -> Void)?,
-        onSessionEnd: (@Sendable () -> Void)?
+        onSessionEnd: (@Sendable () -> Void)?,
+        onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil
    ) {
        offsetNs = connection.clockOffsetNs
        recovery.bind(connection) // arm host-keyframe recovery for this session
@@ -350,6 +351,9 @@ public final class Stage2Pipeline {
        let thread = Thread {
            defer { pumpStopped.signal() } // let stop() join the pump (bounded) before decoder.reset()
            var format: CMVideoFormatDescription?
+            // Report coded dims to the resize overlay only on a CHANGE (new-mode IDR), not per
+            // loss-recovery IDR at the same size (see StreamPump).
+            var lastDecodedDims: CMVideoDimensions?
            var lastFramesDropped = connection.framesDropped()
            // Persistent recovery WANT, not a one-shot edge (see StreamPump for the full rationale):
            // keep asking until an IDR lands so a request swallowed by the throttle is re-sent.
@@ -387,6 +391,11 @@ public final class Stage2Pipeline {
                    onFrame?(au)
                    if let f = connection.videoCodec.formatDescription(fromKeyframe: au.data) {
                        format = f          // refreshed on every IDR (mode changes included)
+                        let dims = CMVideoFormatDescriptionGetDimensions(f)
+                        if lastDecodedDims?.width != dims.width || lastDecodedDims?.height != dims.height {
+                            lastDecodedDims = dims
+                            onDecodedSize?(Int(dims.width), Int(dims.height))
+                        }
                        awaitingIDR = false // a fresh IDR re-anchored decode — recovery complete
                    }
                    guard let f = format, !token.isStopped else { return true }
@@ -21,7 +21,8 @@ final class StreamPump {
        connection: PunktfunkConnection,
        layer: AVSampleBufferDisplayLayer,
        onFrame: (@Sendable (AccessUnit) -> Void)?,
-        onSessionEnd: (@Sendable () -> Void)?
+        onSessionEnd: (@Sendable () -> Void)?,
+        onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil
    ) {
        let token = token
        // Coalesced host keyframe requests (100 ms throttle — see KeyframeRecovery).
@@ -35,6 +36,9 @@ final class StreamPump {

        let thread = Thread {
            var format: CMVideoFormatDescription?
+            // Report the coded dims to the resize overlay only when they CHANGE (a new-mode IDR),
+            // not on every loss-recovery IDR at the same size — so it fires once per real switch.
+            var lastDecodedDims: CMVideoDimensions?
            var lastFramesDropped = connection.framesDropped()
            // Recovery is a persistent WANT, not a one-shot edge: set it on detected loss (or a
            // decoder reset), retry the throttled request EVERY iteration, and clear it only when a
@@ -79,6 +83,11 @@ final class StreamPump {
                    let idrFormat = connection.videoCodec.formatDescription(fromKeyframe: au.data)
                    if let f = idrFormat {
                        format = f          // refreshed on every IDR (mode changes included)
+                        let dims = CMVideoFormatDescriptionGetDimensions(f)
+                        if lastDecodedDims?.width != dims.width || lastDecodedDims?.height != dims.height {
+                            lastDecodedDims = dims
+                            onDecodedSize?(Int(dims.width), Int(dims.height))
+                        }
                        if awaitingIDR {
                            let ms = Int(Date().timeIntervalSince(awaitingSince) * 1000)
                            pumpLog.notice("video: recovery IDR received — resumed after \(ms, privacy: .public) ms")