Merge branch 'midstream-resize': mid-stream resolution resize
apple / swift (push) Successful in 4m19s
ci / rust (push) Failing after 46s
ci / web (push) Successful in 51s
ci / docs-site (push) Successful in 1m2s
decky / build-publish (push) Successful in 20s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 9s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 9s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 8s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 8s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 8s
ci / bench (push) Successful in 5m17s
arch / build-publish (push) Successful in 10m44s
docker / deploy-docs (push) Successful in 11s
android / android (push) Successful in 16m43s
windows-host / package (push) Failing after 8m29s
flatpak / build-publish (push) Failing after 8m13s
deb / build-publish (push) Successful in 11m24s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 3m51s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 4m1s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 12m24s
release / apple (push) Successful in 24m35s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 12m24s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 5m2s
windows / build (x86_64-pc-windows-msvc) (push) Failing after 5m25s
apple / screenshots (push) Successful in 19m55s

Lands the mid-stream resolution resize feature (client-driven Reconfigure so
the host's virtual display + encoder follow a resized client window without a
reconnect), all paths default OFF:

- host hardening H1-H5 + session-binary Match window (C1)
- Apple macOS/iPadOS Match-window trigger + settings (C3) and the resize
  overlay (blur + spinner) client UX
- Windows on-glass fixes: corrective-ack actual resolution + pf-vdisplay
  monitor re-arrival for out-of-list mid-stream modes
- Linux backend matrix + the live-reconfigure gate unit tests

Validated on-glass: Windows IDD-push (.173), Linux Mutter + KWin. Android
(C4) deferred; Apple full build pending on a Mac.
This commit is contained in:
2026-07-11 15:59:07 +02:00
29 changed files with 1505 additions and 162 deletions
@@ -327,12 +327,25 @@ struct ContentView: View {
}()
return ZStack {
stream(captureEnabled: pendingFingerprint == nil)
.blur(radius: pendingFingerprint != nil ? 32 : 0)
// Blur the live stream during the trust prompt (heavy) and during a resize (lighter
// the deliberate "hold on" while the host rebuilds its pipeline and the decoder
// re-inits on the new-mode IDR). Only the resize blur animates; the trust blur snaps
// as before (its own overlay handles the transition).
.blur(radius: pendingFingerprint != nil ? 32 : (model.resizing ? 16 : 0))
.animation(.easeInOut(duration: 0.22), value: model.resizing)
.overlay {
if pendingFingerprint != nil {
Color.black.opacity(0.45)
}
}
// The resize spinner rides over the (blurred) stream; suppressed under the trust
// prompt, which owns the screen. It never hit-tests, so window-drag resizes keep
// steering and the next click still reaches the stream.
.overlay {
if pendingFingerprint == nil {
ResizeIndicatorView(active: model.resizing)
}
}
if let fp = pendingFingerprint {
TrustCardView(
fingerprint: fp,
@@ -410,6 +423,16 @@ struct ContentView: View {
onSessionEnd: { [weak model] in
Task { @MainActor in model?.sessionEnded() }
},
// Resize overlay START the follower is main-actor, so this drives the blur
// + spinner synchronously the instant the window differs from the live mode.
onResizeTarget: { [weak model] w, h in
model?.resizeTargeted(width: w, height: h)
},
// Resize overlay END the coded dims of each new-mode IDR, reported from the
// decode pump thread; hop to the main actor to clear the overlay.
onDecodedSize: { [weak model] w, h in
Task { @MainActor in model?.resizeDecoded(width: w, height: h) }
},
endToEndMeter: model.endToEnd,
decodeMeter: model.decodeStage,
displayMeter: model.displayStage
@@ -0,0 +1,41 @@
// The resize overlay (design/midstream-resolution-resize.md client resize UX). A Match-window
// resize renegotiates the host's virtual display + encoder and re-inits the local VideoToolbox
// decoder on the first new-mode IDR an unavoidable sub-second gap where the last frame lingers,
// briefly freezes, or the picture pops to the new geometry. Rather than let that read as a stutter,
// we make it DELIBERATE: the caller blurs the live stream and this centered spinner + caption
// acknowledges the transition. It clears the instant a frame at the requested size decodes (the
// `onDecodedSize` END signal) or on the follower's safety timeout see `SessionModel.resizing`.
//
// Floating overlay, never a hit-test target: input keeps flowing to the stream underneath so a
// resize the user triggers by dragging the window never swallows their next click.
import PunktfunkKit
import SwiftUI
struct ResizeIndicatorView: View {
/// Mirrors `SessionModel.resizing`; the fade in/out is driven off this.
let active: Bool
var body: some View {
ZStack {
if active {
VStack(spacing: 12) {
ProgressView().controlSize(.large).tint(.white)
Text("Resizing…")
.font(.geist(15, .medium, relativeTo: .callout))
.foregroundStyle(.white.opacity(0.85))
}
.padding(.horizontal, 30)
.padding(.vertical, 24)
.glassBackground(RoundedRectangle(cornerRadius: 20, style: .continuous))
.overlay(
RoundedRectangle(cornerRadius: 20, style: .continuous)
.strokeBorder(.white.opacity(0.12), lineWidth: 1))
.transition(.opacity.combined(with: .scale(scale: 0.92)))
}
}
.environment(\.colorScheme, .dark) // the spinner + glass read over any frame
.animation(.easeInOut(duration: 0.22), value: active)
.allowsHitTesting(false) // the stream keeps receiving input the whole time
}
}
@@ -109,6 +109,16 @@ final class SessionModel: ObservableObject {
/// Mirrors StreamView's capture state (it owns the input capture; this drives the
/// HUD's "click to capture" / " releases" hint).
@Published var mouseCaptured = false
/// Resize overlay (design/midstream-resolution-resize.md client resize UX): true from the
/// instant a Match-window resize starts steering toward a new size until a frame at that size
/// decodes (or a safety timeout). Drives the blur+spinner so the unavoidable host-rebuild delay
/// reads as a deliberate, acknowledged transition instead of a stutter. Pure state lives in
/// `ResizeIndicator`; this mirrors its `active` for SwiftUI.
@Published private(set) var resizing = false
/// START = follower steering (main actor), END = a new-mode IDR's coded dims (decode pump,
/// hopped to main), TIMEOUT = safety net for a rejected/capped switch that never yields a
/// differently-sized frame. Ticked from the 1 Hz stats timer.
private var resizeIndicator = ResizeIndicator()
let meter = FrameMeter()
/// Capturereceived (the host+network stage), fed per AU at receipt by the stream view's
@@ -364,6 +374,8 @@ final class SessionModel: ObservableObject {
lostFrames = 0
lostPct = 0
mouseCaptured = false
resizing = false
resizeIndicator = ResizeIndicator() // no stale target/timer into the next session
}
/// Called (via the main actor) when the pump hits end-of-session.
@@ -374,6 +386,23 @@ final class SessionModel: ObservableObject {
errorMessage = "Session ended by \(name)."
}
/// Resize overlay START (main actor from the Match-window follower's `onResizeTarget`): the
/// window began differing from the live mode, so a `Reconfigure` toward `(width, height)` is
/// imminent. Show the blur+spinner immediately, before the debounced request even leaves.
func resizeTargeted(width: UInt32, height: UInt32) {
resizeIndicator.steering(
width: width, height: height, now: Date().timeIntervalSinceReferenceDate)
resizing = resizeIndicator.active
}
/// Resize overlay END (main actor hopped from the decode pump's `onDecodedSize`): a new-mode
/// IDR decoded at `(width, height)`. Clears the overlay only when that matches the size we're
/// steering to (a same-size loss-recovery IDR, or the initial connect IDR, is a no-op).
func resizeDecoded(width: Int, height: Int) {
resizeIndicator.decoded(width: UInt32(max(width, 0)), height: UInt32(max(height, 0)))
resizing = resizeIndicator.active
}
private func beginStreaming() {
guard let conn = connection else { return }
// Input capture itself is owned by StreamView (engaged by the captureEnabled
@@ -417,6 +446,11 @@ final class SessionModel: ObservableObject {
let timer = Timer(timeInterval: 1.0, repeats: true) { [weak self] _ in
guard let self else { return }
Task { @MainActor in
// Resize-overlay safety net: clear a stuck overlay when a targeted size never
// decodes (a rejected/capped switch). The decoded-frame END clears it promptly on
// success; this only fires after the timeout.
self.resizeIndicator.tick(now: Date().timeIntervalSinceReferenceDate)
self.resizing = self.resizeIndicator.active
let (frames, bytes, total) = self.meter.drain()
self.fps = frames
self.mbps = Double(bytes) * 8 / 1_000_000
@@ -13,6 +13,11 @@ extension SettingsView {
// failed exactly one slice: the iOS archive (macOS/tvOS never compile that branch).
@ViewBuilder var streamModeSection: some View {
Section {
#if os(iOS) || os(macOS)
// Match-window (design/midstream-resolution-resize.md D1): follow the session
// window/scene, renegotiating the host mode on a resize. Off the explicit mode below.
Toggle("Match window", isOn: $matchWindow)
#endif
#if os(iOS)
iosResolutionWheel
iosRefreshRows
@@ -35,8 +40,11 @@ extension SettingsView {
} header: {
Text("Stream mode")
} footer: {
Text("The host creates a virtual output at exactly this mode — "
+ "native resolution, no scaling. \(Self.bitrateFooter)")
Text(matchWindow
? "The stream follows this window — the host resizes its virtual output to match "
+ "as you resize, no scaling. \(Self.bitrateFooter)"
: "The host creates a virtual output at exactly this mode — "
+ "native resolution, no scaling. \(Self.bitrateFooter)")
.font(.geist(12, relativeTo: .caption))
.foregroundStyle(.secondary)
}
@@ -21,6 +21,7 @@ struct SettingsView: View {
@AppStorage(DefaultsKey.streamWidth) var width = 1920
@AppStorage(DefaultsKey.streamHeight) var height = 1080
@AppStorage(DefaultsKey.streamHz) var hz = 60
@AppStorage(DefaultsKey.matchWindow) var matchWindow = false
@AppStorage(DefaultsKey.compositor) var compositor = 0
@AppStorage(DefaultsKey.gamepadType) var gamepadType = 0
@AppStorage(DefaultsKey.bitrateKbps) var bitrateKbps = 0
@@ -11,6 +11,15 @@ public enum DefaultsKey {
public static let streamWidth = "punktfunk.width"
public static let streamHeight = "punktfunk.height"
public static let streamHz = "punktfunk.hz"
/// Match-window resolution policy (design/midstream-resolution-resize.md D1/D2): when on, the
/// stream mode FOLLOWS the session view the connect asks for the view's pixel size and a
/// mid-session resize (a windowed macOS window, an iPad Stage Manager / Split View scene)
/// renegotiates the host's virtual display + encoder (`PunktfunkConnection.requestMode`), so a
/// windowed session streams native-resolution pixels instead of scaling. Off (default): the
/// explicit `streamWidth`/`streamHeight` are used and never auto-resized (a fullscreen session
/// is native either way, so this degenerates to Auto-native there). Read per session by the
/// stream views' `MatchWindowFollower`.
public static let matchWindow = "punktfunk.matchWindow"
public static let compositor = "punktfunk.compositor"
public static let gamepadType = "punktfunk.gamepadType"
public static let gamepadID = "punktfunk.gamepadID"
@@ -0,0 +1,153 @@
// Match-window resize follower (design/midstream-resolution-resize.md D1/D2, client C3).
//
// The presenting view feeds this its PHYSICAL-PIXEL size on every layout; it debounces to
// resize-end, spaces requests 1 s apart, and asks the connection to switch the host's virtual
// display + encoder to match (`PunktfunkConnection.requestMode`) so a windowed macOS session or
// an iPad Stage Manager / Split View scene streams native-resolution pixels instead of scaling.
// The decode/present side needs nothing: VideoToolbox recreates its session on the keyframe-derived
// format-description change (the first new-mode AU is an IDR with fresh parameter sets).
//
// The trigger discipline is the shared cross-client one (mirrors the session binary's
// `resize_decision`): physical pixels rounded DOWN to even (the host rejects odd dimensions) and
// clamped 320×200; debounce to resize-end; 1 s between requests; skip a size equal to the live
// mode; and request each distinct size at most once which both stops re-asking a rejected size
// and keeps a host-side rollback (accepted, rebuild failed, corrective ack restored the old mode)
// from looping request rollback request.
import Foundation
/// The pure, side-effect-free core of the Match-window trigger so the normalize/skip discipline
/// is unit-tested without a live connection or a UI (`MatchWindowTests`).
public enum MatchWindow {
/// Even-floor + clamp a physical-pixel size to a host-valid mode dimension: the host's
/// `validate_dimensions` rejects odd sizes, and we never ask below 320×200.
public static func normalize(widthPx: Int, heightPx: Int) -> (width: UInt32, height: UInt32) {
let evenClamp: (Int, UInt32) -> UInt32 = { px, minimum in
let even = UInt32(max(px, 0)) / 2 * 2
return max(even, minimum)
}
return (evenClamp(widthPx, 320), evenClamp(heightPx, 200))
}
/// Whether to request `target` now (the debounce has already settled; spacing is the caller's
/// timer): `nil` to skip equal to the live mode, or already requested once (a rejected size /
/// a host rollback must not loop). `target` is expected already-[normalize]d.
public static func request(
target: (width: UInt32, height: UInt32),
current: (width: UInt32, height: UInt32),
lastRequested: (width: UInt32, height: UInt32)?
) -> (width: UInt32, height: UInt32)? {
if target.width == current.width, target.height == current.height { return nil }
if let lr = lastRequested, lr.width == target.width, lr.height == target.height { return nil }
return target
}
}
/// Owns the debounce timer + serialization state and drives `PunktfunkConnection.requestMode` from
/// the stream view's layout callbacks. Main-actor: the views feed it on the main thread and it reads
/// the connection's live mode there. Enabled per session from the `matchWindow` setting.
@MainActor
public final class MatchWindowFollower {
private weak var connection: PunktfunkConnection?
private let debounce: TimeInterval
private let minSpacing: TimeInterval
private var enabled: Bool
private var work: DispatchWorkItem?
private var pendingSize: (width: Int, height: Int)?
private var lastRequested: (width: UInt32, height: UInt32)?
private var lastRequestAt: Date?
/// The last size we reported via [`onResizeTarget`] dedups the per-layout stream of a drag so
/// the UI is notified once per distinct target, and reset to `nil` when the window is back in
/// sync with the live mode (so a later resize re-reports).
private var lastSteered: (width: UInt32, height: UInt32)?
/// Fired (on the main actor) the instant the window starts differing from the live mode i.e.
/// a resize is under way and a `Reconfigure` for `(width, height)` is imminent. Drives the
/// resize overlay's INSTANT feedback (blur + spinner) BEFORE the debounced request leaves; the
/// overlay clears when a decoded frame reaches this size (or on a timeout). Deduped per target.
public var onResizeTarget: ((_ width: UInt32, _ height: UInt32) -> Void)?
/// `debounce` = quiet time after the last size event before requesting (Win32 gets
/// `WM_EXITSIZEMOVE` for free; we debounce). `minSpacing` = floor between accepted requests
/// (a full host pipeline rebuild each). Defaults match the other clients.
public init(
connection: PunktfunkConnection,
enabled: Bool,
debounce: TimeInterval = 0.4,
minSpacing: TimeInterval = 1.0
) {
self.connection = connection
self.enabled = enabled
self.debounce = debounce
self.minSpacing = minSpacing
}
/// Turn following on/off live (a mid-session settings change; off cancels a pending request).
public func setEnabled(_ on: Bool) {
enabled = on
if !on {
work?.cancel()
work = nil
pendingSize = nil
lastSteered = nil
}
}
/// Feed the presenting view's current PHYSICAL-PIXEL size (its `bounds` × the backing/display
/// scale). Called from every layout pass; coalesced by the debounce so a drag-resize sends one
/// request at its end, never one per frame.
public func noteSize(widthPx: Int, heightPx: Int) {
guard enabled else { return }
pendingSize = (widthPx, heightPx)
schedule()
reportSteering(widthPx: widthPx, heightPx: heightPx)
}
/// Report the resize overlay's START signal (deduped): the moment the normalized window size
/// differs from the live mode we're steering toward a new size. No connection / no negotiated
/// mode yet nothing to compare against, skip.
private func reportSteering(widthPx: Int, heightPx: Int) {
guard let connection else { return }
let target = MatchWindow.normalize(widthPx: widthPx, heightPx: heightPx)
let mode = connection.currentMode()
guard mode.width > 0, mode.height > 0 else { return }
if target.width == mode.width, target.height == mode.height {
lastSteered = nil // back in sync a later change re-reports
return
}
if lastSteered?.width == target.width, lastSteered?.height == target.height { return }
lastSteered = target
onResizeTarget?(target.width, target.height)
}
private func schedule() {
work?.cancel()
let item = DispatchWorkItem { [weak self] in self?.fire() }
work = item
DispatchQueue.main.asyncAfter(deadline: .now() + debounce, execute: item)
}
private func fire() {
guard enabled, let connection, let size = pendingSize else { return }
// 1 s spacing: a request went out recently re-arm the debounce and retry later rather
// than fire early (keeps at most ~one request outstanding the accept ack round-trips in
// milliseconds, ahead of the host's rebuild).
if let last = lastRequestAt, Date().timeIntervalSince(last) < minSpacing {
schedule()
return
}
let target = MatchWindow.normalize(widthPx: size.width, heightPx: size.height)
let mode = connection.currentMode()
pendingSize = nil
guard let req = MatchWindow.request(
target: target,
current: (mode.width, mode.height),
lastRequested: lastRequested
) else { return }
// Keep the current refresh Match-window follows SIZE, not rate.
connection.requestMode(width: req.width, height: req.height, refreshHz: mode.refreshHz)
lastRequested = req
lastRequestAt = Date()
}
}
@@ -0,0 +1,63 @@
// Resize-in-progress indicator state (design/midstream-resolution-resize.md client UX).
//
// A mid-stream resize takes the host 0.32 s to rebuild its virtual display + encoder, and the
// first new-mode frame is an IDR that the decoder re-inits on. Rather than let the stream scale
// (stretch/blur) to the changing window during that gap, the client EMBRACES the delay: it shows a
// deliberate blur + spinner the instant a resize starts and clears it the instant the sharp
// new-resolution frame is on screen so the wait reads as intentional, not as lag.
//
// This is driven ENTIRELY by signals the client already has (no new protocol):
// * START the Match-window follower reports the size it is steering toward (instant, on the
// first resize layout, before the debounced request even leaves).
// * END the decode pipeline reports each new-mode IDR's dimensions; when they reach the target
// the new picture is here.
// * TIMEOUT the safety net for a switch that never delivers the exact target: the host rejected
// it (gamescope), capped it to an advertised mode, or a corrective ack landed a different size.
//
// Pure + side-effect-free so the transition logic is unit-tested without a live session or UI
// (`ResizeIndicatorTests`); `SessionModel` owns an instance and mirrors `active` into a @Published.
import Foundation
/// The pure state of the resize overlay. `now` is a monotonic time in seconds (the caller passes
/// `ProcessInfo.processInfo.systemUptime` or a test clock).
public struct ResizeIndicator {
/// Whether the blur + spinner should be shown.
public private(set) var active = false
/// The size the follower is steering toward cleared once a decoded frame reaches it.
private var target: (width: UInt32, height: UInt32)?
/// When the current `active` span began the timeout is measured from here.
private var since: TimeInterval?
/// How long to keep the overlay up if the target frame never arrives (rejected / capped switch).
public var timeout: TimeInterval
public init(timeout: TimeInterval = 2.5) { self.timeout = timeout }
/// The follower is steering toward `width`×`height` a resize is under way. Show the overlay now
/// (instant feedback). Called only for a genuine change (the follower skips a target equal to the
/// live mode), possibly many times as a drag moves through sizes; the timeout re-arms whenever the
/// target actually changes so a slow drag never trips it mid-gesture.
public mutating func steering(width: UInt32, height: UInt32, now: TimeInterval) {
if !active || target?.width != width || target?.height != height {
since = now
}
target = (width, height)
active = true
}
/// A decoded frame arrived at `width`×`height` (a new-mode IDR). Clears the overlay once it
/// matches the steered target the sharp new-resolution picture is on glass.
public mutating func decoded(width: UInt32, height: UInt32) {
guard active, let t = target, t.width == width, t.height == height else { return }
active = false
since = nil
}
/// Timeout safety net: stop showing the overlay once `timeout` has elapsed with no matching frame
/// (a rejected or host-capped switch never delivers the exact target).
public mutating func tick(now: TimeInterval) {
guard active, let s = since, now - s >= timeout else { return }
active = false
since = nil
}
}
@@ -85,7 +85,8 @@ final class SessionPresenter {
displayMeter: LatencyMeter? = nil,
makeDisplayLink: (AnyObject, Selector) -> CADisplayLink,
onFrame: (@Sendable (AccessUnit) -> Void)?,
onSessionEnd: (@Sendable () -> Void)?
onSessionEnd: (@Sendable () -> Void)?,
onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil
) {
stop()
self.connection = connection
@@ -128,12 +129,14 @@ final class SessionPresenter {
link.add(to: .main, forMode: .common)
stage2Link = link
syncFrameRate(hz: connection.currentMode().refreshHz)
pipeline.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
pipeline.start(
connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd,
onDecodedSize: onDecodedSize)
} else {
let pump = StreamPump()
pump.start(
connection: connection, layer: baseLayer,
onFrame: onFrame, onSessionEnd: onSessionEnd)
onFrame: onFrame, onSessionEnd: onSessionEnd, onDecodedSize: onDecodedSize)
self.pump = pump
}
}
@@ -329,7 +329,8 @@ public final class Stage2Pipeline {
public func start(
connection: PunktfunkConnection,
onFrame: (@Sendable (AccessUnit) -> Void)?,
onSessionEnd: (@Sendable () -> Void)?
onSessionEnd: (@Sendable () -> Void)?,
onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil
) {
offsetNs = connection.clockOffsetNs
recovery.bind(connection) // arm host-keyframe recovery for this session
@@ -350,6 +351,9 @@ public final class Stage2Pipeline {
let thread = Thread {
defer { pumpStopped.signal() } // let stop() join the pump (bounded) before decoder.reset()
var format: CMVideoFormatDescription?
// Report coded dims to the resize overlay only on a CHANGE (new-mode IDR), not per
// loss-recovery IDR at the same size (see StreamPump).
var lastDecodedDims: CMVideoDimensions?
var lastFramesDropped = connection.framesDropped()
// Persistent recovery WANT, not a one-shot edge (see StreamPump for the full rationale):
// keep asking until an IDR lands so a request swallowed by the throttle is re-sent.
@@ -387,6 +391,11 @@ public final class Stage2Pipeline {
onFrame?(au)
if let f = connection.videoCodec.formatDescription(fromKeyframe: au.data) {
format = f // refreshed on every IDR (mode changes included)
let dims = CMVideoFormatDescriptionGetDimensions(f)
if lastDecodedDims?.width != dims.width || lastDecodedDims?.height != dims.height {
lastDecodedDims = dims
onDecodedSize?(Int(dims.width), Int(dims.height))
}
awaitingIDR = false // a fresh IDR re-anchored decode recovery complete
}
guard let f = format, !token.isStopped else { return true }
@@ -21,7 +21,8 @@ final class StreamPump {
connection: PunktfunkConnection,
layer: AVSampleBufferDisplayLayer,
onFrame: (@Sendable (AccessUnit) -> Void)?,
onSessionEnd: (@Sendable () -> Void)?
onSessionEnd: (@Sendable () -> Void)?,
onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil
) {
let token = token
// Coalesced host keyframe requests (100 ms throttle see KeyframeRecovery).
@@ -35,6 +36,9 @@ final class StreamPump {
let thread = Thread {
var format: CMVideoFormatDescription?
// Report the coded dims to the resize overlay only when they CHANGE (a new-mode IDR),
// not on every loss-recovery IDR at the same size so it fires once per real switch.
var lastDecodedDims: CMVideoDimensions?
var lastFramesDropped = connection.framesDropped()
// Recovery is a persistent WANT, not a one-shot edge: set it on detected loss (or a
// decoder reset), retry the throttled request EVERY iteration, and clear it only when a
@@ -79,6 +83,11 @@ final class StreamPump {
let idrFormat = connection.videoCodec.formatDescription(fromKeyframe: au.data)
if let f = idrFormat {
format = f // refreshed on every IDR (mode changes included)
let dims = CMVideoFormatDescriptionGetDimensions(f)
if lastDecodedDims?.width != dims.width || lastDecodedDims?.height != dims.height {
lastDecodedDims = dims
onDecodedSize?(Int(dims.width), Int(dims.height))
}
if awaitingIDR {
let ms = Int(Date().timeIntervalSince(awaitingSince) * 1000)
pumpLog.notice("video: recovery IDR received — resumed after \(ms, privacy: .public) ms")
@@ -87,6 +87,8 @@ public struct StreamView: NSViewRepresentable {
private let onDisconnectRequest: (() -> Void)?
private let onFrame: (@Sendable (AccessUnit) -> Void)?
private let onSessionEnd: (@Sendable () -> Void)?
private let onResizeTarget: ((UInt32, UInt32) -> Void)?
private let onDecodedSize: (@Sendable (Int, Int) -> Void)?
private let endToEndMeter: LatencyMeter?
private let decodeMeter: LatencyMeter?
private let displayMeter: LatencyMeter?
@@ -108,6 +110,8 @@ public struct StreamView: NSViewRepresentable {
onDisconnectRequest: (() -> Void)? = nil,
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
onSessionEnd: (@Sendable () -> Void)? = nil,
onResizeTarget: ((UInt32, UInt32) -> Void)? = nil,
onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil,
endToEndMeter: LatencyMeter? = nil,
decodeMeter: LatencyMeter? = nil,
displayMeter: LatencyMeter? = nil
@@ -118,6 +122,8 @@ public struct StreamView: NSViewRepresentable {
self.onDisconnectRequest = onDisconnectRequest
self.onFrame = onFrame
self.onSessionEnd = onSessionEnd
self.onResizeTarget = onResizeTarget
self.onDecodedSize = onDecodedSize
self.endToEndMeter = endToEndMeter
self.decodeMeter = decodeMeter
self.displayMeter = displayMeter
@@ -131,6 +137,8 @@ public struct StreamView: NSViewRepresentable {
view.endToEndMeter = endToEndMeter
view.decodeMeter = decodeMeter
view.displayMeter = displayMeter
view.onResizeTarget = onResizeTarget
view.onDecodedSize = onDecodedSize
view.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
return view
}
@@ -142,6 +150,8 @@ public struct StreamView: NSViewRepresentable {
view.endToEndMeter = endToEndMeter
view.decodeMeter = decodeMeter
view.displayMeter = displayMeter
view.onResizeTarget = onResizeTarget
view.onDecodedSize = onDecodedSize
// SwiftUI reuses the NSView across state changes repoint the pump only when the
// connection identity actually changed.
if view.connection !== connection {
@@ -165,6 +175,9 @@ public final class StreamLayerView: NSView {
/// stage-1 StreamPump displayLayer path as the Metal-unavailable / DEBUG fallback.
private let presenter = SessionPresenter()
public private(set) var connection: PunktfunkConnection?
/// Match-window resize follower (C3) non-nil while a session is active AND the `matchWindow`
/// setting is on; fed the view's physical-pixel size on every relayout.
private var matchFollower: MatchWindowFollower?
private let cursorCapture = CursorCapture()
private var inputCapture: InputCapture?
private var appObservers: [NSObjectProtocol] = []
@@ -201,6 +214,13 @@ public final class StreamLayerView: NSView {
/// view can't do that itself (the connection's owner disconnects).
public var onDisconnectRequest: (() -> Void)?
/// Resize overlay signals (design/midstream-resolution-resize.md client UX): `onResizeTarget`
/// (main thread, via the follower) fires the instant the window starts steering toward a new
/// size; `onDecodedSize` (PUMP thread) fires when a new-mode IDR's dims land. The owner drives
/// the blur+spinner from these set before `start()`.
public var onResizeTarget: ((UInt32, UInt32) -> Void)?
public var onDecodedSize: (@Sendable (Int, Int) -> Void)?
/// Main-thread only. False = input capture disabled outright (UI layered over the
/// stream); flipping to true auto-engages once.
public var captureEnabled = true {
@@ -626,15 +646,32 @@ public final class StreamLayerView: NSView {
displayMeter: displayMeter,
makeDisplayLink: { displayLink(target: $0, selector: $1) },
onFrame: onFrame,
onSessionEnd: onSessionEnd)
onSessionEnd: onSessionEnd,
onDecodedSize: onDecodedSize) // resize overlay END signal (new-mode IDR dims)
// Match-window (C3): follow the window's pixel size when the setting is on. Latched at
// session start (mirrors the other clients); the first real `layout()` feeds the initial
// size, so the stream converges to the window even if the connect used the explicit mode.
let follower = MatchWindowFollower(
connection: connection,
enabled: UserDefaults.standard.bool(forKey: DefaultsKey.matchWindow))
follower.onResizeTarget = onResizeTarget // resize overlay START signal (instant, on the follower)
matchFollower = follower
layoutPresenter()
requestAutoCapture() // entering a session is the deliberate "capture me" moment
}
/// Aspect-fit the stage-2 metal sublayer to the view; refresh contentsScale on a
/// retinanon-retina move (see SessionPresenter.layout).
/// retinanon-retina move (see SessionPresenter.layout). Also feeds the Match-window follower
/// the view's physical-pixel size (bounds backing), so a window resize / retina move follows.
private func layoutPresenter() {
presenter.layout(in: bounds, contentsScale: window?.backingScaleFactor ?? 1)
// Feed the follower only once in a window (backing scale is real then) and with real
// bounds a pre-window layout would report point-sized dimensions.
if window != nil, bounds.width > 0, bounds.height > 0 {
let px = convertToBacking(bounds).size
matchFollower?.noteSize(
widthPx: Int(px.width.rounded()), heightPx: Int(px.height.rounded()))
}
}
public override func viewDidChangeBackingProperties() {
@@ -650,6 +687,7 @@ public final class StreamLayerView: NSView {
inputCapture?.stop()
inputCapture = nil
presenter.stop()
matchFollower = nil
connection = nil
}
@@ -53,6 +53,8 @@ public struct StreamView: UIViewControllerRepresentable {
private let onCaptureChange: ((Bool) -> Void)?
private let onFrame: (@Sendable (AccessUnit) -> Void)?
private let onSessionEnd: (@Sendable () -> Void)?
private let onResizeTarget: ((UInt32, UInt32) -> Void)?
private let onDecodedSize: (@Sendable (Int, Int) -> Void)?
private let endToEndMeter: LatencyMeter?
private let decodeMeter: LatencyMeter?
private let displayMeter: LatencyMeter?
@@ -68,6 +70,8 @@ public struct StreamView: UIViewControllerRepresentable {
onDisconnectRequest: (() -> Void)? = nil,
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
onSessionEnd: (@Sendable () -> Void)? = nil,
onResizeTarget: ((UInt32, UInt32) -> Void)? = nil,
onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil,
endToEndMeter: LatencyMeter? = nil,
decodeMeter: LatencyMeter? = nil,
displayMeter: LatencyMeter? = nil
@@ -77,6 +81,8 @@ public struct StreamView: UIViewControllerRepresentable {
self.onCaptureChange = onCaptureChange
self.onFrame = onFrame
self.onSessionEnd = onSessionEnd
self.onResizeTarget = onResizeTarget
self.onDecodedSize = onDecodedSize
self.endToEndMeter = endToEndMeter
self.decodeMeter = decodeMeter
self.displayMeter = displayMeter
@@ -89,6 +95,8 @@ public struct StreamView: UIViewControllerRepresentable {
controller.endToEndMeter = endToEndMeter
controller.decodeMeter = decodeMeter
controller.displayMeter = displayMeter
controller.onResizeTarget = onResizeTarget
controller.onDecodedSize = onDecodedSize
controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
return controller
}
@@ -99,6 +107,8 @@ public struct StreamView: UIViewControllerRepresentable {
controller.endToEndMeter = endToEndMeter
controller.decodeMeter = decodeMeter
controller.displayMeter = displayMeter
controller.onResizeTarget = onResizeTarget
controller.onDecodedSize = onDecodedSize
if controller.connection !== connection {
controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
}
@@ -147,6 +157,11 @@ public final class StreamViewController: StreamViewControllerBase {
/// Capture state at the last resign, restored on the next foreground otherwise the
/// mouse/keyboard stay released after navigating out and nothing re-grabs them.
private var wasCapturedOnResign = false
/// Match-window resize follower (C3) non-nil while a session is active AND the `matchWindow`
/// setting is on; fed the view's physical-pixel size from `viewDidLayoutSubviews` so an iPad
/// Stage Manager / Split View scene resize renegotiates the host mode. iOS only (iPhone
/// naturally no-ops fullscreen; tvOS drives display modes via AVDisplayManager instead).
private var matchFollower: MatchWindowFollower?
#endif
/// Reads whether the scene's pointer is actually locked right now; nil = state
@@ -161,6 +176,13 @@ public final class StreamViewController: StreamViewControllerBase {
}
var onCaptureChange: ((Bool) -> Void)?
/// Resize-overlay START: forwarded to the Match-window follower so a scene resize drives the
/// blur+spinner the instant the window differs from the live mode (iOS only tvOS has no
/// follower). See `MatchWindowFollower.onResizeTarget`.
var onResizeTarget: ((UInt32, UInt32) -> Void)?
/// Resize-overlay END: the presenter reports the coded dims of each new-mode IDR here, so the
/// overlay clears when a frame at the requested size actually decodes.
var onDecodedSize: (@Sendable (Int, Int) -> Void)?
var captureEnabled = true {
didSet {
@@ -327,6 +349,14 @@ public final class StreamViewController: StreamViewControllerBase {
}
capture.start()
inputCapture = capture
// Match-window (C3): follow the scene's pixel size when the setting is on. Latched at
// session start (mirrors the other clients); `viewDidLayoutSubviews` feeds it covers
// Stage Manager / Split View resizes and rotation. iPhone fullscreen naturally no-ops.
let follower = MatchWindowFollower(
connection: connection,
enabled: UserDefaults.standard.bool(forKey: DefaultsKey.matchWindow))
follower.onResizeTarget = onResizeTarget
matchFollower = follower
#endif
// Presenter choice + lifecycle live in SessionPresenter (shared with macOS): stage-2
@@ -340,7 +370,8 @@ public final class StreamViewController: StreamViewControllerBase {
displayMeter: displayMeter,
makeDisplayLink: { CADisplayLink(target: $0, selector: $1) },
onFrame: onFrame,
onSessionEnd: onSessionEnd)
onSessionEnd: onSessionEnd,
onDecodedSize: onDecodedSize)
layoutMetalLayer()
#if os(iOS)
@@ -411,6 +442,7 @@ public final class StreamViewController: StreamViewControllerBase {
streamView.onPointerButton = nil
streamView.onScroll = nil
streamView.currentHostMode = nil
matchFollower = nil
#endif
#if os(tvOS)
// Return the TV to the user's preferred mode the home screen must not stay in the
@@ -425,6 +457,16 @@ public final class StreamViewController: StreamViewControllerBase {
public override func viewDidLayoutSubviews() {
super.viewDidLayoutSubviews()
layoutMetalLayer()
#if os(iOS)
// Match-window (C3): feed the follower the view's physical-pixel size (points × scale).
let b = streamView.bounds
if b.width > 0, b.height > 0 {
let scale = renderScale
matchFollower?.noteSize(
widthPx: Int((b.width * scale).rounded()),
heightPx: Int((b.height * scale).rounded()))
}
#endif
#if os(tvOS)
applyDisplayCriteriaIfNeeded()
#endif
@@ -0,0 +1,43 @@
// The Match-window trigger discipline (design/midstream-resolution-resize.md D2), as pure
// functions the same rules the session binary's `resize_decision` unit-tests: physical pixels
// even-floored and clamped 320×200, skip a size equal to the live mode, and request each
// distinct size at most once (so a rejected size / a host rollback can't loop).
import XCTest
@testable import PunktfunkKit
final class MatchWindowTests: XCTestCase {
func testNormalizeEvenFloorsAndClamps() {
// Odd pixels floor to even (the host rejects odd dimensions).
let a = MatchWindow.normalize(widthPx: 1001, heightPx: 601)
XCTAssertEqual(a.width, 1000)
XCTAssertEqual(a.height, 600)
// Already-even sizes pass through.
let b = MatchWindow.normalize(widthPx: 2560, heightPx: 1440)
XCTAssertEqual(b.width, 2560)
XCTAssertEqual(b.height, 1440)
// Tiny / zero clamp to the host floor.
let c = MatchWindow.normalize(widthPx: 100, heightPx: 80)
XCTAssertEqual(c.width, 320)
XCTAssertEqual(c.height, 200)
let z = MatchWindow.normalize(widthPx: 0, heightPx: -4)
XCTAssertEqual(z.width, 320)
XCTAssertEqual(z.height, 200)
}
func testRequestSkipsEqualAndAlreadyRequested() {
// A new size (different from the live mode, not yet requested) request it.
let r = MatchWindow.request(
target: (1000, 600), current: (1280, 720), lastRequested: (800, 500))
XCTAssertEqual(r?.width, 1000)
XCTAssertEqual(r?.height, 600)
// Equal to the live mode nothing to do.
XCTAssertNil(MatchWindow.request(
target: (1280, 720), current: (1280, 720), lastRequested: nil))
// Already requested once don't re-ask (covers a rejected size AND a host rollback:
// accepted rebuild failed corrective ack restored the old mode must not loop).
XCTAssertNil(MatchWindow.request(
target: (1000, 600), current: (1280, 720), lastRequested: (1000, 600)))
}
}
@@ -0,0 +1,52 @@
import XCTest
@testable import PunktfunkKit
final class ResizeIndicatorTests: XCTestCase {
func testInactiveUntilSteered() {
var r = ResizeIndicator()
XCTAssertFalse(r.active)
// A decoded frame with nothing pending is a no-op (session start / steady state).
r.decoded(width: 1920, height: 1080)
XCTAssertFalse(r.active)
}
func testSteeringActivatesAndDecodedTargetClears() {
var r = ResizeIndicator()
r.steering(width: 2560, height: 1440, now: 0)
XCTAssertTrue(r.active)
// A frame at a DIFFERENT size (the old mode still draining) doesn't clear it.
r.decoded(width: 1920, height: 1080)
XCTAssertTrue(r.active)
// The target frame lands clear.
r.decoded(width: 2560, height: 1440)
XCTAssertFalse(r.active)
}
func testTimeoutClearsWhenTargetNeverArrives() {
var r = ResizeIndicator(timeout: 2.5)
r.steering(width: 2560, height: 1440, now: 10)
r.tick(now: 12) // 2 s < timeout still up
XCTAssertTrue(r.active)
r.tick(now: 12.6) // 2.6 s timeout a rejected/capped switch clears
XCTAssertFalse(r.active)
}
func testDragReArmsTimeoutOnEachNewTarget() {
var r = ResizeIndicator(timeout: 2.5)
r.steering(width: 2000, height: 1200, now: 0)
r.steering(width: 2200, height: 1200, now: 2) // target changed since re-armed to 2
r.tick(now: 4) // only 2 s since the last change still up (drag isn't a timeout)
XCTAssertTrue(r.active)
r.tick(now: 4.6) // 2.6 s since the last change clears
XCTAssertFalse(r.active)
}
func testSteadyDragDoesNotResetTimeout() {
var r = ResizeIndicator(timeout: 2.5)
r.steering(width: 2560, height: 1440, now: 0)
r.steering(width: 2560, height: 1440, now: 1) // SAME target since stays 0
r.tick(now: 2.6) // 2.6 s since the ORIGINAL steer clears (not reset by the repeat)
XCTAssertFalse(r.active)
}
}
+29 -16
View File
@@ -264,21 +264,23 @@ pub fn show(
let page = adw::PreferencesPage::new();
let stream = adw::PreferencesGroup::builder().title("Stream").build();
let res_names: Vec<String> = RESOLUTIONS
.iter()
.map(|&(w, h)| {
if w == 0 {
"Native display".to_string()
} else {
format!("{w} × {h}")
}
})
// The D1 tri-state: Native, Match window (a virtual index 1, stored as the
// `match_window` flag), then the explicit sizes.
let res_names: Vec<String> = std::iter::once("Native display".to_string())
.chain(std::iter::once("Match window".to_string()))
.chain(
RESOLUTIONS
.iter()
.skip(1)
.map(|&(w, h)| format!("{w} × {h}")),
)
.collect();
let res_row = ChoiceRow::new(
&dialog,
inline,
"Resolution",
"The host creates a virtual output at exactly this size",
"The host creates a virtual output at exactly this size — Match window follows \
the stream window, including mid-stream resizes",
&res_names.iter().map(String::as_str).collect::<Vec<_>>(),
);
let hz_names: Vec<String> = REFRESH
@@ -470,10 +472,15 @@ pub fn show(
// Seed from the current settings.
{
let s = settings.borrow();
let res_i = RESOLUTIONS
.iter()
.position(|&(w, h)| w == s.width && h == s.height)
.unwrap_or(0);
let res_i = if s.match_window {
1
} else {
RESOLUTIONS
.iter()
.position(|&(w, h)| w == s.width && h == s.height)
.map(|i| if i == 0 { 0 } else { i + 1 })
.unwrap_or(0)
};
res_row.set_selected(res_i as u32);
let hz_i = REFRESH.iter().position(|&r| r == s.refresh_hz).unwrap_or(0);
hz_row.set_selected(hz_i as u32);
@@ -508,8 +515,14 @@ pub fn show(
dialog.add(&page);
dialog.connect_closed(move |_| {
let mut s = settings.borrow_mut();
let (w, h) = RESOLUTIONS[(res_row.selected() as usize).min(RESOLUTIONS.len() - 1)];
(s.width, s.height) = (w, h);
// Index 1 is the virtual "Match window" option; 0 = Native, 2.. = explicit.
let res_i = (res_row.selected() as usize).min(RESOLUTIONS.len());
s.match_window = res_i == 1;
(s.width, s.height) = if res_i <= 1 {
(0, 0)
} else {
RESOLUTIONS[res_i - 1]
};
s.refresh_hz = REFRESH[(hz_row.selected() as usize).min(REFRESH.len() - 1)];
s.bitrate_kbps = (bitrate_row.value() * 1000.0) as u32;
s.gamepad = GAMEPADS[(pad_row.selected() as usize).min(GAMEPADS.len() - 1)].to_string();
+4
View File
@@ -145,6 +145,10 @@ pub fn run(target: Option<&str>) -> u8 {
trust::touch_last_used(&trust::hex(&fingerprint));
})),
overlay: Some(Box::new(overlay)),
window_size: crate::session_main::window_size(&settings_at_start),
// Latched at console start (like the stats tier above): toggling Match window in
// the console's settings screen applies from the next console launch.
match_window: crate::session_main::match_window(&settings_at_start),
};
let result =
+30
View File
@@ -164,6 +164,34 @@ mod session_main {
}
}
/// The window's starting size under Match-window: the persisted last size, so the
/// first connect's mode already matches the glass; `None` (policy off / never
/// stored) = the 1280×720 default.
pub(crate) fn window_size(settings: &trust::Settings) -> Option<(u32, u32)> {
(settings.match_window && settings.last_window_w > 0 && settings.last_window_h > 0)
.then_some((settings.last_window_w, settings.last_window_h))
}
/// The Match-window policy hook for the presenter loop
/// (design/midstream-resolution-resize.md D1/D2): `Some(persist)` turns the
/// debounced resize→`Reconfigure` machinery on; the callback stores each resize-end's
/// logical window size (load-modify-save, like the console settings screen) so the
/// next launch opens at it.
pub(crate) fn match_window(
settings: &trust::Settings,
) -> Option<Box<dyn FnMut(u32, u32)>> {
settings.match_window.then(|| {
Box::new(|w: u32, h: u32| {
let mut s = trust::Settings::load();
if (s.last_window_w, s.last_window_h) != (w, h) {
s.last_window_w = w;
s.last_window_h = h;
s.save();
}
}) as Box<dyn FnMut(u32, u32)>
})
}
/// One JSON status line on stdout (the shell parses these; strings hand-escaped via
/// the minimal rules a reason string can need). `pub(crate)`: browse mode emits its
/// failure through the same contract when spawned with `--json-status`.
@@ -343,6 +371,8 @@ mod session_main {
overlay: Some(Box::new(pf_console_ui::SkiaOverlay::new())),
#[cfg(not(feature = "ui"))]
overlay: None,
window_size: window_size(&settings),
match_window: match_window(&settings),
};
let outcome =
+23 -15
View File
@@ -136,29 +136,37 @@ pub(crate) fn settings_page(
let s = ctx.settings.lock().unwrap().clone();
// --- Display ---------------------------------------------------------------------------
// The D1 tri-state: Native, Match window (a virtual index 1, stored as the
// `match_window` flag), then the explicit sizes.
let (res_names, res_i) = {
let names: Vec<String> = RESOLUTIONS
.iter()
.map(|&(w, h)| {
if w == 0 {
"Native display".into()
} else {
format!("{w} \u{00D7} {h}")
}
})
let names: Vec<String> = std::iter::once("Native display".to_string())
.chain(std::iter::once("Match window".to_string()))
.chain(
RESOLUTIONS
.iter()
.skip(1)
.map(|&(w, h)| format!("{w} \u{00D7} {h}")),
)
.collect();
let i = RESOLUTIONS
.iter()
.position(|&(w, h)| w == s.width && h == s.height)
.unwrap_or(0);
let i = if s.match_window {
1
} else {
RESOLUTIONS
.iter()
.position(|&(w, h)| w == s.width && h == s.height)
.map(|i| if i == 0 { 0 } else { i + 1 })
.unwrap_or(0)
};
(names, i)
};
let res_combo = setting_combo(ctx, "Resolution", res_names, res_i, |s, i| {
(s.width, s.height) = RESOLUTIONS[i];
s.match_window = i == 1;
(s.width, s.height) = if i <= 1 { (0, 0) } else { RESOLUTIONS[i - 1] };
})
.tooltip(
"The host creates a virtual display at exactly this size. \u{201C}Native display\u{201D} \
resolves to the monitor this window is on at connect.",
resolves to the monitor this window is on at connect; \u{201C}Match window\u{201D} \
follows the stream window, including mid-stream resizes.",
);
let (hz_names, hz_i) = {
let names: Vec<String> = REFRESH