Merge pull request 'fix(client/ios): Escape keeps the pointer captured instead of handing it back to iPadOS' (#19) from worktree-ipad-esc-pointer-relock into main

Reviewed-on: #19

clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift

@@ -175,6 +175,46 @@ public final class StreamViewController: StreamViewControllerBase {
     /// renegotiates the host mode (1:1, no presenter resample). iOS only (iPhone naturally no-ops
     /// its fixed full-screen scene; tvOS drives display modes via AVDisplayManager instead).
     private var matchFollower: MatchWindowFollower?
+    // MARK: Escape-drop re-lock
+    //
+    // iPadOS releases the pointer lock BY ITSELF when the user presses Escape — the platform's
+    // built-in "let me out", mirroring the web Pointer Lock API's default unlock gesture. Nothing
+    // in our code does it: a bare Esc never touches `captured`, so it keeps forwarding to the host
+    // as the game key it is. But the lock going away flips the mouse onto the absolute UIKit path
+    // and un-hides the iPadOS cursor, so hitting Esc for an in-game menu silently costs the capture
+    // until the user clicks to win it back. Esc is a GAME key here, not a request to hand the
+    // pointer back to iPadOS, so an unwanted drop is re-requested below. The DELIBERATE releases
+    // (⌘⎋, ⌃⌥⇧Q, the Stream menu, backgrounding) all clear `captured` first, so `wantsPointerLock`
+    // is already false when their drop is observed and none of them are fought here.
+    /// Whether this capture ever actually held the lock. Only a lock we HELD is worth winning back
+    /// — never having been granted one means the scene doesn't qualify, not that Esc took it.
+    /// Cleared when capture ends, so each capture starts from a clean slate.
+    private var pointerLockWasEngaged = false
+    /// Attempts spent in the current re-lock burst, and when the burst began.
+    private var pointerRelockAttempt = 0
+    private var pointerRelockBurstStart: CFTimeInterval = 0
+    /// True from an unwanted drop until the lock is back (or the burst gives up). While pending,
+    /// the local cursor stays hidden and absolute pointer MOTION stays muted, so a re-lock that
+    /// lands a frame or two later is invisible instead of flashing the iPadOS cursor and
+    /// teleporting the host's to the pointer's absolute position.
+    private var pointerRelockPending = false
+    /// Forces `prefersPointerLocked` to report false for one resolve pass, so the escalated attempt
+    /// presents the system with a genuine false→true transition instead of re-asserting a value it
+    /// already holds. See `requestPointerRelock()`.
+    private var pointerLockForcedOff = false
+    /// A burst is 3 attempts, and a burst can't restart inside 2 s. A scene the system will never
+    /// lock (Stage Manager, Split View) therefore costs three cheap re-resolves and then falls back
+    /// to today's click-to-recapture, rather than retrying forever.
+    private static let pointerRelockAttemptLimit = 3
+    private static let pointerRelockBurstWindow: CFTimeInterval = 2
+    /// Gap between attempts in a burst — long enough for the system to answer the previous
+    /// re-resolve, short enough that the whole burst fits in ~0.6 s. Must exceed
+    /// `pointerLockForcedOffHold` so an escalated attempt is back to preferring the lock before the
+    /// next attempt evaluates.
+    private static let pointerRelockRetryDelay: TimeInterval = 0.2
+    /// How long an escalated attempt reports `prefersPointerLocked == false` before flipping back,
+    /// so the system observes a real transition instead of coalescing the flip away.
+    private static let pointerLockForcedOffHold: TimeInterval = 0.05
     #endif
 
     /// Reads whether the scene's pointer is actually locked right now; nil = state
@@ -260,7 +300,7 @@ public final class StreamViewController: StreamViewControllerBase {
         captured && pointerCaptureEnabled && UIDevice.current.userInterfaceIdiom == .pad
     }
 
-    public override var prefersPointerLocked: Bool { wantsPointerLock }
+    public override var prefersPointerLocked: Bool { wantsPointerLock && !pointerLockForcedOff }
     public override var prefersHomeIndicatorAutoHidden: Bool { true }
 
     // NOTE: we deliberately do NOT override `childViewControllerForPointerLock`. The default
@@ -383,6 +423,11 @@ public final class StreamViewController: StreamViewControllerBase {
         // is the exact mirror of the GCMouse handlers, which fire only while locked.
         streamView.onPointerMoveAbs = { [weak self] p in
             guard let self, self.inputCapture?.gcMouseForwarding == false else { return }
+            // A re-lock is in flight after an Esc-drop: the absolute path would teleport the host
+            // cursor to wherever the local pointer sits, undoing the relative aiming we're about to
+            // resume. Motion only — BUTTONS still forward (they carry no position, so a click during
+            // the couple of frames a re-lock takes must not be swallowed mid-firefight).
+            guard !self.pointerRelockPending else { return }
             self.inputCapture?.sendMouseAbs(
                 x: p.x, y: p.y, surfaceWidth: p.w, surfaceHeight: p.h)
         }
@@ -693,6 +738,24 @@ public final class StreamViewController: StreamViewControllerBase {
     /// change and capture toggle. Main queue.
     private func syncPointerLock() {
         let locked = pointerLockEngaged() == true
+        // Wanted, previously HELD, and now gone is the Esc-drop signature. The "previously held"
+        // half matters: a lock that was never granted is a scene that doesn't qualify (Stage
+        // Manager, Split View), and burst-requesting there would hide the cursor for the burst's
+        // duration to win a lock that isn't coming. A first grant is already driven by the chain
+        // engage in setCaptured/viewDidAppear.
+        if locked {
+            pointerLockWasEngaged = true
+            pointerRelockPending = false
+            pointerRelockAttempt = 0
+        } else if wantsPointerLock, pointerLockWasEngaged {
+            requestPointerRelock()
+        } else {
+            // Capture is gone (or the lock was never ours) — settle, and let the next capture
+            // start from a clean "never held" slate.
+            if !wantsPointerLock { pointerLockWasEngaged = false }
+            pointerRelockPending = false
+            pointerRelockAttempt = 0
+        }
         let useGCMouse = captured && locked
         // Lock dropped (or capture ended) while the GCMouse path held a button down: once
         // gcMouseForwarding flips false its release handler is gated off, so flush any held
@@ -704,7 +767,83 @@ public final class StreamViewController: StreamViewControllerBase {
         pointerInteraction?.invalidate() // re-resolve the hidden/visible cursor for the state
         if iosInputDebug {
             iosInputLog.debug(
-                "pointer lock isLocked=\(locked, privacy: .public) captured=\(self.captured, privacy: .public)")
+                """
+                pointer lock isLocked=\(locked, privacy: .public) \
+                captured=\(self.captured, privacy: .public) \
+                relockPending=\(self.pointerRelockPending, privacy: .public) \
+                relockAttempt=\(self.pointerRelockAttempt, privacy: .public)
+                """)
+        }
+    }
+
+    /// Ask the system for the lock back after it dropped one we still want (see the Escape-drop
+    /// note on the state above). Bounded to a short burst; idempotent within it. Main queue.
+    private func requestPointerRelock() {
+        // Only a frontmost scene can hold the lock at all. Anywhere else the drop is the system
+        // saying we don't qualify, not the Esc key — re-asking would be noise, and the qualifying
+        // states (foreground, appearance, reparent) each re-resolve on their own already.
+        guard view.window?.windowScene?.activationState == .foregroundActive else {
+            pointerRelockPending = false
+            return
+        }
+        let now = CACurrentMediaTime()
+        // attempt == 0 is a fresh burst (first drop, or one the settle branch cleared); the window
+        // is the backstop for the pathological case where a grant is immediately revoked again and
+        // re-arms us. Even then this stays timer-driven at a few Hz — never a spin.
+        if pointerRelockAttempt == 0 || now - pointerRelockBurstStart > Self.pointerRelockBurstWindow {
+            pointerRelockBurstStart = now
+            pointerRelockAttempt = 0
+        }
+        guard pointerRelockAttempt < Self.pointerRelockAttemptLimit else {
+            // Out of budget: fall back to exactly today's behavior — the iPadOS cursor comes back
+            // and a click into the video re-captures. The caller invalidates the interaction, so
+            // the cursor can never stay hidden on a lock the system won't grant.
+            pointerRelockPending = false
+            return
+        }
+        pointerRelockAttempt += 1
+        pointerRelockPending = true
+        let escalate = pointerRelockAttempt > 1
+        // Deferred a turn so a ⌘⎋ whose GC keystroke lands after the system's unlock notification
+        // has already cleared `captured` — then the guard below drops this attempt instead of
+        // fighting the user's own release.
+        DispatchQueue.main.async { [weak self] in
+            guard let self, self.pointerRelockPending else { return }
+            guard self.wantsPointerLock, self.pointerLockEngaged() != true else {
+                // The grant landed, or the capture went away under us (⌘⎋ / ⌃⌥⇧Q / resign).
+                // Settle through the one decision point rather than returning with `pending` still
+                // set — that flag hides the cursor, so it must never outlive the burst.
+                self.syncPointerLock()
+                return
+            }
+            if escalate {
+                // Re-asserting a value the system already holds didn't take. Present a real
+                // false→true transition instead — the documented way to change your mind about the
+                // lock — and re-anchor the chain in case a reparent broke the downward walk to us.
+                // Held for a beat rather than cleared on the next turn: the system resolves the
+                // property asynchronously, and a same-turn flip back to true can be coalesced into
+                // no transition at all. We are already unlocked, so the false pass costs nothing.
+                self.pointerLockForcedOff = true
+                self.setNeedsUpdateOfPrefersPointerLocked()
+                self.updatePointerLockChain()
+                DispatchQueue.main.asyncAfter(deadline: .now() + Self.pointerLockForcedOffHold) {
+                    [weak self] in
+                    guard let self else { return }
+                    self.pointerLockForcedOff = false
+                    self.setNeedsUpdateOfPrefersPointerLocked()
+                }
+            } else {
+                self.setNeedsUpdateOfPrefersPointerLocked()
+            }
+            // A GRANT arrives as a didChange → syncPointerLock, which settles the burst and makes
+            // this retry a no-op. Routed back through syncPointerLock (not straight into another
+            // requestPointerRelock) so the give-up path re-resolves the cursor through the one
+            // place that does it.
+            DispatchQueue.main.asyncAfter(deadline: .now() + Self.pointerRelockRetryDelay) {
+                [weak self] in
+                guard let self, self.pointerRelockPending else { return }
+                self.syncPointerLock()
+            }
         }
     }
     #endif
@@ -724,7 +863,11 @@ extension StreamViewController: UIPointerInteractionDelegate {
         // host renders its own cursor from GCMouse deltas and a visible local one would just
         // diverge. When the lock isn't held the cursor stays VISIBLE so the user can aim; the
         // pointer is forwarded as an absolute position, both cursors tracking together.
-        captured && pointerLockEngaged() == true ? .hidden() : nil
+        // …except across an Esc-drop we're actively re-locking (`pointerRelockPending`): staying
+        // hidden for those couple of frames is what turns the fix into "Esc did nothing to my
+        // mouse" rather than a cursor that blinks in and out. The burst is bounded and clears
+        // itself on give-up, so the cursor can never stay hidden on a lock that isn't coming.
+        captured && (pointerLockEngaged() == true || pointerRelockPending) ? .hidden() : nil
     }
 }
 #endif