feat(apple): a Pencil near the glass pins the panel at the link ceiling

UIKit delivers Pencil events at the PANEL's cadence, and the deadline link's rate hint pins the panel to the stream rate (min = preferred = stream Hz, deliberately — the anti-idle-stall floor). Net effect: a 60 fps stream on a 120 Hz iPad silently halved pencil sampling, 8.3 → 16.7 ms per event batch — exactly the workload (drawing) that feels it most. FrameRateHint gains a boost that lifts min/preferred to the range ceiling while a Pencil is in range (hover or contact): PencilStream surfaces proximity transitions from its emit() choke point, the view controller debounces release by 2 s (edge-of-canvas hover flicker must not thrash the link), and Stage2Pipeline/SessionPresenter stage it like the rate hint — applied from the link's own thread, no-op under arrival/glass pacing. Presents still pace at stream rate (extra updates vend into the newest-wins stash), so the cost is empty link wakes scoped to proximity. pf.present logs engage/release for on-glass verification. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-31 17:41:46 +02:00
parent 7cf71dd218
commit 957cf7781c
4 changed files with 79 additions and 1 deletions
@@ -355,6 +355,13 @@ final class SessionPresenter {
    /// behavior — iOS keeps a 30 Hz floor; macOS leaves the NSView link at its display's native
    /// rate (it already tracks the display and must NOT be capped to the stream rate).
    /// Re-applied from `layout` so a mid-session `Reconfigure` picks up a new refresh.
+    /// Pen-proximity panel-rate boost pass-through (Stage2Pipeline.setInteractionBoost):
+    /// deadline pacing only — under arrival/glass the staged hint feeds no link, so this
+    /// is a no-op there. MAIN thread.
+    func setInteractionBoost(_ on: Bool) {
+        stage2?.setInteractionBoost(on)
+    }
+
    private func syncFrameRate(hz: UInt32) {
        guard hz > 0 else { return }
        // Deadline pacing: the hint goes to the pipeline's CAMetalDisplayLink instead (staged;
@@ -279,10 +279,27 @@ final class LatestBox<T>: @unchecked Sendable {
 private final class FrameRateHint: @unchecked Sendable {
    private let lock = NSLock()
    private var pending: CAFrameRateRange?
+    private var streamHz: Float = 0
+    private var boosted = false
    func stage(hz: Float) {
        guard hz > 0 else { return }
        lock.lock()
-        pending = CAFrameRateRange(minimum: hz, maximum: max(hz, 120), preferred: hz)
+        streamHz = hz
+        pending = Self.range(hz: hz, boosted: boosted)
+        lock.unlock()
+    }
+    /// Pen-proximity boost: pin `minimum = preferred` at the range's CEILING instead of the
+    /// stream rate. UIKit delivers touch/Pencil events at the PANEL's cadence, and the panel
+    /// follows this link's vote — so a 60 fps stream on a 120 Hz iPad halves pencil sampling
+    /// unless a boost lifts the panel while the Pencil is in range. Presents still pace at
+    /// stream rate (extra link updates just vend into the newest-wins stash), so the cost is
+    /// empty wakes, scoped to pen proximity.
+    func setBoost(_ on: Bool) {
+        lock.lock()
+        if boosted != on {
+            boosted = on
+            if streamHz > 0 { pending = Self.range(hz: streamHz, boosted: on) }
+        }
        lock.unlock()
    }
    func drain() -> CAFrameRateRange? {
@@ -292,6 +309,11 @@ private final class FrameRateHint: @unchecked Sendable {
        pending = nil
        return p
    }
+    private static func range(hz: Float, boosted: Bool) -> CAFrameRateRange {
+        let cap = max(hz, 120)
+        let preferred = boosted ? cap : hz
+        return CAFrameRateRange(minimum: preferred, maximum: cap, preferred: preferred)
+    }
 }

 /// The client half of phase-locked capture (design/phase-locked-capture.md): the decode
@@ -1230,6 +1252,15 @@ public final class Stage2Pipeline {
        frameRateHint.stage(hz: hz)
    }

+    /// Pen-proximity rate boost (drawing workloads): drive the deadline link — and with it the
+    /// panel, whose cadence paces UIKit's touch/Pencil event delivery — at the range ceiling
+    /// while a Pencil is in range, so a sub-panel-rate stream stops halving pencil sampling.
+    /// Staged like the rate hint; no-op under arrival/glass pacing. MAIN thread.
+    public func setInteractionBoost(_ on: Bool) {
+        frameRateHint.setBoost(on)
+        presentLog.info("pen boost \(on ? "engaged" : "released", privacy: .public)")
+    }
+
    /// Forward the layout-derived drawable pixel size to the presenter (MAIN thread — see
    /// `MetalVideoPresenter.setDrawableTarget`).
    public func setDrawableTarget(_ size: CGSize) {
@@ -23,6 +23,10 @@ final class PencilStream: NSObject, UIPencilInteractionDelegate {

    /// One assembled batch (≤ `PUNKTFUNK_PEN_BATCH_MAX` samples) ready for the connection.
    var send: (([PunktfunkPenSample]) -> Void)?
+    /// Proximity transitions (in-range ∪ touching) — drives the panel-rate boost while the
+    /// Pencil is near the glass. Fired from emit(), the choke point every state change exits
+    /// through.
+    var onProximity: ((Bool) -> Void)?
    /// View-space point → normalized [0,1] video coordinates (the letterbox mapping the
    /// touch path already uses). nil until a mode is negotiated — samples are dropped then.
    var videoNorm: ((CGPoint) -> (Float, Float)?)?
@@ -42,6 +46,8 @@ final class PencilStream: NSObject, UIPencilInteractionDelegate {
    private var heartbeat: Timer?
    /// When the last batch (event or keepalive) went out — the heartbeat tick's idle test.
    private var lastSendTs: TimeInterval = 0
+    /// The last proximity state surfaced through `onProximity` (transition detection).
+    private var lastProximity = false

    // MARK: - Contact path (UITouch, `.pencil` only)

@@ -217,6 +223,11 @@ final class PencilStream: NSObject, UIPencilInteractionDelegate {
        }
        lastSendTs = CACurrentMediaTime()
        syncHeartbeat()
+        let near = inRange || touching
+        if near != lastProximity {
+            lastProximity = near
+            onProximity?(near)
+        }
    }

    /// The ≤100 ms keepalive while in range (see the file header): ONE long-lived 50 ms timer
@@ -154,6 +154,9 @@ public final class StreamViewController: StreamViewControllerBase {
    /// The shared presenter stack: stage-2 (CAMetalLayer sublayer + display link) with the
    /// stage-1 StreamPump → displayLayer path as the Metal-unavailable / DEBUG fallback.
    private let presenter = SessionPresenter()
+    /// Pending pen-boost release — 2 s of hysteresis so hover flicker at the glass edge
+    /// doesn't thrash the deadline link's rate range (see `setInteractionBoost`).
+    private var penBoostRelease: DispatchWorkItem?
    #if os(tvOS)
    /// The window's display manager the session's mode request was set on — held weakly so
    /// stop() can clear the request even after the view has left the window.
@@ -351,6 +354,26 @@ public final class StreamViewController: StreamViewControllerBase {
            guard self?.captureEnabled == true else { return }
            connection?.sendPen(batch)
        }
+        // Pencil near the glass ⇒ pin the panel (and with it UIKit's event cadence) at the
+        // link range's ceiling, so a sub-panel-rate stream stops halving pencil sampling.
+        // Engage is immediate; release waits 2 s so edge-of-canvas hover flicker can't
+        // thrash the link. MAIN thread (UIKit events + main-queue work item).
+        streamView.onPenProximity = { [weak self] near in
+            guard let self else { return }
+            self.penBoostRelease?.cancel()
+            self.penBoostRelease = nil
+            if near {
+                self.presenter.setInteractionBoost(true)
+            } else {
+                let release = DispatchWorkItem { [weak self] in
+                    guard let self else { return }
+                    self.penBoostRelease = nil
+                    self.presenter.setInteractionBoost(false)
+                }
+                self.penBoostRelease = release
+                DispatchQueue.main.asyncAfter(deadline: .now() + 2, execute: release)
+            }
+        }
        // Indirect pointer (mouse/trackpad) WITHOUT a lock → absolute cursor + buttons + scroll.
        // While the scene is pointer-LOCKED the GCMouse path owns motion AND buttons AND scroll, so
        // the whole UIKit indirect path is gated off here (`gcMouseForwarding`). The trackpad and a
@@ -523,6 +546,9 @@ public final class StreamViewController: StreamViewControllerBase {
        streamView.resetTouchInput()
        streamView.onTouchEvent = nil
        streamView.onPenBatch = nil // after reset — the pen's leave-range sample rides it
+        streamView.onPenProximity = nil // after reset — its leave-range transition fired above
+        penBoostRelease?.cancel()
+        penBoostRelease = nil
        streamView.penEnabled = false
        streamView.onPointerMoveAbs = nil
        streamView.onPointerButton = nil
@@ -723,6 +749,8 @@ final class StreamLayerUIView: UIView {
    /// The host advertised `HOST_CAP_PEN`, so Pencil input splits out of the finger path onto
    /// the pen plane — independent of the touch-input mode (drawing must not depend on it).
    var penEnabled = false
+    /// Pencil proximity transitions (hover or contact) — the presenter's panel-rate boost.
+    var onPenProximity: ((Bool) -> Void)?
    /// Indirect pointer (mouse/trackpad with no lock) → absolute cursor moves.
    var onPointerMoveAbs: ((HostPoint) -> Void)?
    /// Indirect-pointer buttons (GameStream ids: 1=left 3=right); `down` = press.
@@ -750,6 +778,7 @@ final class StreamLayerUIView: UIView {
    private lazy var pencil: PencilStream = {
        let stream = PencilStream()
        stream.send = { [weak self] batch in self?.onPenBatch?(batch) }
+        stream.onProximity = { [weak self] near in self?.onPenProximity?(near) }
        stream.videoNorm = { [weak self] point in
            guard let h = self?.hostPoint(from: point) else { return nil }
            return (Float(h.x) / Float(max(h.w - 1, 1)), Float(h.y) / Float(max(h.h - 1, 1)))