From c8aa3ed48b68caea77dbfd2c8c01e7129fc37865 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Thu, 23 Jul 2026 16:34:07 +0200 Subject: [PATCH] =?UTF-8?q?feat(apple/ios):=20pen=20P4=20=E2=80=94=20Apple?= =?UTF-8?q?=20Pencil=20(Pro)=20capture=20onto=20the=20stylus=20plane?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flagship leg of design/pen-tablet-input.md §7: against a HOST_CAP_PEN host the Pencil splits out of the finger path (independent of touch-input mode) into PencilStream — state-full samples with force→pressure, altitude→polar tilt, azimuth (+90° to the wire's north-clockwise convention), Pencil Pro rollAngle→ barrel roll (17.5+), coalesced touches batched ≤8/send for full 240Hz fidelity, and Pencil hover (zOffset>0 distinguishes it from trackpad hover) as in-range samples with distance. Squeeze holds barrel 1, double-tap clicks barrel 2 (the Pencil has no hardware buttons/eraser — these are how host apps get those affordances). Implements the ≤100ms heartbeat wire contract (80ms timer) so a stationary held stroke survives the host's 200ms dead-client failsafe. Toward a pen-less host nothing changes (Pencil stays a finger). Also fixes hostSupportsCursor testing the STALE 0x04 bit — HOST_CAP_CURSOR moved to 0x08 when TEXT_INPUT claimed 0x04; the old test would mistake a text-input host (e.g. Windows) for a cursor grant and double-render the pointer. Co-Authored-By: Claude Fable 5 --- .../Connection/PunktfunkConnection.swift | 23 +- .../PunktfunkKit/Views/PencilStream.swift | 268 ++++++++++++++++++ .../PunktfunkKit/Views/StreamViewIOS.swift | 55 +++- 3 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 clients/apple/Sources/PunktfunkKit/Views/PencilStream.swift diff --git a/clients/apple/Sources/PunktfunkKit/Connection/PunktfunkConnection.swift b/clients/apple/Sources/PunktfunkKit/Connection/PunktfunkConnection.swift index 94e0e74a..e0d7631e 100644 --- a/clients/apple/Sources/PunktfunkKit/Connection/PunktfunkConnection.swift +++ b/clients/apple/Sources/PunktfunkKit/Connection/PunktfunkConnection.swift @@ -387,8 +387,16 @@ public final class PunktfunkConnection { /// The host answered `HOST_CAP_CURSOR`: it stopped compositing the pointer and forwards /// shape/state on the cursor planes — the client MUST draw the cursor locally. + /// `0x08` — the bit moved when `HOST_CAP_TEXT_INPUT` claimed `0x04` on main; testing the + /// old bit would mistake a text-input-capable host (e.g. Windows) for a cursor grant. public var hostSupportsCursor: Bool { - hostCaps & 0x04 != 0 + hostCaps & 0x08 != 0 + } + + /// The host injects full-fidelity stylus input (`HOST_CAP_PEN`) — the gate for splitting + /// Apple Pencil out of the touch path onto the pen plane (``sendPen(_:)``). + public var hostSupportsPen: Bool { + hostCaps & UInt8(PUNKTFUNK_HOST_CAP_PEN) != 0 } /// One forwarded host-cursor shape (the cursor channel, ABI v11): straight-alpha RGBA, @@ -1133,6 +1141,19 @@ public final class PunktfunkConnection { _ = punktfunk_connection_send_input(h, &ev) } + /// Send one stylus sample batch (≤ `PUNKTFUNK_PEN_BATCH_MAX`, oldest first) on the pen + /// plane. Gate on ``hostSupportsPen`` — the core refuses toward a host without the cap. + /// Thread-safe; silently dropped after close (input is lossy by design). + public func sendPen(_ samples: [PunktfunkPenSample]) { + guard !samples.isEmpty else { return } + abiLock.lock() + defer { abiLock.unlock() } + guard let h = handle, !closeRequested else { return } + samples.withUnsafeBufferPointer { buf in + _ = punktfunk_connection_send_pen(h, buf.baseAddress, UInt32(buf.count)) + } + } + /// Signal a **deliberate** user-initiated quit before ``close()``: the connection closes with /// `QUIT_CLOSE_CODE` (81) so the host tears the session down immediately instead of holding the /// keep-alive linger for a reconnect. Call only from an explicit "Disconnect" action — NOT from a diff --git a/clients/apple/Sources/PunktfunkKit/Views/PencilStream.swift b/clients/apple/Sources/PunktfunkKit/Views/PencilStream.swift new file mode 100644 index 00000000..518cbc01 --- /dev/null +++ b/clients/apple/Sources/PunktfunkKit/Views/PencilStream.swift @@ -0,0 +1,268 @@ +// Apple Pencil → state-full wire pen samples (design/pen-tablet-input.md §7). +// +// Every sample carries the COMPLETE pen state (in-range/touching/buttons + all axes) — the +// host diffs consecutive samples and synthesizes down/up/button transitions itself, so a lost +// datagram self-heals and this file never sends edge events. Three sources feed one stream: +// UITouch contacts (with coalesced samples for full 240 Hz fidelity), the hover gesture +// (zOffset > 0 distinguishes a hovering Pencil from a trackpad pointer), and +// UIPencilInteraction (squeeze held → barrel 1, double-tap → a momentary barrel 2 — +// Apple Pencil has no hardware eraser end or barrel buttons, so these mappings are how +// host-side apps get their stylus button/eraser affordances). +// +// HEARTBEAT (wire contract — see `PunktfunkPenSample` in punktfunk_core.h): while the pen is +// in range or touching, the last sample repeats every ≤100 ms even when nothing changed. +// UIKit is silent for a stationary Pencil, and the host force-releases the stroke after +// 200 ms without samples (its dead-client failsafe) — the timer keeps a held stroke alive. + +#if os(iOS) +import PunktfunkCore +import UIKit + +final class PencilStream: NSObject, UIPencilInteractionDelegate { + enum Phase { case down, move, up, cancel } + + /// One assembled batch (≤ `PUNKTFUNK_PEN_BATCH_MAX` samples) ready for the connection. + var send: (([PunktfunkPenSample]) -> Void)? + /// View-space point → normalized [0,1] video coordinates (the letterbox mapping the + /// touch path already uses). nil until a mode is negotiated — samples are dropped then. + var videoNorm: ((CGPoint) -> (Float, Float)?)? + + private var inRange = false + private var touching = false + /// Squeeze held (mapped to wire BARREL1). + private var squeezeHeld = false + /// Whether this device/Pencil pair has demonstrated hover — decides what a lift means: + /// hover-capable hardware keeps proximity (the hover recognizer owns the exit), anything + /// else leaves range on lift so the host never parks a phantom hovering pen. + private var sawHover = false + /// A hover gesture is live right now (routes ended-state hover callbacks to us even when + /// the recognizer's final zOffset reads 0). + private(set) var hoverActive = false + private var last = PencilStream.idleSample() + private var heartbeat: Timer? + + // MARK: - Contact path (UITouch, `.pencil` only) + + func touches(_ touches: Set, event: UIEvent?, phase: Phase, in view: UIView) { + // At most one Pencil exists; a set with several is UIKit batching phases of the same + // stylus — the last one carries the freshest state. + guard let touch = touches.max(by: { $0.timestamp < $1.timestamp }) else { return } + switch phase { + case .down, .move: + touching = true + inRange = true + // Coalesced samples restore the Pencil's full capture rate (UIKit delivers at + // display cadence); oldest first, `dt_us` preserving their spacing. + let raw = event?.coalescedTouches(for: touch) ?? [touch] + var batch: [PunktfunkPenSample] = [] + var prevTs: TimeInterval? + for t in raw.suffix(Int(PUNKTFUNK_PEN_BATCH_MAX)) { + guard let s = contactSample(t, in: view, prevTs: prevTs) else { continue } + prevTs = t.timestamp + batch.append(s) + } + emit(batch) + case .up: + touching = false + // Hover-capable hardware: lift back to hover, the recognizer exits range later. + // Otherwise a lift IS the range exit (mirror of the host's GameStream heuristic). + inRange = sawHover + var s = last + s.pressure = 0 + s.state = stateBits() + if let posSample = contactSample(touch, in: view, prevTs: nil) { + s.x = posSample.x + s.y = posSample.y + } + emit([s]) + case .cancel: + release() + } + } + + // MARK: - Hover path (forwarded from the view's hover recognizer) + + /// Returns whether the event was consumed as Pencil hover; `false` hands it back to the + /// pointer path. A hovering Pencil reports `zOffset > 0`; trackpad/mouse hover is 0. + func maybeHover(_ r: UIHoverGestureRecognizer, in view: UIView) -> Bool { + switch r.state { + case .began, .changed: + guard r.zOffset > 0 || hoverActive else { return false } + hoverActive = true + sawHover = true + inRange = true + touching = false + guard let (x, y) = videoNorm?(r.location(in: view)) else { return true } + var s = PencilStream.idleSample() + s.state = stateBits() + s.x = x + s.y = y + s.distance = UInt16((r.zOffset.clamped(to: 0...1) * 65534).rounded()) + s.tilt_deg = Self.tiltDeg(altitude: r.altitudeAngle) + s.azimuth_deg = Self.azimuthDeg(r.azimuthAngle(in: view)) + if #available(iOS 17.5, *) { s.roll_deg = Self.rollDeg(r.rollAngle) } + emit([s]) + return true + case .ended, .cancelled, .failed: + guard hoverActive else { return false } + hoverActive = false + if !touching { release() } + return true + default: + return hoverActive + } + } + + // MARK: - UIPencilInteractionDelegate (squeeze → barrel 1 held, tap → barrel 2 click) + + @available(iOS 17.5, *) + func pencilInteraction( + _ interaction: UIPencilInteraction, didReceiveSqueeze squeeze: UIPencilInteraction.Squeeze + ) { + switch squeeze.phase { + case .began: + squeezeHeld = true + case .ended, .cancelled: + squeezeHeld = false + default: + return + } + guard inRange || touching else { return } + var s = last + s.state = stateBits() + emit([s]) + } + + func pencilInteractionDidTap(_ interaction: UIPencilInteraction) { + guard inRange || touching else { return } + // A momentary barrel-2 click: press + release as two state-full samples in ONE batch + // — the host's tracker emits the button press and release in order. + var press = last + press.state = stateBits() | UInt8(PUNKTFUNK_PEN_BARREL2) + var releaseS = last + releaseS.state = stateBits() + emit([press, releaseS]) + } + + // MARK: - Lifecycle + + /// Session stop / view teardown: leave range so the host lifts anything held. + func reset() { + if inRange || touching { release() } + sawHover = false + hoverActive = false + squeezeHeld = false + } + + private func release() { + touching = false + inRange = false + var s = last + s.pressure = 0 + s.state = 0 + emit([s]) + } + + // MARK: - Sample assembly + + private func contactSample( + _ t: UITouch, in view: UIView, prevTs: TimeInterval? + ) -> PunktfunkPenSample? { + guard let (x, y) = videoNorm?(t.location(in: view)) else { return nil } + var s = PencilStream.idleSample() + s.state = stateBits() + s.x = x + s.y = y + // maximumPossibleForce is 0 until the system knows the stylus — full force then + // (binary-stylus semantics, matching the host's unknown-pressure rule). + let maxForce = t.maximumPossibleForce + s.pressure = + maxForce > 0 + ? UInt16((Double(t.force / maxForce).clamped(to: 0...1) * 65535).rounded()) + : UInt16.max + s.distance = 0 + s.tilt_deg = Self.tiltDeg(altitude: t.altitudeAngle) + s.azimuth_deg = Self.azimuthDeg(t.azimuthAngle(in: view)) + if #available(iOS 17.5, *) { s.roll_deg = Self.rollDeg(t.rollAngle) } + if let prevTs { + s.dt_us = UInt16(((t.timestamp - prevTs) * 1_000_000).clamped(to: 0...65535)) + } + return s + } + + private func stateBits() -> UInt8 { + var bits: UInt8 = 0 + if inRange || touching { bits |= UInt8(PUNKTFUNK_PEN_IN_RANGE) } + if touching { bits |= UInt8(PUNKTFUNK_PEN_TOUCHING) } + if squeezeHeld { bits |= UInt8(PUNKTFUNK_PEN_BARREL1) } + return bits + } + + private func emit(_ batch: [PunktfunkPenSample]) { + guard !batch.isEmpty else { return } + last = batch[batch.count - 1] + last.dt_us = 0 + send?(batch) + armHeartbeat() + } + + /// The ≤100 ms keepalive while in range (see the file header). 80 ms leaves headroom + /// under the host's 200 ms failsafe even with one lost datagram. + private func armHeartbeat() { + heartbeat?.invalidate() + guard inRange || touching else { + heartbeat = nil + return + } + heartbeat = Timer.scheduledTimer(withTimeInterval: 0.08, repeats: true) { + [weak self] _ in + guard let self, self.inRange || self.touching else { + self?.heartbeat?.invalidate() + self?.heartbeat = nil + return + } + self.send?([self.last]) + } + } + + // MARK: - Angle conversions + + /// Altitude (π/2 = perpendicular) → wire tilt-from-normal in degrees, 0...90. + private static func tiltDeg(altitude: CGFloat) -> UInt8 { + UInt8((90 - altitude * 180 / .pi).rounded().clamped(to: 0...90)) + } + + /// Apple azimuth (0 along the view's +x axis, clockwise, y-down) → wire azimuth + /// (0 = north/up on screen, clockwise): +90° offset. + private static func azimuthDeg(_ apple: CGFloat) -> UInt16 { + let deg = (apple * 180 / .pi + 90).truncatingRemainder(dividingBy: 360) + return UInt16((deg + 360).truncatingRemainder(dividingBy: 360).rounded()) % 360 + } + + /// Pencil Pro roll (radians, −π...π) → wire barrel roll 0...359°. + private static func rollDeg(_ roll: CGFloat) -> UInt16 { + let deg = (roll * 180 / .pi).truncatingRemainder(dividingBy: 360) + return UInt16(((deg + 360).truncatingRemainder(dividingBy: 360)).rounded()) % 360 + } + + /// All-unknown baseline: sentinel angles/distance, tool = pen (the eraser is host-side + /// state driven by the squeeze/tap mappings, not a hardware end). + private static func idleSample() -> PunktfunkPenSample { + PunktfunkPenSample( + x: 0, y: 0, pressure: 0, + distance: UInt16(PUNKTFUNK_PEN_DISTANCE_UNKNOWN), + azimuth_deg: UInt16(PUNKTFUNK_PEN_ANGLE_UNKNOWN), + roll_deg: UInt16(PUNKTFUNK_PEN_ANGLE_UNKNOWN), + dt_us: 0, state: 0, + tool: UInt8(PUNKTFUNK_PEN_TOOL_PEN), + tilt_deg: UInt8(PUNKTFUNK_PEN_TILT_UNKNOWN), + _reserved: (0, 0, 0)) + } +} + +extension Comparable { + fileprivate func clamped(to range: ClosedRange) -> Self { + min(max(self, range.lowerBound), range.upperBound) + } +} +#endif diff --git a/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift b/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift index 6107d9e8..b132e25e 100644 --- a/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift +++ b/clients/apple/Sources/PunktfunkKit/Views/StreamViewIOS.swift @@ -333,6 +333,13 @@ public final class StreamViewController: StreamViewControllerBase { guard self?.captureEnabled == true else { return } connection?.send(event) } + // Apple Pencil → the stylus plane, only against a pen-capable host (elsewhere the + // Pencil stays a finger, exactly as before). Same trust gate as touch. + streamView.penEnabled = connection.hostSupportsPen + streamView.onPenBatch = { [weak self, weak connection] batch in + guard self?.captureEnabled == true else { return } + connection?.sendPen(batch) + } // Indirect pointer (mouse/trackpad) WITHOUT a lock → absolute cursor + buttons + scroll. // While the scene is pointer-LOCKED the GCMouse path owns motion AND buttons AND scroll, so // the whole UIKit indirect path is gated off here (`gcMouseForwarding`). The trackpad and a @@ -499,6 +506,8 @@ public final class StreamViewController: StreamViewControllerBase { // onTouchEvent can still deliver the button-up. streamView.resetTouchInput() streamView.onTouchEvent = nil + streamView.onPenBatch = nil // after reset — the pen's leave-range sample rides it + streamView.penEnabled = false streamView.onPointerMoveAbs = nil streamView.onPointerButton = nil streamView.onScroll = nil @@ -692,6 +701,12 @@ final class StreamLayerUIView: UIView { /// Direct fingers / Pencil → wire events: real touches in passthrough mode, or the /// touch-driven mouse events (`TouchMouse`) in the trackpad/pointer modes. var onTouchEvent: ((PunktfunkInputEvent) -> Void)? + /// Apple Pencil → state-full pen sample batches (the stylus plane). Active only while + /// `penEnabled`; without it the Pencil stays on the finger path exactly as before. + var onPenBatch: (([PunktfunkPenSample]) -> Void)? + /// The host advertised `HOST_CAP_PEN`, so Pencil input splits out of the finger path onto + /// the pen plane — independent of the touch-input mode (drawing must not depend on it). + var penEnabled = false /// Indirect pointer (mouse/trackpad with no lock) → absolute cursor moves. var onPointerMoveAbs: ((HostPoint) -> Void)? /// Indirect-pointer buttons (GameStream ids: 1=left 3=right); `down` = press. @@ -715,10 +730,21 @@ final class StreamLayerUIView: UIView { /// The finger route latched at gesture start — a Settings change mid-gesture applies to /// the NEXT touch, so one gesture never splits across input models. private var fingerRoute: TouchInputMode? + /// The Apple Pencil pipeline (contacts + hover + squeeze/tap → pen samples). + private lazy var pencil: PencilStream = { + let stream = PencilStream() + stream.send = { [weak self] batch in self?.onPenBatch?(batch) } + stream.videoNorm = { [weak self] point in + guard let h = self?.hostPoint(from: point) else { return nil } + return (Float(h.x) / Float(max(h.w - 1, 1)), Float(h.y) / Float(max(h.h - 1, 1))) + } + return stream + }() /// Release anything the touch-driven mouse holds and forget gesture state — session stop. func resetTouchInput() { touchMouse.reset() + pencil.reset() // leaves range → the host lifts anything still inked fingerRoute = nil setSoftKeyboardVisible(false) // a stream that's gone takes its keyboard with it } @@ -755,6 +781,11 @@ final class StreamLayerUIView: UIView { scrollPan.allowedScrollTypesMask = .all scrollPan.allowedTouchTypes = [] addGestureRecognizer(scrollPan) + // Pencil squeeze / double-tap → the pen plane's barrel buttons (no-op while + // `penEnabled` is false — PencilStream ignores interactions out of range). + let pencilInteraction = UIPencilInteraction() + pencilInteraction.delegate = pencil + addInteraction(pencilInteraction) #endif backgroundColor = .black } @@ -779,17 +810,32 @@ final class StreamLayerUIView: UIView { private enum TouchKind { case down, move, up, cancel } /// Split a touch batch by kind: an INDIRECT POINTER (mouse/trackpad with no lock) drives - /// the host cursor as an absolute mouse; everything else (direct finger, Pencil) is a host - /// touch. Mixed batches are possible, so partition rather than branch on the first touch. + /// the host cursor as an absolute mouse; a Pencil goes to the pen plane when the host + /// supports it; everything else (direct finger — and the Pencil toward a pen-less host) + /// is a host touch. Mixed batches are possible, so partition rather than branch on the + /// first touch. private func route(_ touches: Set, event: UIEvent?, kind: TouchKind) { var fingers: Set = [] + var pencilTouches: Set = [] for touch in touches { if touch.type == .indirectPointer { handleIndirectPointer(touch, event: event, kind: kind) + } else if penEnabled, touch.type == .pencil { + pencilTouches.insert(touch) } else { fingers.insert(touch) } } + if !pencilTouches.isEmpty { + let phase: PencilStream.Phase = + switch kind { + case .down: .down + case .move: .move + case .up: .up + case .cancel: .cancel + } + pencil.touches(pencilTouches, event: event, phase: phase, in: self) + } if !fingers.isEmpty { forwardFingers(fingers, kind: kind) } } @@ -861,8 +907,11 @@ final class StreamLayerUIView: UIView { } } - /// Button-less mouse/trackpad movement (no lock) → absolute cursor move. + /// Button-less mouse/trackpad movement (no lock) → absolute cursor move — unless it is a + /// hovering PENCIL (`zOffset > 0`) on a pen-capable host, which becomes in-range pen + /// samples (hover preview with distance/tilt/azimuth) instead of a cursor move. @objc private func handleHover(_ recognizer: UIHoverGestureRecognizer) { + if penEnabled, pencil.maybeHover(recognizer, in: self) { return } switch recognizer.state { case .began, .changed: if let h = hostPoint(from: recognizer.location(in: self)) { onPointerMoveAbs?(h) }