Files
punktfunk/clients/apple/Sources/PunktfunkKit/Views/PencilStream.swift
T
enricobuehlerandClaude Fable 5 957cf7781c
ci / web (push) Successful in 2m4s
ci / docs-site (push) Successful in 2m19s
docker / builders (ci/android-ci.Dockerfile, punktfunk-android-ci) (push) Successful in 11s
docker / builders (ci/arch-ci.Dockerfile, punktfunk-arch-ci) (push) Successful in 9s
docker / builders (ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 9s
docker / builders (ci/rust-ci-noble.Dockerfile, punktfunk-rust-ci-noble) (push) Successful in 9s
ci / rust-arm64 (push) Successful in 3m42s
docker / builders (ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 11s
deb / build-publish-client-arm64 (push) Successful in 2m26s
docker / apps (., web/Dockerfile, punktfunk-web) (push) Successful in 1m16s
docker / apps (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 1m11s
ci / rust (push) Failing after 2m7s
apple / swift (push) Successful in 4m59s
deb / build-publish (push) Successful in 6m51s
docker / builders (--build-arg FEDORA_VERSION=44, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm, -f44) (push) Successful in 8s
deb / build-publish-host (push) Successful in 7m10s
docker / builders-arm64cross (push) Successful in 8s
android / android (push) Successful in 8m44s
arch / build-publish (push) Successful in 8m35s
docker / deploy-docs (push) Successful in 33s
flatpak / build-publish (push) Successful in 8m0s
windows-host / package (push) Successful in 18m16s
windows-host / winget-source (push) Skipped
windows-host / canary-manifest (push) Successful in 21s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 3m27s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 17m29s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 18m10s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 3m27s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 3m5s
windows / build (x86_64-pc-windows-msvc) (push) Successful in 4m1s
release / apple (push) Successful in 25m56s
apple / screenshots (push) Canceled after 4s
feat(apple): a Pencil near the glass pins the panel at the link ceiling
UIKit delivers Pencil events at the PANEL's cadence, and the deadline
link's rate hint pins the panel to the stream rate (min = preferred =
stream Hz, deliberately — the anti-idle-stall floor). Net effect: a 60 fps
stream on a 120 Hz iPad silently halved pencil sampling, 8.3 → 16.7 ms per
event batch — exactly the workload (drawing) that feels it most.

FrameRateHint gains a boost that lifts min/preferred to the range ceiling
while a Pencil is in range (hover or contact): PencilStream surfaces
proximity transitions from its emit() choke point, the view controller
debounces release by 2 s (edge-of-canvas hover flicker must not thrash the
link), and Stage2Pipeline/SessionPresenter stage it like the rate hint —
applied from the link's own thread, no-op under arrival/glass pacing.
Presents still pace at stream rate (extra updates vend into the
newest-wins stash), so the cost is empty link wakes scoped to proximity.
pf.present logs engage/release for on-glass verification.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-31 17:41:46 +02:00

302 lines
13 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Apple Pencil → state-full wire pen samples (design/pen-tablet-input.md §7).
//
// Every sample carries the COMPLETE pen state (in-range/touching/buttons + all axes) — the
// host diffs consecutive samples and synthesizes down/up/button transitions itself, so a lost
// datagram self-heals and this file never sends edge events. Three sources feed one stream:
// UITouch contacts (with coalesced samples for full 240 Hz fidelity), the hover gesture
// (zOffset > 0 distinguishes a hovering Pencil from a trackpad pointer), and
// UIPencilInteraction (squeeze held → barrel 1, double-tap → a momentary barrel 2 —
// Apple Pencil has no hardware eraser end or barrel buttons, so these mappings are how
// host-side apps get their stylus button/eraser affordances).
//
// HEARTBEAT (wire contract — see `PunktfunkPenSample` in punktfunk_core.h): while the pen is
// in range or touching, the last sample repeats every ≤100 ms even when nothing changed.
// UIKit is silent for a stationary Pencil, and the host force-releases the stroke after
// 200 ms without samples (its dead-client failsafe) — the timer keeps a held stroke alive.
#if os(iOS)
import PunktfunkCore
import UIKit
final class PencilStream: NSObject, UIPencilInteractionDelegate {
enum Phase { case down, move, up, cancel }
/// One assembled batch (≤ `PUNKTFUNK_PEN_BATCH_MAX` samples) ready for the connection.
var send: (([PunktfunkPenSample]) -> Void)?
/// Proximity transitions (in-range touching) — drives the panel-rate boost while the
/// Pencil is near the glass. Fired from emit(), the choke point every state change exits
/// through.
var onProximity: ((Bool) -> Void)?
/// View-space point → normalized [0,1] video coordinates (the letterbox mapping the
/// touch path already uses). nil until a mode is negotiated — samples are dropped then.
var videoNorm: ((CGPoint) -> (Float, Float)?)?
private var inRange = false
private var touching = false
/// Squeeze held (mapped to wire BARREL1).
private var squeezeHeld = false
/// Whether this device/Pencil pair has demonstrated hover — decides what a lift means:
/// hover-capable hardware keeps proximity (the hover recognizer owns the exit), anything
/// else leaves range on lift so the host never parks a phantom hovering pen.
private var sawHover = false
/// A hover gesture is live right now (routes ended-state hover callbacks to us even when
/// the recognizer's final zOffset reads 0).
private(set) var hoverActive = false
private var last = PencilStream.idleSample()
private var heartbeat: Timer?
/// When the last batch (event or keepalive) went out — the heartbeat tick's idle test.
private var lastSendTs: TimeInterval = 0
/// The last proximity state surfaced through `onProximity` (transition detection).
private var lastProximity = false
// MARK: - Contact path (UITouch, `.pencil` only)
func touches(_ touches: Set<UITouch>, event: UIEvent?, phase: Phase, in view: UIView) {
// At most one Pencil exists; a set with several is UIKit batching phases of the same
// stylus — the last one carries the freshest state.
guard let touch = touches.max(by: { $0.timestamp < $1.timestamp }) else { return }
switch phase {
case .down, .move:
touching = true
inRange = true
// Coalesced samples restore the Pencil's full capture rate (UIKit delivers at
// display cadence); oldest first, `dt_us` preserving their spacing. The whole run
// is kept — emit() splits anything over the wire's batch cap.
let raw = event?.coalescedTouches(for: touch) ?? [touch]
var batch: [PunktfunkPenSample] = []
var prevTs: TimeInterval?
for t in raw {
guard let s = contactSample(t, in: view, prevTs: prevTs) else { continue }
prevTs = t.timestamp
batch.append(s)
}
emit(batch)
case .up:
touching = false
// Hover-capable hardware: lift back to hover, the recognizer exits range later.
// Otherwise a lift IS the range exit (mirror of the host's GameStream heuristic).
inRange = sawHover
var s = last
s.pressure = 0
s.state = stateBits()
if let posSample = contactSample(touch, in: view, prevTs: nil) {
s.x = posSample.x
s.y = posSample.y
}
emit([s])
case .cancel:
release()
}
}
// MARK: - Hover path (forwarded from the view's hover recognizer)
/// Returns whether the event was consumed as Pencil hover; `false` hands it back to the
/// pointer path. A hovering Pencil reports `zOffset > 0`; trackpad/mouse hover is 0.
func maybeHover(_ r: UIHoverGestureRecognizer, in view: UIView) -> Bool {
switch r.state {
case .began, .changed:
guard r.zOffset > 0 || hoverActive else { return false }
hoverActive = true
sawHover = true
inRange = true
touching = false
guard let (x, y) = videoNorm?(r.location(in: view)) else { return true }
var s = PencilStream.idleSample()
s.state = stateBits()
s.x = x
s.y = y
s.distance = UInt16((r.zOffset.clamped(to: 0...1) * 65534).rounded())
s.tilt_deg = Self.tiltDeg(altitude: r.altitudeAngle)
s.azimuth_deg = Self.azimuthDeg(r.azimuthAngle(in: view))
if #available(iOS 17.5, *) { s.roll_deg = Self.rollDeg(r.rollAngle) }
emit([s])
return true
case .ended, .cancelled, .failed:
guard hoverActive else { return false }
hoverActive = false
if !touching { release() }
return true
default:
return hoverActive
}
}
// MARK: - UIPencilInteractionDelegate (squeeze → barrel 1 held, tap → barrel 2 click)
@available(iOS 17.5, *)
func pencilInteraction(
_ interaction: UIPencilInteraction, didReceiveSqueeze squeeze: UIPencilInteraction.Squeeze
) {
switch squeeze.phase {
case .began:
squeezeHeld = true
case .ended, .cancelled:
squeezeHeld = false
default:
return
}
guard inRange || touching else { return }
var s = last
s.state = stateBits()
emit([s])
}
func pencilInteractionDidTap(_ interaction: UIPencilInteraction) {
guard inRange || touching else { return }
// A momentary barrel-2 click: press + release as two state-full samples in ONE batch
// — the host's tracker emits the button press and release in order.
var press = last
press.state = stateBits() | UInt8(PUNKTFUNK_PEN_BARREL2)
var releaseS = last
releaseS.state = stateBits()
emit([press, releaseS])
}
// MARK: - Lifecycle
/// Session stop / view teardown: leave range so the host lifts anything held.
func reset() {
if inRange || touching { release() }
sawHover = false
hoverActive = false
squeezeHeld = false
}
private func release() {
touching = false
inRange = false
var s = last
s.pressure = 0
s.state = 0
emit([s])
}
// MARK: - Sample assembly
private func contactSample(
_ t: UITouch, in view: UIView, prevTs: TimeInterval?
) -> PunktfunkPenSample? {
guard let (x, y) = videoNorm?(t.location(in: view)) else { return nil }
var s = PencilStream.idleSample()
s.state = stateBits()
s.x = x
s.y = y
// maximumPossibleForce is 0 until the system knows the stylus — full force then
// (binary-stylus semantics, matching the host's unknown-pressure rule).
let maxForce = t.maximumPossibleForce
s.pressure =
maxForce > 0
? UInt16((Double(t.force / maxForce).clamped(to: 0...1) * 65535).rounded())
: UInt16.max
s.distance = 0
s.tilt_deg = Self.tiltDeg(altitude: t.altitudeAngle)
s.azimuth_deg = Self.azimuthDeg(t.azimuthAngle(in: view))
if #available(iOS 17.5, *) { s.roll_deg = Self.rollDeg(t.rollAngle) }
if let prevTs {
s.dt_us = UInt16(((t.timestamp - prevTs) * 1_000_000).clamped(to: 0...65535))
}
return s
}
private func stateBits() -> UInt8 {
var bits: UInt8 = 0
if inRange || touching { bits |= UInt8(PUNKTFUNK_PEN_IN_RANGE) }
if touching { bits |= UInt8(PUNKTFUNK_PEN_TOUCHING) }
if squeezeHeld { bits |= UInt8(PUNKTFUNK_PEN_BARREL1) }
return bits
}
private func emit(_ batch: [PunktfunkPenSample]) {
guard !batch.isEmpty else { return }
last = batch[batch.count - 1]
last.dt_us = 0
// A run longer than the wire's batch cap is SPLIT into consecutive sends, never
// truncated (the send_pen contract): an over-cap run means the main thread hitched
// past a frame of 240 Hz samples, and dropping its head would notch exactly the
// stroke geometry a drawing app can least afford to lose.
var start = 0
while start < batch.count {
let end = min(start + Int(PUNKTFUNK_PEN_BATCH_MAX), batch.count)
send?(Array(batch[start..<end]))
start = end
}
lastSendTs = CACurrentMediaTime()
syncHeartbeat()
let near = inRange || touching
if near != lastProximity {
lastProximity = near
onProximity?(near)
}
}
/// The ≤100 ms keepalive while in range (see the file header): ONE long-lived 50 ms timer
/// in `.common` run-loop mode, armed on range entry and torn down on exit. `.default`-mode
/// scheduling pauses during run-loop tracking, where a stationary pen would silently cross
/// the host's 200 ms force-release mid-stroke; the old per-emit re-arm also churned the run
/// loop once per event during a stroke. The tick resends only after ≥50 ms of send silence,
/// so an active stroke costs nothing and the worst-case gap stays ≈100 ms — the wire bound.
private func syncHeartbeat() {
guard inRange || touching else {
heartbeat?.invalidate()
heartbeat = nil
return
}
guard heartbeat == nil else { return }
let timer = Timer(timeInterval: 0.05, repeats: true) { [weak self] t in
guard let self, self.inRange || self.touching else {
t.invalidate()
self?.heartbeat = nil
return
}
if CACurrentMediaTime() - self.lastSendTs >= 0.05 {
self.send?([self.last])
self.lastSendTs = CACurrentMediaTime()
}
}
heartbeat = timer
RunLoop.main.add(timer, forMode: .common)
}
// MARK: - Angle conversions
/// Altitude (π/2 = perpendicular) → wire tilt-from-normal in degrees, 0...90.
private static func tiltDeg(altitude: CGFloat) -> UInt8 {
UInt8((90 - altitude * 180 / .pi).rounded().clamped(to: 0...90))
}
/// Apple azimuth (0 along the view's +x axis, clockwise, y-down) → wire azimuth
/// (0 = north/up on screen, clockwise): +90° offset.
private static func azimuthDeg(_ apple: CGFloat) -> UInt16 {
let deg = (apple * 180 / .pi + 90).truncatingRemainder(dividingBy: 360)
return UInt16((deg + 360).truncatingRemainder(dividingBy: 360).rounded()) % 360
}
/// Pencil Pro roll (radians, −π...π) → wire barrel roll 0...359°.
private static func rollDeg(_ roll: CGFloat) -> UInt16 {
let deg = (roll * 180 / .pi).truncatingRemainder(dividingBy: 360)
return UInt16(((deg + 360).truncatingRemainder(dividingBy: 360)).rounded()) % 360
}
/// All-unknown baseline: sentinel angles/distance, tool = pen (the eraser is host-side
/// state driven by the squeeze/tap mappings, not a hardware end).
private static func idleSample() -> PunktfunkPenSample {
PunktfunkPenSample(
x: 0, y: 0, pressure: 0,
distance: UInt16(PUNKTFUNK_PEN_DISTANCE_UNKNOWN),
azimuth_deg: UInt16(PUNKTFUNK_PEN_ANGLE_UNKNOWN),
roll_deg: UInt16(PUNKTFUNK_PEN_ANGLE_UNKNOWN),
dt_us: 0, state: 0,
tool: UInt8(PUNKTFUNK_PEN_TOOL_PEN),
tilt_deg: UInt8(PUNKTFUNK_PEN_TILT_UNKNOWN),
_reserved: (0, 0, 0))
}
}
extension Comparable {
fileprivate func clamped(to range: ClosedRange<Self>) -> Self {
min(max(self, range.lowerBound), range.upperBound)
}
}
#endif