From 4714235fe6edb228bab8acc1ffefc3cb56e34bf3 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Thu, 23 Jul 2026 00:27:04 +0200 Subject: [PATCH] =?UTF-8?q?feat(core):=20stylus=20wire=20P0=20=E2=80=94=20?= =?UTF-8?q?state-full=20RICH=5FPEN=20batches,=20PenTracker,=20HOST=5FCAP?= =?UTF-8?q?=5FPEN?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pen plane from design/pen-tablet-input.md, protocol side only (the P1 uinput tablet injector will consume it): 0xCC kind 0x05 carries batches of state-full PenSamples (pressure, polar tilt + azimuth, barrel roll, hover distance, eraser tool, barrel buttons); PenTracker diffs samples into injector transitions so a lost datagram self-heals; HOST_CAP_PEN (0x10) gates sending and stays unadvertised until a real backend exists. C ABI: PunktfunkPenSample + punktfunk_connection_send_pen, documented in docs/embedding-the-c-abi.md §8. Co-Authored-By: Claude Fable 5 --- crates/punktfunk-core/src/abi.rs | 155 ++++- crates/punktfunk-core/src/client/mod.rs | 44 +- crates/punktfunk-core/src/client/pump.rs | 7 +- crates/punktfunk-core/src/client/worker.rs | 5 +- crates/punktfunk-core/src/quic/caps.rs | 12 + crates/punktfunk-core/src/quic/datagram.rs | 9 +- crates/punktfunk-core/src/quic/mod.rs | 3 + crates/punktfunk-core/src/quic/pen.rs | 689 +++++++++++++++++++++ docs/embedding-the-c-abi.md | 32 + include/punktfunk_core.h | 177 +++++- 10 files changed, 1116 insertions(+), 17 deletions(-) create mode 100644 crates/punktfunk-core/src/quic/pen.rs diff --git a/crates/punktfunk-core/src/abi.rs b/crates/punktfunk-core/src/abi.rs index e6833263..9e15751e 100644 --- a/crates/punktfunk-core/src/abi.rs +++ b/crates/punktfunk-core/src/abi.rs @@ -870,6 +870,94 @@ impl PunktfunkRichInputEx { } } +/// [`PunktfunkPenSample::state`] bit: the pen hovers in range (implied by `TOUCHING`). +pub const PUNKTFUNK_PEN_IN_RANGE: u8 = 0x01; +/// [`PunktfunkPenSample::state`] bit: the tip is in contact. +pub const PUNKTFUNK_PEN_TOUCHING: u8 = 0x02; +/// [`PunktfunkPenSample::state`] bit: primary barrel button (or squeeze mapping) held. +pub const PUNKTFUNK_PEN_BARREL1: u8 = 0x04; +/// [`PunktfunkPenSample::state`] bit: secondary barrel button (or double-tap mapping) held. +pub const PUNKTFUNK_PEN_BARREL2: u8 = 0x08; +/// [`PunktfunkPenSample::tool`]: the pen tip. +pub const PUNKTFUNK_PEN_TOOL_PEN: u8 = 0; +/// [`PunktfunkPenSample::tool`]: the eraser (a client-side mode — Apple Pencil has no +/// hardware eraser end; the squeeze/double-tap mapping usually drives this). +pub const PUNKTFUNK_PEN_TOOL_ERASER: u8 = 1; +/// Most samples one [`punktfunk_connection_send_pen`] call accepts (one wire batch). +pub const PUNKTFUNK_PEN_BATCH_MAX: u32 = 8; +/// [`PunktfunkPenSample::tilt_deg`] sentinel: no tilt reading. +pub const PUNKTFUNK_PEN_TILT_UNKNOWN: u8 = 0xFF; +/// [`PunktfunkPenSample::azimuth_deg`] / `roll_deg` sentinel: no reading. +pub const PUNKTFUNK_PEN_ANGLE_UNKNOWN: u16 = 0xFFFF; +/// [`PunktfunkPenSample::distance`] sentinel: no hover-distance reading. +pub const PUNKTFUNK_PEN_DISTANCE_UNKNOWN: u16 = 0xFFFF; + +/// One complete stylus state at one instant ([`punktfunk_connection_send_pen`]; +/// design/pen-tablet-input.md). STATE-FULL, never an edge event: fill every field on every +/// sample (unknown axes take their `*_UNKNOWN` sentinel) — the host diffs consecutive samples +/// and synthesizes down/up/button transitions itself, which is what makes a lost datagram +/// self-heal. `x`/`y` are normalized `0.0..=1.0` in VIDEO-FRAME space (map your letterbox +/// before filling, exactly like wire touches). +#[cfg(feature = "quic")] +#[repr(C)] +#[derive(Clone, Copy)] +pub struct PunktfunkPenSample { + /// Normalized `0.0..=1.0` across the video frame. Must be finite. + pub x: f32, + /// Normalized `0.0..=1.0` across the video frame. Must be finite. + pub y: f32, + /// Tip force, `0..=65535` full scale (`0` while hovering). + pub pressure: u16, + /// Hover distance `0..=65534` (0 = at the hover floor), or `PUNKTFUNK_PEN_DISTANCE_UNKNOWN`. + pub distance: u16, + /// Tilt azimuth, degrees `0..=359` clockwise from north, or `PUNKTFUNK_PEN_ANGLE_UNKNOWN`. + pub azimuth_deg: u16, + /// Barrel roll (Apple Pencil Pro `rollAngle`), degrees `0..=359`, or + /// `PUNKTFUNK_PEN_ANGLE_UNKNOWN`. + pub roll_deg: u16, + /// µs since the previous sample in the same call (`0` for the first) — the coalesced + /// capture spacing. + pub dt_us: u16, + /// Bitfield of `PUNKTFUNK_PEN_*` state bits. Unknown bits are rejected (`InvalidArg`). + pub state: u8, + /// `PUNKTFUNK_PEN_TOOL_PEN` or `PUNKTFUNK_PEN_TOOL_ERASER`. + pub tool: u8, + /// Tilt from the surface normal, degrees `0..=90`, or `PUNKTFUNK_PEN_TILT_UNKNOWN`. + pub tilt_deg: u8, + /// Set to 0. + pub _reserved: [u8; 3], +} + +#[cfg(feature = "quic")] +impl PunktfunkPenSample { + /// `None` = invalid field (non-finite coordinate, unknown state bit, unknown tool) — + /// embedder input is validated strictly, unlike the loss-tolerant wire decode. + fn to_sample(self) -> Option { + use crate::quic as q; + let known = q::PEN_IN_RANGE | q::PEN_TOUCHING | q::PEN_BARREL1 | q::PEN_BARREL2; + if !self.x.is_finite() || !self.y.is_finite() || self.state & !known != 0 { + return None; + } + let tool = match self.tool { + PUNKTFUNK_PEN_TOOL_PEN => q::PenTool::Pen, + PUNKTFUNK_PEN_TOOL_ERASER => q::PenTool::Eraser, + _ => return None, + }; + Some(q::PenSample { + state: self.state, + tool, + x: self.x, + y: self.y, + pressure: self.pressure, + distance: self.distance, + tilt_deg: self.tilt_deg, + azimuth_deg: self.azimuth_deg, + roll_deg: self.roll_deg, + dt_us: self.dt_us, + }) + } +} + /// Read an optional NUL-terminated UTF-8 string parameter; `Err` = invalid pointer/UTF-8. #[cfg(feature = "quic")] unsafe fn opt_cstr<'a>(p: *const std::os::raw::c_char) -> std::result::Result, ()> { @@ -988,6 +1076,12 @@ pub const PUNKTFUNK_HOST_CAP_GAMEPAD_STATE: u8 = 0x01; /// Host-capability bit in [`punktfunk_connection_host_caps`]: the host supports the shared /// clipboard, so a client may offer the toggle. (Mirrors `quic::HOST_CAP_CLIPBOARD`.) pub const PUNKTFUNK_HOST_CAP_CLIPBOARD: u8 = 0x02; +/// Host-capability bit in [`punktfunk_connection_host_caps`]: the host injects full-fidelity +/// stylus input, so a capable client splits pen contacts out of its touch path and sends them +/// via [`punktfunk_connection_send_pen`]; without the bit that call returns `Unsupported` and +/// the client keeps its pen-as-touch fallback. (Mirrors `quic::HOST_CAP_PEN`; +/// design/pen-tablet-input.md.) +pub const PUNKTFUNK_HOST_CAP_PEN: u8 = 0x10; // Keep the ABI cap bits in lockstep with the wire constants (compile-time guard against drift). #[cfg(feature = "quic")] @@ -1001,6 +1095,15 @@ const _: () = { assert!(PUNKTFUNK_CODEC_PYROWAVE == crate::quic::CODEC_PYROWAVE); assert!(PUNKTFUNK_HOST_CAP_GAMEPAD_STATE == crate::quic::HOST_CAP_GAMEPAD_STATE); assert!(PUNKTFUNK_HOST_CAP_CLIPBOARD == crate::quic::HOST_CAP_CLIPBOARD); + assert!(PUNKTFUNK_HOST_CAP_PEN == crate::quic::HOST_CAP_PEN); + assert!(PUNKTFUNK_PEN_IN_RANGE == crate::quic::PEN_IN_RANGE); + assert!(PUNKTFUNK_PEN_TOUCHING == crate::quic::PEN_TOUCHING); + assert!(PUNKTFUNK_PEN_BARREL1 == crate::quic::PEN_BARREL1); + assert!(PUNKTFUNK_PEN_BARREL2 == crate::quic::PEN_BARREL2); + assert!(PUNKTFUNK_PEN_BATCH_MAX as usize == crate::quic::PEN_BATCH_MAX); + assert!(PUNKTFUNK_PEN_TILT_UNKNOWN == crate::quic::PEN_TILT_UNKNOWN); + assert!(PUNKTFUNK_PEN_ANGLE_UNKNOWN == crate::quic::PEN_ANGLE_UNKNOWN); + assert!(PUNKTFUNK_PEN_DISTANCE_UNKNOWN == crate::quic::PEN_DISTANCE_UNKNOWN); }; // Keep the ABI gamepad constants in lockstep with the wire enum (compile-time guard against drift). @@ -2763,6 +2866,51 @@ pub unsafe extern "C" fn punktfunk_connection_send_rich_input2( }) } +/// Send one stylus sample batch — `count` (`1..=PUNKTFUNK_PEN_BATCH_MAX`) state-full +/// [`PunktfunkPenSample`]s, oldest first (a capture callback's coalesced samples) — as one +/// `0xCC/0x05` pen datagram (non-blocking enqueue; design/pen-tablet-input.md). Split longer +/// runs into consecutive calls. Gate on `punktfunk_connection_host_caps() & +/// PUNKTFUNK_HOST_CAP_PEN`: toward a host without the bit this returns +/// [`PunktfunkStatus::Unsupported`] — keep the pen-as-touch fallback there. +/// [`PunktfunkStatus::InvalidArg`] on a bad count or a bad sample (non-finite coordinate, +/// unknown state bit / tool). +/// +/// # Safety +/// `c` is a valid connection handle; `samples` is null or points to `count` valid +/// [`PunktfunkPenSample`]s. +#[cfg(feature = "quic")] +#[no_mangle] +pub unsafe extern "C" fn punktfunk_connection_send_pen( + c: *mut PunktfunkConnection, + samples: *const PunktfunkPenSample, + count: u32, +) -> PunktfunkStatus { + guard(|| { + let c = match unsafe { c.as_ref() } { + Some(c) => c, + None => return PunktfunkStatus::NullPointer, + }; + if samples.is_null() { + return PunktfunkStatus::NullPointer; + } + if count == 0 || count > PUNKTFUNK_PEN_BATCH_MAX { + return PunktfunkStatus::InvalidArg; + } + let raw = unsafe { std::slice::from_raw_parts(samples, count as usize) }; + let mut batch = [crate::quic::PenSample::default(); crate::quic::PEN_BATCH_MAX]; + for (slot, s) in batch.iter_mut().zip(raw) { + match s.to_sample() { + Some(v) => *slot = v, + None => return PunktfunkStatus::InvalidArg, + } + } + match c.inner.send_pen(&batch[..count as usize]) { + Ok(()) => PunktfunkStatus::Ok, + Err(e) => e.status(), + } + }) +} + /// The currently active session mode — the Welcome's, until an accepted /// [`punktfunk_connection_request_mode`] switches it. Safe any time after connect. /// @@ -2977,9 +3125,10 @@ fn build_clip_event( } /// The host capability bitfield the session's `Welcome` carried — a bitfield of -/// `PUNKTFUNK_HOST_CAP_GAMEPAD_STATE` / `PUNKTFUNK_HOST_CAP_CLIPBOARD`. A client tests -/// `caps & PUNKTFUNK_HOST_CAP_CLIPBOARD` to decide whether to offer the shared-clipboard toggle. -/// Safe any time after connect. +/// `PUNKTFUNK_HOST_CAP_GAMEPAD_STATE` / `PUNKTFUNK_HOST_CAP_CLIPBOARD` / +/// `PUNKTFUNK_HOST_CAP_PEN`. A client tests `caps & PUNKTFUNK_HOST_CAP_CLIPBOARD` to decide +/// whether to offer the shared-clipboard toggle, `caps & PUNKTFUNK_HOST_CAP_PEN` before +/// sending stylus batches. Safe any time after connect. /// /// # Safety /// `c` is a valid connection handle; `caps` is writable (NULL is skipped). diff --git a/crates/punktfunk-core/src/client/mod.rs b/crates/punktfunk-core/src/client/mod.rs index 45862f99..0751c432 100644 --- a/crates/punktfunk-core/src/client/mod.rs +++ b/crates/punktfunk-core/src/client/mod.rs @@ -20,7 +20,7 @@ use crate::quic::{ RfiRequest, RichInput, }; use crate::session::Frame; -use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU32, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU16, AtomicU32, AtomicU64, Ordering}; use std::sync::mpsc::{Receiver, RecvTimeoutError}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -104,8 +104,11 @@ pub struct NativeClient { /// Bounded ([`MIC_QUEUE`]): a wedged worker drops fresh frames (logged) instead of queueing /// audio-latency (and memory) without limit — mic is best-effort end to end. mic_tx: tokio::sync::mpsc::Sender<(u32, u64, Vec)>, - /// Outbound rich input (DualSense touchpad / motion) → 0xCC datagrams by the worker. - rich_input_tx: tokio::sync::mpsc::UnboundedSender, + /// Outbound 0xCC rich-input plane, PRE-ENCODED datagrams: [`RichInput`] touchpad/motion + /// (encoded in [`NativeClient::send_rich_input`]) and stylus [`crate::quic::PenBatch`]es + /// (encoded in [`NativeClient::send_pen`]) share the channel — the worker's task just + /// forwards bytes, so a new 0xCC kind never touches the pump. + rich_input_tx: tokio::sync::mpsc::UnboundedSender>, /// Outbound control-stream requests (mode switch, speed test) → the worker's control task. /// Bounded ([`CTRL_QUEUE`]) — the requests are sparse; a full queue means the control task /// is wedged/dead, and callers treat it like a closed session. @@ -121,6 +124,9 @@ pub struct NativeClient { /// Monotonic id for outbound fetches ([`NativeClient::clip_fetch`]); stays below /// [`crate::clipboard::INBOUND_REQ_FLAG`] so it never collides with an inbound serve `req_id`. next_xfer_id: AtomicU32, + /// Wrapping per-connection [`crate::quic::PenBatch::seq`] counter, stamped by + /// [`NativeClient::send_pen`] (the host's reorder gate compares it). + pen_seq: AtomicU16, /// The host capability bitfield ([`crate::quic::Welcome::host_caps`]) — see /// [`NativeClient::host_caps`]. pub host_caps: u8, @@ -344,7 +350,7 @@ impl NativeClient { std::sync::mpsc::sync_channel::(HOST_TIMING_QUEUE); let (input_tx, input_rx) = tokio::sync::mpsc::unbounded_channel::(); let (mic_tx, mic_rx) = tokio::sync::mpsc::channel::<(u32, u64, Vec)>(MIC_QUEUE); - let (rich_input_tx, rich_input_rx) = tokio::sync::mpsc::unbounded_channel::(); + let (rich_input_tx, rich_input_rx) = tokio::sync::mpsc::unbounded_channel::>(); let (ctrl_tx, ctrl_rx) = tokio::sync::mpsc::channel::(CTRL_QUEUE); let (clip_event_tx, clip_event_rx) = std::sync::mpsc::sync_channel::(CLIP_EVENT_QUEUE); @@ -473,6 +479,7 @@ impl NativeClient { clip: Mutex::new(clip_event_rx), clip_cmd_tx, next_xfer_id: AtomicU32::new(1), + pen_seq: AtomicU16::new(0), host_caps: negotiated.host_caps, probe, shutdown, @@ -1073,7 +1080,34 @@ impl NativeClient { /// loss like every datagram. No-op unless the host runs the DualSense gamepad backend. pub fn send_rich_input(&self, rich: RichInput) -> Result<()> { self.rich_input_tx - .send(rich) + .send(rich.encode()) + .map_err(|_| PunktfunkError::Closed) + } + + /// Queue one stylus sample batch for delivery as a `0xCC/0x05` pen datagram + /// (design/pen-tablet-input.md). `samples` are state-full and oldest-first (a capture + /// callback's coalesced samples), at most [`crate::quic::PEN_BATCH_MAX`] per call — split + /// longer runs into consecutive calls so the stamped wrapping `seq` keeps them ordered. + /// Best-effort like every datagram: a lost batch self-heals on the next one (the samples + /// carry full state, the host diffs — see [`crate::quic::PenTracker`]). + /// + /// Requires the host to have advertised [`crate::quic::HOST_CAP_PEN`]; toward an older + /// host this returns `Unsupported` (embedders keep their pen-as-touch fallback instead of + /// spraying 240 Hz datagrams the host drops unread). + pub fn send_pen(&self, samples: &[crate::quic::PenSample]) -> Result<()> { + if self.host_caps & crate::quic::HOST_CAP_PEN == 0 { + return Err(PunktfunkError::Unsupported( + "host did not advertise HOST_CAP_PEN", + )); + } + if samples.is_empty() || samples.len() > crate::quic::PEN_BATCH_MAX { + return Err(PunktfunkError::InvalidArg( + "pen batch must hold 1..=PEN_BATCH_MAX samples", + )); + } + let seq = self.pen_seq.fetch_add(1, Ordering::Relaxed); + self.rich_input_tx + .send(crate::quic::PenBatch::new(seq, samples).encode()) .map_err(|_| PunktfunkError::Closed) } diff --git a/crates/punktfunk-core/src/client/pump.rs b/crates/punktfunk-core/src/client/pump.rs index a84f2bd9..9d14e945 100644 --- a/crates/punktfunk-core/src/client/pump.rs +++ b/crates/punktfunk-core/src/client/pump.rs @@ -99,11 +99,12 @@ pub(super) async fn run_pump(args: WorkerArgs) { } }); - // Rich-input task: embedder DualSense touchpad / motion → 0xCC uplink datagrams. + // Rich-input task: pre-encoded 0xCC uplink datagrams (DualSense touchpad / motion, pen + // batches — encoded at the NativeClient surface so new plane kinds never touch the pump). let rich_conn = conn.clone(); tokio::spawn(async move { - while let Some(rich) = rich_input_rx.recv().await { - let _ = rich_conn.send_datagram(rich.encode().into()); + while let Some(d) = rich_input_rx.recv().await { + let _ = rich_conn.send_datagram(d.into()); } }); diff --git a/crates/punktfunk-core/src/client/worker.rs b/crates/punktfunk-core/src/client/worker.rs index e2b277e1..b0605dcc 100644 --- a/crates/punktfunk-core/src/client/worker.rs +++ b/crates/punktfunk-core/src/client/worker.rs @@ -5,7 +5,7 @@ use crate::clipboard::{ClipCommand, ClipEventCore}; use crate::config::{CompositorPref, GamepadPref, Mode}; use crate::error::Result; use crate::input::InputEvent; -use crate::quic::{HdrMeta, HidOutput, RichInput}; +use crate::quic::{HdrMeta, HidOutput}; use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64}; use std::sync::mpsc::SyncSender; use std::sync::{Arc, Mutex}; @@ -43,7 +43,8 @@ pub(crate) struct WorkerArgs { pub(crate) cursor_state_tx: SyncSender, pub(crate) input_rx: tokio::sync::mpsc::UnboundedReceiver, pub(crate) mic_rx: tokio::sync::mpsc::Receiver<(u32, u64, Vec)>, - pub(crate) rich_input_rx: tokio::sync::mpsc::UnboundedReceiver, + /// Pre-encoded 0xCC datagrams (rich input AND pen batches — see `NativeClient.rich_input_tx`). + pub(crate) rich_input_rx: tokio::sync::mpsc::UnboundedReceiver>, pub(crate) ctrl_rx: tokio::sync::mpsc::Receiver, pub(crate) ctrl_tx: tokio::sync::mpsc::Sender, /// Inbound clipboard event plane feed — the control task pushes ClipState/ClipOffer, the diff --git a/crates/punktfunk-core/src/quic/caps.rs b/crates/punktfunk-core/src/quic/caps.rs index 73f54dcc..a43d8cee 100644 --- a/crates/punktfunk-core/src/quic/caps.rs +++ b/crates/punktfunk-core/src/quic/caps.rs @@ -100,6 +100,18 @@ pub const CLIENT_CAP_CURSOR: u8 = 0x01; /// [`HOST_CAP_TEXT_INPUT`], `0x01`/`0x02` are gamepad-state / clipboard. pub const HOST_CAP_CURSOR: u8 = 0x08; +/// [`Welcome::host_caps`] bit: the host injects full-fidelity stylus input — it routes +/// [`PenBatch`](super::pen::PenBatch) `0xCC/0x05` datagrams (pressure, tilt, azimuth, barrel +/// roll, hover, eraser, barrel buttons) through the [`PenTracker`](super::pen::PenTracker) +/// into a virtual tablet device (design/pen-tablet-input.md). A capable client (Apple Pencil, +/// Android stylus) then splits pen contacts out of its finger/touch path and sends pen +/// batches; absent the bit it keeps folding the pen into touch/pointer like today, and +/// [`NativeClient::send_pen`](crate::client::NativeClient::send_pen) refuses to send. The +/// wire ships ahead of the backend (P0): no host sets this bit until the P1 injector lands — +/// which is exactly why the gate exists. `0x10` — `0x08` is [`HOST_CAP_CURSOR`], `0x04` is +/// [`HOST_CAP_TEXT_INPUT`], `0x01`/`0x02` are gamepad-state / clipboard. +pub const HOST_CAP_PEN: u8 = 0x10; + /// [`Hello::video_codecs`] bit: the client can decode H.264 / AVC. The GPU-less **software** /// encode path (openh264) emits H.264, so a client that wants to stream from a software host MUST /// advertise this. diff --git a/crates/punktfunk-core/src/quic/datagram.rs b/crates/punktfunk-core/src/quic/datagram.rs index b8cd283e..bdfdaeca 100644 --- a/crates/punktfunk-core/src/quic/datagram.rs +++ b/crates/punktfunk-core/src/quic/datagram.rs @@ -162,6 +162,12 @@ pub(super) const RICH_TOUCHPAD: u8 = 0x01; pub(super) const RICH_MOTION: u8 = 0x02; pub(super) const RICH_TOUCHPAD_EX: u8 = 0x03; pub(super) const RICH_HID_REPORT: u8 = 0x04; +/// Claimed by the stylus plane ([`PenBatch`](super::pen::PenBatch)), which is NOT a +/// [`RichInput`] variant: rich input is pad-indexed controller state consumed by the gamepad +/// backends, while a pen batch routes to the (P1) tablet injector — so it gets its own decoder +/// and [`RichInput::decode`] keeps returning `None` here (= the documented unknown-kind drop +/// on a pre-pen host). Registered in this list so 0xCC kind bytes stay unique. +pub(super) const RICH_PEN: u8 = 0x05; /// Longest raw HID report a [`RichInput::HidReport`] / [`HidOutput::HidRaw`] can carry — the /// 64-byte interrupt/feature report size every Valve controller uses (Triton input reports are @@ -171,7 +177,8 @@ pub const HID_REPORT_MAX: usize = 64; /// A rich client→host controller input beyond the fixed [`InputEvent`](crate::input::InputEvent): /// the DualSense touchpad and motion sensors. `pad` is the gamepad index. Wire form is /// `[0xCC][kind][fields…]` — variable-length and kind-tagged (forward-compatible: an unknown -/// kind decodes to `None` and is dropped). +/// kind decodes to `None` and is dropped). Kind `0x05` on this plane is the stylus batch +/// ([`PenBatch`](super::pen::PenBatch)) with its own decoder — see [`RICH_PEN`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum RichInput { /// One touchpad contact. `x`/`y` are normalized `0..=65535` in SCREEN convention — diff --git a/crates/punktfunk-core/src/quic/mod.rs b/crates/punktfunk-core/src/quic/mod.rs index da830e2c..326374f5 100644 --- a/crates/punktfunk-core/src/quic/mod.rs +++ b/crates/punktfunk-core/src/quic/mod.rs @@ -26,6 +26,7 @@ //! positional Hello/Welcome/Start codecs, `caps` the capability/codec-negotiation //! vocabulary, `control` the typed control + clipboard messages, `pairing` the pairing //! message codecs with [`pake`] the SPAKE2 itself, `datagram` the 0xC9–0xCF plane codecs, +//! `pen` the stylus batch (0xCC kind 0x05) + host stroke tracker, //! [`io`] framed stream IO, `clock` skew estimation + mid-stream re-sync, [`endpoint`] the //! quinn constructors, [`clipstream`] the per-transfer clipboard fetch streams. Every item //! is re-exported here, so all existing `crate::quic::X` paths compile unchanged; each @@ -46,6 +47,7 @@ mod control; mod datagram; mod handshake; mod pairing; +mod pen; /// quinn endpoint constructors. Host: self-signed identity (fresh, or persisted PEMs via /// [`endpoint::server_with_identity`]). Client: fingerprint pinning / TOFU via @@ -72,6 +74,7 @@ pub use control::*; pub use datagram::*; pub use handshake::*; pub use pairing::*; +pub use pen::*; // Typed rejection close codes + [`RejectReason`] live in `crate::reject` (ungated — the // error enum references them even in `quic`-less builds) and are re-exported here so the diff --git a/crates/punktfunk-core/src/quic/pen.rs b/crates/punktfunk-core/src/quic/pen.rs new file mode 100644 index 00000000..dcb09a1f --- /dev/null +++ b/crates/punktfunk-core/src/quic/pen.rs @@ -0,0 +1,689 @@ +//! The stylus plane: full-fidelity pen input on the 0xCC rich-input datagram +//! (design/pen-tablet-input.md). +//! +//! A pen datagram is a batch of **state-full samples**: every sample carries the *complete* +//! pen state — in-range/touching/buttons/tool plus all axes — never an edge ("down"/"up") +//! event. The host diffs each sample against its own tracked state ([`PenTracker`]) and +//! synthesizes the transitions, so a lost datagram self-heals on the next sample: a dropped +//! "first contact" batch becomes a tip-down when the next in-contact sample arrives, and a +//! dropped lift heals on the next hover/out-of-range sample (with the +//! [`PEN_TOUCH_TIMEOUT_MS`] failsafe for a client that dies mid-stroke). That is what makes +//! the lossy datagram plane sound for a 240 Hz stylus without any reliable-delivery +//! machinery — the same idempotent-snapshot argument as [`GamepadSnapshot`] +//! (crate::input::GamepadSnapshot) and [`RichInput::HidReport`](super::RichInput). +//! +//! Batches are ordered by a wrapping `u16` sequence number and dropped **whole** when stale +//! ([`pen_seq_newer`]) — applying a stale state-full sample would rewind the stroke. +//! +//! Clients send this only after the host advertised [`HOST_CAP_PEN`](super::HOST_CAP_PEN); +//! a pre-pen host drops the unknown 0xCC kind by the plane's documented forward-compat rule. + +use super::datagram::{RICH_INPUT_MAGIC, RICH_PEN}; + +/// [`PenSample::state`] bit: the pen is in the hover range of the surface. Implied by +/// [`PEN_TOUCHING`] (decode normalizes, so a client that only sets TOUCHING still produces a +/// coherent contact). +pub const PEN_IN_RANGE: u8 = 0x01; +/// [`PenSample::state`] bit: the tip is in contact with the surface. +pub const PEN_TOUCHING: u8 = 0x02; +/// [`PenSample::state`] bit: the primary barrel button (or the client's squeeze mapping) is held. +pub const PEN_BARREL1: u8 = 0x04; +/// [`PenSample::state`] bit: the secondary barrel button (or the client's double-tap mapping) +/// is held. +pub const PEN_BARREL2: u8 = 0x08; +/// [`PenSample::state`] bit, RESERVED: a predicted (not yet observed) sample. Never sent v1; +/// receivers MUST ignore samples carrying it until a capability negotiates otherwise +/// (design/pen-tablet-input.md §8). +pub const PEN_PREDICTED: u8 = 0x80; + +/// The button subset of [`PenSample::state`]. +const PEN_BUTTONS_MASK: u8 = PEN_BARREL1 | PEN_BARREL2; + +/// [`PenSample::tilt_deg`] sentinel: the client has no tilt sensor / no reading. +pub const PEN_TILT_UNKNOWN: u8 = 0xFF; +/// [`PenSample::azimuth_deg`] / [`PenSample::roll_deg`] sentinel: no reading. +pub const PEN_ANGLE_UNKNOWN: u16 = 0xFFFF; +/// [`PenSample::distance`] sentinel: no hover-distance reading. +pub const PEN_DISTANCE_UNKNOWN: u16 = 0xFFFF; + +/// Most samples one [`PenBatch`] can carry. Sized for coalesced capture at video-frame cadence +/// (240 Hz pen ÷ 30 fps = 8); a client producing more splits into consecutive batches. +pub const PEN_BATCH_MAX: usize = 8; + +/// Wire length of one encoded [`PenSample`]. +pub const PEN_SAMPLE_WIRE_LEN: usize = 21; + +/// `[0xCC][0x05][flags][count][u16 seq LE]` — bytes before the first sample. +const PEN_HEADER_LEN: usize = 6; + +/// Host-side failsafe (design/pen-tablet-input.md §2): a tracker still touching after this many +/// ms without a sample force-releases ([`PenTracker::force_release`]) — a client that died +/// mid-stroke must not leave the host's virtual pen inked-down forever. Far above any real +/// send cadence (a touching pen streams samples continuously), so it never fires on a live +/// slow stroke. +pub const PEN_TOUCH_TIMEOUT_MS: u32 = 200; + +/// Which end of the stylus (or which mapped mode) a sample describes. A tool *switch* while in +/// range is a physical re-entry — [`PenTracker`] emits a full release + re-proximity, matching +/// how a real tablet treats each tool as its own proximity session. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(u8)] +pub enum PenTool { + #[default] + Pen = 0, + Eraser = 1, + /// An unrecognized wire value (a future tool from a newer client) — injectors treat it as + /// [`PenTool::Pen`]. Inside a proximity session it inherits the session's tool instead of + /// forcing a spurious re-entry. + Unknown = 0xFF, +} + +impl PenTool { + fn from_u8(v: u8) -> PenTool { + match v { + 0 => PenTool::Pen, + 1 => PenTool::Eraser, + _ => PenTool::Unknown, + } + } +} + +/// One complete stylus state at one instant. All axes ride every sample (state-full — see the +/// module doc); unknown axes carry their sentinel, never 0. +/// +/// `x`/`y` are normalized `0.0..=1.0` in **video-frame space** — the client maps its letterbox +/// / viewport before sending (exactly as its wire touches already do), so the host scales +/// straight to the streamed output. f32 keeps sub-pixel precision at any resolution. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PenSample { + /// Bitfield of `PEN_*` state bits ([`PEN_IN_RANGE`] … [`PEN_PREDICTED`]). + pub state: u8, + /// Active tool. Meaningful while in range; [`PenTool::Unknown`] inherits (see [`PenTool`]). + pub tool: PenTool, + /// Normalized `0.0..=1.0` across the video frame (decode clamps; see [`PenBatch::decode`]). + pub x: f32, + /// Normalized `0.0..=1.0` across the video frame. + pub y: f32, + /// Tip force, `0..=65535` full scale, `0` while hovering. Injectors rescale (Windows pens + /// are 0..1024, uinput declares its own range) — full u16 keeps every source's precision. + pub pressure: u16, + /// Hover distance, `0..=65534` normalized (0 = touching the hover floor), or + /// [`PEN_DISTANCE_UNKNOWN`]. + pub distance: u16, + /// Tilt from the surface normal, degrees `0..=90`, or [`PEN_TILT_UNKNOWN`]. Polar form — + /// what Apple capture and the GameStream wire both produce; injectors needing tiltX/tiltY + /// convert (design/pen-tablet-input.md §2). + pub tilt_deg: u8, + /// Tilt azimuth, degrees `0..=359` clockwise from north, or [`PEN_ANGLE_UNKNOWN`]. + pub azimuth_deg: u16, + /// Barrel roll (Apple Pencil Pro `rollAngle`), degrees `0..=359`, or [`PEN_ANGLE_UNKNOWN`]. + pub roll_deg: u16, + /// µs since the previous sample in the same batch (`0` for the first) — preserves the + /// coalesced capture spacing for injectors/consumers that pace. + pub dt_us: u16, +} + +impl Default for PenSample { + fn default() -> PenSample { + PenSample { + state: 0, + tool: PenTool::Pen, + x: 0.0, + y: 0.0, + pressure: 0, + distance: PEN_DISTANCE_UNKNOWN, + tilt_deg: PEN_TILT_UNKNOWN, + azimuth_deg: PEN_ANGLE_UNKNOWN, + roll_deg: PEN_ANGLE_UNKNOWN, + dt_us: 0, + } + } +} + +impl PenSample { + fn encode_into(&self, out: &mut Vec) { + out.push(self.state); + out.push(self.tool as u8); + out.extend_from_slice(&self.x.to_le_bytes()); + out.extend_from_slice(&self.y.to_le_bytes()); + out.extend_from_slice(&self.pressure.to_le_bytes()); + out.extend_from_slice(&self.distance.to_le_bytes()); + out.push(self.tilt_deg); + out.extend_from_slice(&self.azimuth_deg.to_le_bytes()); + out.extend_from_slice(&self.roll_deg.to_le_bytes()); + out.extend_from_slice(&self.dt_us.to_le_bytes()); + } + + /// Decode one sample from exactly [`PEN_SAMPLE_WIRE_LEN`] bytes (caller bounds-checks). + /// `None` on a non-finite coordinate — an attacker-forged NaN/∞ must never reach an + /// injector's pixel scaling. Finite out-of-range coordinates clamp to `0.0..=1.0` (a + /// stroke drifting a hair past the letterbox edge is real input, not corruption). + fn decode(b: &[u8]) -> Option { + let f32at = |o: usize| f32::from_le_bytes([b[o], b[o + 1], b[o + 2], b[o + 3]]); + let u16at = |o: usize| u16::from_le_bytes([b[o], b[o + 1]]); + let (x, y) = (f32at(2), f32at(6)); + if !x.is_finite() || !y.is_finite() { + return None; + } + Some(PenSample { + state: b[0], + tool: PenTool::from_u8(b[1]), + x: x.clamp(0.0, 1.0), + y: y.clamp(0.0, 1.0), + pressure: u16at(10), + distance: u16at(12), + tilt_deg: b[14], + azimuth_deg: u16at(15), + roll_deg: u16at(17), + dt_us: u16at(19), + }) + } +} + +/// One pen datagram: `[0xCC][0x05][flags][count][u16 seq LE]` + `count` × +/// [`PEN_SAMPLE_WIRE_LEN`]-byte samples, oldest first. `flags` is reserved (sent 0, ignored on +/// decode — semantic changes take a new 0xCC kind, never a flag reinterpretation). `seq` is the +/// sender's wrapping batch counter, the reorder gate ([`pen_seq_newer`]). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PenBatch { + pub seq: u16, + count: u8, + samples: [PenSample; PEN_BATCH_MAX], +} + +impl PenBatch { + /// Build a batch from up to [`PEN_BATCH_MAX`] samples (a longer slice truncates — senders + /// with more coalesced samples split into consecutive batches so nothing is lost). + pub fn new(seq: u16, samples: &[PenSample]) -> PenBatch { + let count = samples.len().min(PEN_BATCH_MAX); + let mut buf = [PenSample::default(); PEN_BATCH_MAX]; + buf[..count].copy_from_slice(&samples[..count]); + PenBatch { + seq, + count: count as u8, + samples: buf, + } + } + + /// The batch's samples, oldest first. + pub fn samples(&self) -> &[PenSample] { + &self.samples[..self.count as usize] + } + + pub fn encode(&self) -> Vec { + let n = self.count as usize; + let mut out = Vec::with_capacity(PEN_HEADER_LEN + n * PEN_SAMPLE_WIRE_LEN); + out.extend_from_slice(&[RICH_INPUT_MAGIC, RICH_PEN, 0, self.count]); + out.extend_from_slice(&self.seq.to_le_bytes()); + for s in self.samples() { + s.encode_into(&mut out); + } + out + } + + /// Parse a pen datagram. `None` on bad tag/kind, an empty batch, or a forged coordinate + /// (see [`PenSample::decode`]). Every read is bounded: `count` clamps to the declared + /// value, the fixed maximum, AND what the buffer actually holds — a torn datagram yields + /// the complete samples that arrived, never an over-read (the + /// [`RichInput::HidReport`](super::RichInput) truncation contract). + pub fn decode(b: &[u8]) -> Option { + if b.len() < PEN_HEADER_LEN || b[0] != RICH_INPUT_MAGIC || b[1] != RICH_PEN { + return None; + } + let count = (b[3] as usize) + .min(PEN_BATCH_MAX) + .min((b.len() - PEN_HEADER_LEN) / PEN_SAMPLE_WIRE_LEN); + if count == 0 { + return None; + } + let mut samples = [PenSample::default(); PEN_BATCH_MAX]; + for (i, slot) in samples.iter_mut().enumerate().take(count) { + let o = PEN_HEADER_LEN + i * PEN_SAMPLE_WIRE_LEN; + *slot = PenSample::decode(&b[o..o + PEN_SAMPLE_WIRE_LEN])?; + } + Some(PenBatch { + seq: u16::from_le_bytes([b[4], b[5]]), + count: count as u8, + samples, + }) + } +} + +/// The batch reorder gate: is `new` strictly newer than `last` on the wrapping u16 circle? +/// `None` (nothing applied yet) always passes. The u16 analog of +/// [`GamepadSnapshot::seq_newer`](crate::input::GamepadSnapshot::seq_newer): newer ⇔ the +/// forward distance is `1..=0x7FFF`, so reordered stale batches drop and a wrap (65535 → 0) +/// still counts as newer. +pub fn pen_seq_newer(new: u16, last: Option) -> bool { + match last { + None => true, + Some(last) => (new.wrapping_sub(last) as i16) > 0, + } +} + +/// One synthesized stroke transition, in the order [`PenTracker`] emits them for a sample: +/// `ProximityIn?` → `Motion` → `TipDown?` → `ButtonsChanged?` → `TipUp?` → `ProximityOut?`. +/// `Motion` precedes `TipDown` so the contact lands where the sample says; a release +/// ([`PenTracker::force_release`] or an out-of-range sample) orders +/// `ButtonsChanged?` → `TipUp?` → `ProximityOut` so nothing is left held. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum PenTransition { + /// The pen entered hover range. Injectors map [`PenTool::Unknown`] to a plain pen. + ProximityIn { tool: PenTool }, + /// Position + all axes moved to this sample's values (emitted for every in-range sample). + Motion { sample: PenSample }, + /// The tip made contact. + TipDown, + /// Barrel buttons changed: `pressed` / `released` are disjoint `PEN_BARREL*` subsets. + ButtonsChanged { pressed: u8, released: u8 }, + /// The tip lifted. + TipUp, + /// The pen left hover range. + ProximityOut, +} + +/// The host-side stroke state machine (one per session): diffs state-full [`PenSample`]s +/// against tracked state and appends the synthesized [`PenTransition`]s. Pure and clock-free — +/// the owner arms its own [`PEN_TOUCH_TIMEOUT_MS`] timer over [`PenTracker::is_active`] and +/// calls [`PenTracker::force_release`] when it fires (and on session teardown). +#[derive(Debug, Default)] +pub struct PenTracker { + last_seq: Option, + in_range: bool, + touching: bool, + buttons: u8, + tool: PenTool, +} + +impl PenTracker { + /// Apply one decoded batch, appending transitions to `out` (callers reuse the buffer). A + /// stale batch ([`pen_seq_newer`]) is dropped whole with no transitions. Samples carrying + /// the reserved [`PEN_PREDICTED`] bit are skipped (never injected — module doc). + pub fn apply(&mut self, batch: &PenBatch, out: &mut Vec) { + if !pen_seq_newer(batch.seq, self.last_seq) { + return; + } + self.last_seq = Some(batch.seq); + for s in batch.samples() { + if s.state & PEN_PREDICTED != 0 { + continue; + } + self.apply_sample(s, out); + } + } + + /// The tracker holds live state a dead client could leave stuck (in range, or mid-stroke). + pub fn is_active(&self) -> bool { + self.in_range || self.touching + } + + /// Release everything held (buttons → tip → proximity) — the [`PEN_TOUCH_TIMEOUT_MS`] + /// failsafe and session teardown. Keeps the seq gate armed so a late stale datagram from + /// the dead stroke cannot re-apply after the release. + pub fn force_release(&mut self, out: &mut Vec) { + if self.buttons != 0 { + out.push(PenTransition::ButtonsChanged { + pressed: 0, + released: self.buttons, + }); + self.buttons = 0; + } + if self.touching { + out.push(PenTransition::TipUp); + self.touching = false; + } + if self.in_range { + out.push(PenTransition::ProximityOut); + self.in_range = false; + } + } + + fn apply_sample(&mut self, s: &PenSample, out: &mut Vec) { + let touching = s.state & PEN_TOUCHING != 0; + // Touching implies in-range (a contact IS a proximity) — normalize here once so every + // consumer sees coherent states. + let in_range = touching || s.state & PEN_IN_RANGE != 0; + if !in_range { + self.force_release(out); + return; + } + // Unknown inherits the session's tool (a newer client's future tool must not thrash + // proximity); outside a session it grounds to the default. + let tool = match s.tool { + PenTool::Unknown if self.in_range => self.tool, + t => t, + }; + // A tool switch mid-session is a physical re-entry (see [`PenTool`]). + if self.in_range && tool != self.tool { + self.force_release(out); + } + if !self.in_range { + out.push(PenTransition::ProximityIn { tool }); + self.in_range = true; + } + self.tool = tool; + out.push(PenTransition::Motion { sample: *s }); + if touching && !self.touching { + out.push(PenTransition::TipDown); + self.touching = true; + } + let buttons = s.state & PEN_BUTTONS_MASK; + if buttons != self.buttons { + out.push(PenTransition::ButtonsChanged { + pressed: buttons & !self.buttons, + released: self.buttons & !buttons, + }); + self.buttons = buttons; + } + if !touching && self.touching { + out.push(PenTransition::TipUp); + self.touching = false; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::quic::RichInput; + + fn hover(x: f32, y: f32) -> PenSample { + PenSample { + state: PEN_IN_RANGE, + x, + y, + distance: 300, + ..Default::default() + } + } + + fn touch(x: f32, y: f32, pressure: u16) -> PenSample { + PenSample { + state: PEN_IN_RANGE | PEN_TOUCHING, + x, + y, + pressure, + distance: 0, + tilt_deg: 35, + azimuth_deg: 180, + roll_deg: 90, + ..Default::default() + } + } + + #[test] + fn pen_batch_roundtrip_and_truncation() { + let samples = [hover(0.25, 0.5), touch(0.26, 0.5, 40000), { + let mut s = touch(0.27, 0.51, 42000); + s.state |= PEN_BARREL1; + s.dt_us = 4167; + s + }]; + let b = PenBatch::new(7, &samples); + let d = b.encode(); + assert_eq!(d[0], RICH_INPUT_MAGIC); + assert_eq!(d.len(), 6 + 3 * PEN_SAMPLE_WIRE_LEN); + let back = PenBatch::decode(&d).unwrap(); + assert_eq!(back.seq, 7); + assert_eq!(back.samples(), &samples); + + // A torn datagram yields exactly the complete samples that arrived — never an + // over-read, never a partial sample. + let torn = PenBatch::decode(&d[..6 + 2 * PEN_SAMPLE_WIRE_LEN + 5]).unwrap(); + assert_eq!(torn.samples(), &samples[..2]); + // Header-only / empty batches and short buffers are rejected whole. + assert!(PenBatch::decode(&d[..PEN_HEADER_LEN]).is_none()); + assert!(PenBatch::decode(&PenBatch::new(0, &[]).encode()).is_none()); + // Wrong tag / wrong kind are None before any read. + let mut bad = d.clone(); + bad[0] = 0xC8; + assert!(PenBatch::decode(&bad).is_none()); + let mut bad = d; + bad[1] = 0x01; // RICH_TOUCHPAD + assert!(PenBatch::decode(&bad).is_none()); + } + + #[test] + fn pen_batch_oversize_truncates_and_flags_reserved() { + // 10 samples truncate to PEN_BATCH_MAX on construction (senders split instead). + let many: Vec = (0..10).map(|i| hover(i as f32 / 10.0, 0.5)).collect(); + let b = PenBatch::new(1, &many); + assert_eq!(b.samples().len(), PEN_BATCH_MAX); + // A declared count larger than the payload clamps to what arrived. + let mut d = b.encode(); + d[3] = 200; + assert_eq!(PenBatch::decode(&d).unwrap().samples().len(), PEN_BATCH_MAX); + // A nonzero reserved flags byte still parses (receivers MUST ignore). + d[2] = 0xAA; + assert!(PenBatch::decode(&d).is_some()); + } + + #[test] + fn pen_batch_rejects_forged_floats_and_clamps_stragglers() { + // NaN / ∞ coordinates kill the whole batch — nothing legitimate produces them. + for forged in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { + let mut s = touch(0.5, 0.5, 1); + s.x = forged; + assert!(PenBatch::decode(&PenBatch::new(0, &[s]).encode()).is_none()); + } + // A finite coordinate a hair outside the letterbox clamps instead (real input). + let mut s = hover(0.5, 0.5); + s.x = -0.01; + s.y = 1.25; + let back = PenBatch::decode(&PenBatch::new(0, &[s]).encode()).unwrap(); + assert_eq!((back.samples()[0].x, back.samples()[0].y), (0.0, 1.0)); + } + + #[test] + fn pen_plane_is_disjoint_from_rich_input() { + // A pen datagram shares the 0xCC tag but is NOT a RichInput: a pre-pen host takes the + // documented unknown-kind drop, and the pen decoder rejects every RichInput kind. + let d = PenBatch::new(3, &[touch(0.5, 0.5, 100)]).encode(); + assert!(RichInput::decode(&d).is_none()); + let rich = RichInput::Touchpad { + pad: 0, + finger: 0, + active: true, + x: 1, + y: 2, + } + .encode(); + assert!(PenBatch::decode(&rich).is_none()); + } + + #[test] + fn seq_gate_wraps_and_drops_stale() { + assert!(pen_seq_newer(0, None)); + assert!(pen_seq_newer(6, Some(5))); + assert!(!pen_seq_newer(5, Some(5))); + assert!(!pen_seq_newer(4, Some(5))); + assert!(pen_seq_newer(2, Some(0xFFFE))); // wrap + assert!(!pen_seq_newer(0xFFFE, Some(2))); // stale across the wrap + } + + /// Drives a tracker and returns the transitions of one batch. + fn run(t: &mut PenTracker, seq: u16, samples: &[PenSample]) -> Vec { + let mut out = Vec::new(); + t.apply(&PenBatch::new(seq, samples), &mut out); + out + } + + #[test] + fn tracker_full_stroke_lifecycle() { + let mut t = PenTracker::default(); + // Hover in → ProximityIn + Motion. + let h = hover(0.2, 0.2); + assert_eq!( + run(&mut t, 0, &[h]), + vec![ + PenTransition::ProximityIn { tool: PenTool::Pen }, + PenTransition::Motion { sample: h }, + ] + ); + // Contact: Motion precedes TipDown so ink lands at the sample's position. + let c = touch(0.21, 0.2, 30000); + assert_eq!( + run(&mut t, 1, &[c]), + vec![PenTransition::Motion { sample: c }, PenTransition::TipDown] + ); + assert!(t.is_active()); + // Drag: motion only. + let m = touch(0.3, 0.25, 45000); + assert_eq!( + run(&mut t, 2, &[m]), + vec![PenTransition::Motion { sample: m }] + ); + // Lift back to hover, then leave range: buttons(none) → TipUp, then ProximityOut. + let l = hover(0.3, 0.25); + assert_eq!( + run(&mut t, 3, &[l]), + vec![PenTransition::Motion { sample: l }, PenTransition::TipUp] + ); + let gone = PenSample::default(); // state 0 = out of range + assert_eq!(run(&mut t, 4, &[gone]), vec![PenTransition::ProximityOut]); + assert!(!t.is_active()); + } + + #[test] + fn tracker_self_heals_lost_transitions() { + let mut t = PenTracker::default(); + // The hover batch AND the tip-down batch were lost: the first surviving mid-stroke + // sample synthesizes the whole entry. + let m = touch(0.5, 0.5, 20000); + assert_eq!( + run(&mut t, 10, &[m]), + vec![ + PenTransition::ProximityIn { tool: PenTool::Pen }, + PenTransition::Motion { sample: m }, + PenTransition::TipDown, + ] + ); + // The lift batch was lost; the next out-of-range sample heals it fully. + assert_eq!( + run(&mut t, 11, &[PenSample::default()]), + vec![PenTransition::TipUp, PenTransition::ProximityOut] + ); + } + + #[test] + fn tracker_drops_stale_batches_whole() { + let mut t = PenTracker::default(); + let c = touch(0.5, 0.5, 100); + assert!(!run(&mut t, 5, &[c]).is_empty()); + // A reordered older batch (a hover from before the contact) must not rewind the stroke. + assert!(run(&mut t, 4, &[hover(0.4, 0.4)]).is_empty()); + assert!(t.is_active()); + } + + #[test] + fn tracker_buttons_and_eraser_reentry() { + let mut t = PenTracker::default(); + let mut held = touch(0.5, 0.5, 100); + held.state |= PEN_BARREL1; + // Buttons apply after TipDown on entry… + assert_eq!( + run(&mut t, 0, &[held]), + vec![ + PenTransition::ProximityIn { tool: PenTool::Pen }, + PenTransition::Motion { sample: held }, + PenTransition::TipDown, + PenTransition::ButtonsChanged { + pressed: PEN_BARREL1, + released: 0 + }, + ] + ); + // …swap BARREL1 → BARREL2 in one sample: one delta, both directions. + let mut swapped = held; + swapped.state = (swapped.state & !PEN_BARREL1) | PEN_BARREL2; + assert_eq!( + run(&mut t, 1, &[swapped]), + vec![ + PenTransition::Motion { sample: swapped }, + PenTransition::ButtonsChanged { + pressed: PEN_BARREL2, + released: PEN_BARREL1 + }, + ] + ); + // Tool switch to eraser mid-contact = full release + re-entry as the eraser, with the + // held button released first so nothing sticks across tools. + let mut erase = touch(0.5, 0.5, 200); + erase.tool = PenTool::Eraser; + assert_eq!( + run(&mut t, 2, &[erase]), + vec![ + PenTransition::ButtonsChanged { + pressed: 0, + released: PEN_BARREL2 + }, + PenTransition::TipUp, + PenTransition::ProximityOut, + PenTransition::ProximityIn { + tool: PenTool::Eraser + }, + PenTransition::Motion { sample: erase }, + PenTransition::TipDown, + ] + ); + // Unknown tool inside the session inherits (no re-entry thrash from a newer client). + let mut unk = touch(0.51, 0.5, 210); + unk.tool = PenTool::Unknown; + assert_eq!( + run(&mut t, 3, &[unk]), + vec![PenTransition::Motion { sample: unk }] + ); + } + + #[test] + fn tracker_force_release_and_late_stale_datagram() { + let mut t = PenTracker::default(); + let mut held = touch(0.5, 0.5, 100); + held.state |= PEN_BARREL2; + run(&mut t, 100, &[held]); + // The 200 ms failsafe / teardown: buttons → tip → proximity, all released. + let mut out = Vec::new(); + t.force_release(&mut out); + assert_eq!( + out, + vec![ + PenTransition::ButtonsChanged { + pressed: 0, + released: PEN_BARREL2 + }, + PenTransition::TipUp, + PenTransition::ProximityOut, + ] + ); + assert!(!t.is_active()); + // Idempotent. + let mut out = Vec::new(); + t.force_release(&mut out); + assert!(out.is_empty()); + // The seq gate survives the release: a late datagram from the dead stroke is stale. + assert!(run(&mut t, 99, &[held]).is_empty()); + assert!(!t.is_active()); + // But the stroke after it proceeds normally. + assert!(!run(&mut t, 101, &[held]).is_empty()); + assert!(t.is_active()); + } + + #[test] + fn tracker_skips_reserved_predicted_samples() { + let mut t = PenTracker::default(); + let mut p = touch(0.5, 0.5, 100); + p.state |= PEN_PREDICTED; + // A v1 host must never inject a predicted sample — and skipping must not corrupt + // tracking for the real samples around it. + let real = touch(0.6, 0.5, 120); + let out = run(&mut t, 0, &[p, real]); + assert_eq!( + out, + vec![ + PenTransition::ProximityIn { tool: PenTool::Pen }, + PenTransition::Motion { sample: real }, + PenTransition::TipDown, + ] + ); + } +} diff --git a/docs/embedding-the-c-abi.md b/docs/embedding-the-c-abi.md index d066d326..25f169a7 100644 --- a/docs/embedding-the-c-abi.md +++ b/docs/embedding-the-c-abi.md @@ -437,6 +437,38 @@ punktfunk_connection_send_input(c, &ev); - `punktfunk_connection_send_rich_input2(c, &richEx)` — the forward-compatible superset (Steam trackpads, signed coords, pressure); set `struct_size = sizeof(PunktfunkRichInputEx)`. +### Stylus / pen + +Full-fidelity pen (pressure, tilt, azimuth, barrel roll, hover, eraser, barrel buttons) has its +own plane. **Gate on the capability first** — only send when +`punktfunk_connection_host_caps(c, &caps)` shows `PUNKTFUNK_HOST_CAP_PEN`; without it the call +returns `UNSUPPORTED` and you keep your pen-as-touch fallback (what every client does today). + +Samples are **state-full**: every sample carries the complete pen state (in-range / touching / +buttons in `state`, plus all axes — unknown axes take their `PUNKTFUNK_PEN_*_UNKNOWN` sentinel, +never 0). There are no down/up events to send; the host diffs consecutive samples and +synthesizes the transitions, so a lost datagram self-heals on the next one. Batch a capture +callback's coalesced samples (oldest first, `dt_us` = spacing) into one call, at most +`PUNKTFUNK_PEN_BATCH_MAX` per call — split longer runs into consecutive calls. + +```c +// Example: an in-contact Apple Pencil sample, mapped into video-frame space +PunktfunkPenSample s; memset(&s, 0, sizeof s); +s.state = PUNKTFUNK_PEN_IN_RANGE | PUNKTFUNK_PEN_TOUCHING; +s.tool = PUNKTFUNK_PEN_TOOL_PEN; +s.x = 0.5f; s.y = 0.5f; // normalized 0..1 across the video frame +s.pressure = (uint16_t)(force_norm * 65535); // 0 while hovering +s.tilt_deg = 90 - altitude_deg; // polar tilt-from-normal +s.azimuth_deg = azimuth_deg; // 0..359, clockwise from north +s.roll_deg = PUNKTFUNK_PEN_ANGLE_UNKNOWN; // Pencil Pro: rollAngle in degrees +s.distance = PUNKTFUNK_PEN_DISTANCE_UNKNOWN; +punktfunk_connection_send_pen(c, &s, 1); +``` + +While hovering, send `PUNKTFUNK_PEN_IN_RANGE` samples (with `distance` if you have it); when the +pen leaves range, send one final sample with `state = 0` so the host releases proximity — +though a host-side timeout releases a vanished client's stroke regardless. + --- ## 9. Feedback planes (rumble, HID, HDR) diff --git a/include/punktfunk_core.h b/include/punktfunk_core.h index 8c41c269..ae0e6767 100644 --- a/include/punktfunk_core.h +++ b/include/punktfunk_core.h @@ -88,6 +88,37 @@ // `punktfunk_connection_send_rich_input2` (added with client capture). #define PUNKTFUNK_RICH_TOUCHPAD_EX 3 +// [`PunktfunkPenSample::state`] bit: the pen hovers in range (implied by `TOUCHING`). +#define PUNKTFUNK_PEN_IN_RANGE 1 + +// [`PunktfunkPenSample::state`] bit: the tip is in contact. +#define PUNKTFUNK_PEN_TOUCHING 2 + +// [`PunktfunkPenSample::state`] bit: primary barrel button (or squeeze mapping) held. +#define PUNKTFUNK_PEN_BARREL1 4 + +// [`PunktfunkPenSample::state`] bit: secondary barrel button (or double-tap mapping) held. +#define PUNKTFUNK_PEN_BARREL2 8 + +// [`PunktfunkPenSample::tool`]: the pen tip. +#define PUNKTFUNK_PEN_TOOL_PEN 0 + +// [`PunktfunkPenSample::tool`]: the eraser (a client-side mode — Apple Pencil has no +// hardware eraser end; the squeeze/double-tap mapping usually drives this). +#define PUNKTFUNK_PEN_TOOL_ERASER 1 + +// Most samples one [`punktfunk_connection_send_pen`] call accepts (one wire batch). +#define PUNKTFUNK_PEN_BATCH_MAX 8 + +// [`PunktfunkPenSample::tilt_deg`] sentinel: no tilt reading. +#define PUNKTFUNK_PEN_TILT_UNKNOWN 255 + +// [`PunktfunkPenSample::azimuth_deg`] / `roll_deg` sentinel: no reading. +#define PUNKTFUNK_PEN_ANGLE_UNKNOWN 65535 + +// [`PunktfunkPenSample::distance`] sentinel: no hover-distance reading. +#define PUNKTFUNK_PEN_DISTANCE_UNKNOWN 65535 + // Compositor preference for [`punktfunk_connect_ex`] (`compositor` arg). `AUTO` lets the host // pick (auto-detect from its running desktop); a concrete value is honored only if that backend // is available on the host right now, else the host falls back to auto-detect. The resolved @@ -219,6 +250,13 @@ // clipboard, so a client may offer the toggle. (Mirrors `quic::HOST_CAP_CLIPBOARD`.) #define PUNKTFUNK_HOST_CAP_CLIPBOARD 2 +// Host-capability bit in [`punktfunk_connection_host_caps`]: the host injects full-fidelity +// stylus input, so a capable client splits pen contacts out of its touch path and sends them +// via [`punktfunk_connection_send_pen`]; without the bit that call returns `Unsupported` and +// the client keeps its pen-as-touch fallback. (Mirrors `quic::HOST_CAP_PEN`; +// design/pen-tablet-input.md.) +#define PUNKTFUNK_HOST_CAP_PEN 16 + // [`punktfunk_connect_ex9`] `client_caps` bit: render the host cursor locally (the cursor // channel, `design/remote-desktop-sweep.md` M2). #define PUNKTFUNK_CLIENT_CAP_CURSOR 1 @@ -541,6 +579,20 @@ #define HOST_CAP_CURSOR 8 #endif +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`Welcome::host_caps`] bit: the host injects full-fidelity stylus input — it routes +// [`PenBatch`](super::pen::PenBatch) `0xCC/0x05` datagrams (pressure, tilt, azimuth, barrel +// roll, hover, eraser, barrel buttons) through the [`PenTracker`](super::pen::PenTracker) +// into a virtual tablet device (design/pen-tablet-input.md). A capable client (Apple Pencil, +// Android stylus) then splits pen contacts out of its finger/touch path and sends pen +// batches; absent the bit it keeps folding the pen into touch/pointer like today, and +// [`NativeClient::send_pen`](crate::client::NativeClient::send_pen) refuses to send. The +// wire ships ahead of the backend (P0): no host sets this bit until the P1 injector lands — +// which is exactly why the gate exists. `0x10` — `0x08` is [`HOST_CAP_CURSOR`], `0x04` is +// [`HOST_CAP_TEXT_INPUT`], `0x01`/`0x02` are gamepad-state / clipboard. +#define HOST_CAP_PEN 16 +#endif + #if defined(PUNKTFUNK_FEATURE_QUIC) // [`Hello::video_codecs`] bit: the client can decode H.264 / AVC. The GPU-less **software** // encode path (openh264) emits H.264, so a client that wants to stream from a software host MUST @@ -985,6 +1037,71 @@ #define MSG_PAIR_RESULT 19 #endif +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`PenSample::state`] bit: the pen is in the hover range of the surface. Implied by +// [`PEN_TOUCHING`] (decode normalizes, so a client that only sets TOUCHING still produces a +// coherent contact). +#define PEN_IN_RANGE 1 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`PenSample::state`] bit: the tip is in contact with the surface. +#define PEN_TOUCHING 2 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`PenSample::state`] bit: the primary barrel button (or the client's squeeze mapping) is held. +#define PEN_BARREL1 4 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`PenSample::state`] bit: the secondary barrel button (or the client's double-tap mapping) +// is held. +#define PEN_BARREL2 8 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`PenSample::state`] bit, RESERVED: a predicted (not yet observed) sample. Never sent v1; +// receivers MUST ignore samples carrying it until a capability negotiates otherwise +// (design/pen-tablet-input.md §8). +#define PEN_PREDICTED 128 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`PenSample::tilt_deg`] sentinel: the client has no tilt sensor / no reading. +#define PEN_TILT_UNKNOWN 255 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`PenSample::azimuth_deg`] / [`PenSample::roll_deg`] sentinel: no reading. +#define PEN_ANGLE_UNKNOWN 65535 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// [`PenSample::distance`] sentinel: no hover-distance reading. +#define PEN_DISTANCE_UNKNOWN 65535 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// Most samples one [`PenBatch`] can carry. Sized for coalesced capture at video-frame cadence +// (240 Hz pen ÷ 30 fps = 8); a client producing more splits into consecutive batches. +#define PEN_BATCH_MAX 8 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// Wire length of one encoded [`PenSample`]. +#define PEN_SAMPLE_WIRE_LEN 21 +#endif + +#if defined(PUNKTFUNK_FEATURE_QUIC) +// Host-side failsafe (design/pen-tablet-input.md §2): a tracker still touching after this many +// ms without a sample force-releases ([`PenTracker::force_release`]) — a client that died +// mid-stroke must not leave the host's virtual pen inked-down forever. Far above any real +// send cadence (a touching pen streams samples continuously), so it never fires on a live +// slow stroke. +#define PEN_TOUCH_TIMEOUT_MS 200 +#endif + #if defined(PUNKTFUNK_FEATURE_QUIC) // Stream-kind byte: a clipboard fetch (request/response of one format). Future stream kinds // (e.g. a bulk file-content push) mux under the same [`STREAM_MAGIC`] with a different byte. @@ -1489,6 +1606,41 @@ typedef struct { } PunktfunkRichInputEx; #endif +#if defined(PUNKTFUNK_FEATURE_QUIC) +// One complete stylus state at one instant ([`punktfunk_connection_send_pen`]; +// design/pen-tablet-input.md). STATE-FULL, never an edge event: fill every field on every +// sample (unknown axes take their `*_UNKNOWN` sentinel) — the host diffs consecutive samples +// and synthesizes down/up/button transitions itself, which is what makes a lost datagram +// self-heal. `x`/`y` are normalized `0.0..=1.0` in VIDEO-FRAME space (map your letterbox +// before filling, exactly like wire touches). +typedef struct { + // Normalized `0.0..=1.0` across the video frame. Must be finite. + float x; + // Normalized `0.0..=1.0` across the video frame. Must be finite. + float y; + // Tip force, `0..=65535` full scale (`0` while hovering). + uint16_t pressure; + // Hover distance `0..=65534` (0 = at the hover floor), or `PUNKTFUNK_PEN_DISTANCE_UNKNOWN`. + uint16_t distance; + // Tilt azimuth, degrees `0..=359` clockwise from north, or `PUNKTFUNK_PEN_ANGLE_UNKNOWN`. + uint16_t azimuth_deg; + // Barrel roll (Apple Pencil Pro `rollAngle`), degrees `0..=359`, or + // `PUNKTFUNK_PEN_ANGLE_UNKNOWN`. + uint16_t roll_deg; + // µs since the previous sample in the same call (`0` for the first) — the coalesced + // capture spacing. + uint16_t dt_us; + // Bitfield of `PUNKTFUNK_PEN_*` state bits. Unknown bits are rejected (`InvalidArg`). + uint8_t state; + // `PUNKTFUNK_PEN_TOOL_PEN` or `PUNKTFUNK_PEN_TOOL_ERASER`. + uint8_t tool; + // Tilt from the surface normal, degrees `0..=90`, or `PUNKTFUNK_PEN_TILT_UNKNOWN`. + uint8_t tilt_deg; + // Set to 0. + uint8_t _reserved[3]; +} PunktfunkPenSample; +#endif + #if defined(PUNKTFUNK_FEATURE_QUIC) // One advertised clipboard format passed to [`punktfunk_connection_clipboard_offer`]. typedef struct { @@ -2310,6 +2462,24 @@ PunktfunkStatus punktfunk_connection_send_rich_input2(PunktfunkConnection *c, const PunktfunkRichInputEx *rich); #endif +#if defined(PUNKTFUNK_FEATURE_QUIC) +// Send one stylus sample batch — `count` (`1..=PUNKTFUNK_PEN_BATCH_MAX`) state-full +// [`PunktfunkPenSample`]s, oldest first (a capture callback's coalesced samples) — as one +// `0xCC/0x05` pen datagram (non-blocking enqueue; design/pen-tablet-input.md). Split longer +// runs into consecutive calls. Gate on `punktfunk_connection_host_caps() & +// PUNKTFUNK_HOST_CAP_PEN`: toward a host without the bit this returns +// [`PunktfunkStatus::Unsupported`] — keep the pen-as-touch fallback there. +// [`PunktfunkStatus::InvalidArg`] on a bad count or a bad sample (non-finite coordinate, +// unknown state bit / tool). +// +// # Safety +// `c` is a valid connection handle; `samples` is null or points to `count` valid +// [`PunktfunkPenSample`]s. +PunktfunkStatus punktfunk_connection_send_pen(PunktfunkConnection *c, + const PunktfunkPenSample *samples, + uint32_t count); +#endif + #if defined(PUNKTFUNK_FEATURE_QUIC) // The currently active session mode — the Welcome's, until an accepted // [`punktfunk_connection_request_mode`] switches it. Safe any time after connect. @@ -2335,9 +2505,10 @@ PunktfunkStatus punktfunk_connection_gamepad(const PunktfunkConnection *c, uint3 #if defined(PUNKTFUNK_FEATURE_QUIC) // The host capability bitfield the session's `Welcome` carried — a bitfield of -// `PUNKTFUNK_HOST_CAP_GAMEPAD_STATE` / `PUNKTFUNK_HOST_CAP_CLIPBOARD`. A client tests -// `caps & PUNKTFUNK_HOST_CAP_CLIPBOARD` to decide whether to offer the shared-clipboard toggle. -// Safe any time after connect. +// `PUNKTFUNK_HOST_CAP_GAMEPAD_STATE` / `PUNKTFUNK_HOST_CAP_CLIPBOARD` / +// `PUNKTFUNK_HOST_CAP_PEN`. A client tests `caps & PUNKTFUNK_HOST_CAP_CLIPBOARD` to decide +// whether to offer the shared-clipboard toggle, `caps & PUNKTFUNK_HOST_CAP_PEN` before +// sending stylus batches. Safe any time after connect. // // # Safety // `c` is a valid connection handle; `caps` is writable (NULL is skipped).