feat(gamestream): pen P2 — SS_PEN/SS_TOUCH ingest, featureFlags advertised on capable hosts
The Moonlight leg of design/pen-tablet-input.md §4: DESCRIBE now advertises SS_FF_PEN_TOUCH_EVENTS wherever pen injection exists (Linux+uinput, same gate as HOST_CAP_PEN — elsewhere the flag stays 0 and Moonlight keeps client-side mouse emulation exactly as today). SS_PEN packets merge over last-known state into state-full PenSamples (BUTTON_ONLY per spec carries no position; unknown contact pressure 0.0 inks at full scale; UP keeps proximity only for clients that demonstrated hover) and drive the same PenTracker→VirtualPen chain as the native plane — Moonlight iOS + Apple Pencil now exercises the full injection path end to end. SS_TOUCH forwards as wire touch on a synthetic 65535² surface with CANCEL_ALL replayed per tracked contact. No stroke timeout on this plane: ENet is ordered/reliable and Moonlight doesn't heartbeat a stationary pen. Packet layouts pinned against moonlight-common-c Input.h/Limelight.h upstream. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -71,6 +71,9 @@ pub fn spawn(state: Arc<AppState>) -> Result<()> {
|
|||||||
// from `seq`, so a per-message-type counter would reuse (key, nonce) pairs across
|
// from `seq`, so a per-message-type counter would reuse (key, nonce) pairs across
|
||||||
// message types in the host direction.
|
// message types in the host direction.
|
||||||
let mut pads = GamepadManager::new();
|
let mut pads = GamepadManager::new();
|
||||||
|
// Pen/touch translator (SS_PEN/SS_TOUCH → virtual tablet / wire touch). Sent only
|
||||||
|
// by clients that saw our SS_FF_PEN_TOUCH_EVENTS feature flag (rtsp.rs).
|
||||||
|
let mut pointer = super::pen::GsPointer::new();
|
||||||
let mut host_seq: u32 = 0;
|
let mut host_seq: u32 = 0;
|
||||||
// One-shot latch for the HDR-mode control message (0x010e); re-armed on Disconnect.
|
// One-shot latch for the HDR-mode control message (0x010e); re-armed on Disconnect.
|
||||||
let mut hdr_sent = false;
|
let mut hdr_sent = false;
|
||||||
@@ -90,8 +93,10 @@ pub fn spawn(state: Arc<AppState>) -> Result<()> {
|
|||||||
peer = None;
|
peer = None;
|
||||||
// Re-arm the HDR-mode signal for the next connection.
|
// Re-arm the HDR-mode signal for the next connection.
|
||||||
hdr_sent = false;
|
hdr_sent = false;
|
||||||
// Unplug the session's virtual pads.
|
// Unplug the session's virtual pads + tablet (destroying the
|
||||||
|
// uinput pen releases any held tool/tip kernel-side).
|
||||||
pads = GamepadManager::new();
|
pads = GamepadManager::new();
|
||||||
|
pointer = super::pen::GsPointer::new();
|
||||||
}
|
}
|
||||||
Event::Receive {
|
Event::Receive {
|
||||||
channel_id, packet, ..
|
channel_id, packet, ..
|
||||||
@@ -104,6 +109,7 @@ pub fn spawn(state: Arc<AppState>) -> Result<()> {
|
|||||||
&mut decrypt_fails,
|
&mut decrypt_fails,
|
||||||
&inj_tx,
|
&inj_tx,
|
||||||
&mut pads,
|
&mut pads,
|
||||||
|
&mut pointer,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -196,6 +202,7 @@ fn decode_rfi_range(pt: &[u8]) -> Option<(i64, i64)> {
|
|||||||
|
|
||||||
/// Handle one received control packet: decrypt it (learning the GCM scheme on the first one),
|
/// Handle one received control packet: decrypt it (learning the GCM scheme on the first one),
|
||||||
/// decode any input event, and inject it into the host session.
|
/// decode any input event, and inject it into the host session.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
fn on_receive(
|
fn on_receive(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
_channel_id: u8,
|
_channel_id: u8,
|
||||||
@@ -204,6 +211,7 @@ fn on_receive(
|
|||||||
decrypt_fails: &mut u64,
|
decrypt_fails: &mut u64,
|
||||||
inj_tx: &Sender<InputEvent>,
|
inj_tx: &Sender<InputEvent>,
|
||||||
pads: &mut GamepadManager,
|
pads: &mut GamepadManager,
|
||||||
|
pointer: &mut super::pen::GsPointer,
|
||||||
) {
|
) {
|
||||||
let Some(key) = state.launch.lock().unwrap().map(|s| s.gcm_key) else {
|
let Some(key) = state.launch.lock().unwrap().map(|s| s.gcm_key) else {
|
||||||
return; // control traffic before /launch — no key yet
|
return; // control traffic before /launch — no key yet
|
||||||
@@ -274,6 +282,15 @@ fn on_receive(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Pen/touch extension events (Moonlight sends them only after seeing our feature flag):
|
||||||
|
// pen drives this session's virtual tablet; touch forwards as ordinary wire touches.
|
||||||
|
if let Some(p) = super::input::decode_pointer(&pt) {
|
||||||
|
pointer.apply(&p, |ev| {
|
||||||
|
let _ = inj_tx.send(ev);
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
let events = super::input::decode(&pt);
|
let events = super::input::decode(&pt);
|
||||||
if events.is_empty() {
|
if events.is_empty() {
|
||||||
return; // keepalive / QoS / unhandled input kind
|
return; // keepalive / QoS / unhandled input kind
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ const MAGIC_MOUSE_BTN_UP: u32 = 0x09;
|
|||||||
const MAGIC_SCROLL_GEN5: u32 = 0x0A;
|
const MAGIC_SCROLL_GEN5: u32 = 0x0A;
|
||||||
const MAGIC_UTF8: u32 = 0x17;
|
const MAGIC_UTF8: u32 = 0x17;
|
||||||
const MAGIC_HSCROLL: u32 = 0x5500_0001;
|
const MAGIC_HSCROLL: u32 = 0x5500_0001;
|
||||||
|
const MAGIC_SS_TOUCH: u32 = 0x5500_0002;
|
||||||
|
const MAGIC_SS_PEN: u32 = 0x5500_0003;
|
||||||
|
|
||||||
/// `code` value marking a [`InputKind::MouseScroll`] as horizontal (vs `0` = vertical).
|
/// `code` value marking a [`InputKind::MouseScroll`] as horizontal (vs `0` = vertical).
|
||||||
pub const SCROLL_HORIZONTAL: u32 = 1;
|
pub const SCROLL_HORIZONTAL: u32 = 1;
|
||||||
@@ -111,6 +113,84 @@ fn decode_input_packet(p: &[u8]) -> Option<InputEvent> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// One decoded `SS_PEN_PACKET` body (moonlight-common-c `Input.h`; all fields little-endian,
|
||||||
|
/// coordinates/pressure as normalized floats). Semantics — `pressure_or_distance` is pressure
|
||||||
|
/// (0..1, 0 = unknown) for contact events and hover distance (1 = farthest) for hover;
|
||||||
|
/// `rotation` is the tilt azimuth (0..360, `0xFFFF` unknown); `tilt` is degrees from the
|
||||||
|
/// surface normal (0..90, `0xFF` unknown) — are interpreted in [`super::pen`].
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||||
|
pub struct SsPen {
|
||||||
|
pub event_type: u8,
|
||||||
|
pub tool: u8,
|
||||||
|
pub buttons: u8,
|
||||||
|
pub x: f32,
|
||||||
|
pub y: f32,
|
||||||
|
pub pressure_or_distance: f32,
|
||||||
|
pub rotation: u16,
|
||||||
|
pub tilt: u8,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One decoded `SS_TOUCH_PACKET` body (same conventions as [`SsPen`]; contact-area fields are
|
||||||
|
/// not carried — the wire touch kinds have nowhere to put them yet).
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||||
|
pub struct SsTouch {
|
||||||
|
pub event_type: u8,
|
||||||
|
pub rotation: u16,
|
||||||
|
pub pointer_id: u32,
|
||||||
|
pub x: f32,
|
||||||
|
pub y: f32,
|
||||||
|
pub pressure_or_distance: f32,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A Sunshine-extension pointer event (sent only after we advertise
|
||||||
|
/// `SS_FF_PEN_TOUCH_EVENTS` — see [`super::rtsp`]).
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||||
|
pub enum SsPointer {
|
||||||
|
Pen(SsPen),
|
||||||
|
Touch(SsTouch),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decode a control plaintext into a pen/touch pointer event, or `None` for every other
|
||||||
|
/// message (the caller then falls through to [`decode`]). Bounds- and sanity-checked like the
|
||||||
|
/// rest of the plane: short bodies and non-finite floats (a forged NaN must never reach the
|
||||||
|
/// injectors' scaling) drop the packet whole.
|
||||||
|
pub fn decode_pointer(plaintext: &[u8]) -> Option<SsPointer> {
|
||||||
|
if plaintext.len() < 12 || u16::from_le_bytes([plaintext[0], plaintext[1]]) != INPUT_DATA_TYPE {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let p = &plaintext[4..];
|
||||||
|
let magic = u32::from_le_bytes([p[4], p[5], p[6], p[7]]);
|
||||||
|
let b = &p[8..];
|
||||||
|
let f32at = |o: usize| -> Option<f32> {
|
||||||
|
let v = f32::from_le_bytes([*b.get(o)?, *b.get(o + 1)?, *b.get(o + 2)?, *b.get(o + 3)?]);
|
||||||
|
v.is_finite().then_some(v)
|
||||||
|
};
|
||||||
|
match magic {
|
||||||
|
// eventType, zero[1], rotation u16, pointerId u32, x, y, pressureOrDistance, areas.
|
||||||
|
MAGIC_SS_TOUCH => Some(SsPointer::Touch(SsTouch {
|
||||||
|
event_type: *b.first()?,
|
||||||
|
rotation: u16::from_le_bytes([*b.get(2)?, *b.get(3)?]),
|
||||||
|
pointer_id: u32::from_le_bytes([*b.get(4)?, *b.get(5)?, *b.get(6)?, *b.get(7)?]),
|
||||||
|
x: f32at(8)?,
|
||||||
|
y: f32at(12)?,
|
||||||
|
pressure_or_distance: f32at(16)?,
|
||||||
|
})),
|
||||||
|
// eventType, toolType, penButtons, zero[1], x, y, pressureOrDistance, rotation u16,
|
||||||
|
// tilt, zero2[1], areas.
|
||||||
|
MAGIC_SS_PEN => Some(SsPointer::Pen(SsPen {
|
||||||
|
event_type: *b.first()?,
|
||||||
|
tool: *b.get(1)?,
|
||||||
|
buttons: *b.get(2)?,
|
||||||
|
x: f32at(4)?,
|
||||||
|
y: f32at(8)?,
|
||||||
|
pressure_or_distance: f32at(12)?,
|
||||||
|
rotation: u16::from_le_bytes([*b.get(16)?, *b.get(17)?]),
|
||||||
|
tilt: *b.get(18)?,
|
||||||
|
})),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn ev(kind: InputKind, code: u32, x: i32, y: i32, flags: u32) -> InputEvent {
|
fn ev(kind: InputKind, code: u32, x: i32, y: i32, flags: u32) -> InputEvent {
|
||||||
InputEvent {
|
InputEvent {
|
||||||
kind,
|
kind,
|
||||||
@@ -176,6 +256,68 @@ mod tests {
|
|||||||
assert!(decode(&bad).is_empty());
|
assert!(decode(&bad).is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decodes_ss_pen_and_touch_golden_bytes() {
|
||||||
|
// SS_PEN body per Input.h: DOWN, pen tool, primary button, x=0.5 y=0.25,
|
||||||
|
// pressure=0.75, rotation=180, tilt=45, then contact areas (present but ignored).
|
||||||
|
let mut body = vec![0x01, 0x01, 0x01, 0x00];
|
||||||
|
for f in [0.5f32, 0.25, 0.75] {
|
||||||
|
body.extend_from_slice(&f.to_le_bytes());
|
||||||
|
}
|
||||||
|
body.extend_from_slice(&180u16.to_le_bytes());
|
||||||
|
body.extend_from_slice(&[45, 0x00]);
|
||||||
|
for f in [0.0f32, 0.0] {
|
||||||
|
body.extend_from_slice(&f.to_le_bytes());
|
||||||
|
}
|
||||||
|
let pt = wrap(0x5500_0003, &body);
|
||||||
|
assert_eq!(
|
||||||
|
decode_pointer(&pt),
|
||||||
|
Some(SsPointer::Pen(SsPen {
|
||||||
|
event_type: 0x01,
|
||||||
|
tool: 0x01,
|
||||||
|
buttons: 0x01,
|
||||||
|
x: 0.5,
|
||||||
|
y: 0.25,
|
||||||
|
pressure_or_distance: 0.75,
|
||||||
|
rotation: 180,
|
||||||
|
tilt: 45,
|
||||||
|
}))
|
||||||
|
);
|
||||||
|
// A pen packet is invisible to the classic decoder (no misparse as mouse/key).
|
||||||
|
assert!(decode(&pt).is_empty());
|
||||||
|
|
||||||
|
// SS_TOUCH body: MOVE, rotation unknown, pointerId 42, x=1.0 y=0.0, pressure 1.0.
|
||||||
|
let mut body = vec![0x03, 0x00];
|
||||||
|
body.extend_from_slice(&0xFFFFu16.to_le_bytes());
|
||||||
|
body.extend_from_slice(&42u32.to_le_bytes());
|
||||||
|
for f in [1.0f32, 0.0, 1.0, 0.0, 0.0] {
|
||||||
|
body.extend_from_slice(&f.to_le_bytes());
|
||||||
|
}
|
||||||
|
let pt = wrap(0x5500_0002, &body);
|
||||||
|
assert_eq!(
|
||||||
|
decode_pointer(&pt),
|
||||||
|
Some(SsPointer::Touch(SsTouch {
|
||||||
|
event_type: 0x03,
|
||||||
|
rotation: 0xFFFF,
|
||||||
|
pointer_id: 42,
|
||||||
|
x: 1.0,
|
||||||
|
y: 0.0,
|
||||||
|
pressure_or_distance: 1.0,
|
||||||
|
}))
|
||||||
|
);
|
||||||
|
|
||||||
|
// Truncated bodies and forged NaN coordinates drop the packet whole.
|
||||||
|
assert_eq!(decode_pointer(&pt[..pt.len() - 18]), None);
|
||||||
|
let mut nan = body.clone();
|
||||||
|
nan[8..12].copy_from_slice(&f32::NAN.to_le_bytes());
|
||||||
|
assert_eq!(decode_pointer(&wrap(0x5500_0002, &nan)), None);
|
||||||
|
// Non-pointer magics fall through to the classic decoder.
|
||||||
|
assert_eq!(
|
||||||
|
decode_pointer(&wrap(MAGIC_MOUSE_REL_GEN5, &[0, 0, 0, 0])),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ignores_non_input_type() {
|
fn ignores_non_input_type() {
|
||||||
let mut pt = vec![0x00, 0x02]; // type 0x0200 (keepalive)
|
let mut pt = vec![0x00, 0x02]; // type 0x0200 (keepalive)
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ mod input;
|
|||||||
mod mdns;
|
mod mdns;
|
||||||
mod nvhttp;
|
mod nvhttp;
|
||||||
mod pairing;
|
mod pairing;
|
||||||
|
/// Moonlight `SS_PEN`/`SS_TOUCH` → the native pen model / wire touch (design/pen-tablet-input.md §4).
|
||||||
|
mod pen;
|
||||||
mod rtsp;
|
mod rtsp;
|
||||||
mod serverinfo;
|
mod serverinfo;
|
||||||
mod stream;
|
mod stream;
|
||||||
|
|||||||
@@ -0,0 +1,360 @@
|
|||||||
|
//! Translate Moonlight's edge-based `SS_PEN` / `SS_TOUCH` events (design/pen-tablet-input.md
|
||||||
|
//! §4) into punktfunk's state-full pen model and wire-touch events.
|
||||||
|
//!
|
||||||
|
//! Pen: each packet becomes one complete [`PenSample`] (packet fields merged over the last
|
||||||
|
//! known state, since e.g. `BUTTON_ONLY` carries no position by spec), fed through the same
|
||||||
|
//! [`PenTracker`] → [`VirtualPen`] chain the native plane uses — so Moonlight iOS with an
|
||||||
|
//! Apple Pencil exercises the exact injection path our own clients will.
|
||||||
|
//!
|
||||||
|
//! No stroke timeout here, deliberately: the native plane's 200 ms failsafe exists because
|
||||||
|
//! datagrams are lossy and clients heartbeat; Moonlight's control stream is ordered/reliable
|
||||||
|
//! ENet and its clients do NOT heartbeat a stationary pen — a timeout would lift a held
|
||||||
|
//! stroke. Lost-client cleanup is the ENet disconnect (the control loop re-news this
|
||||||
|
//! translator, and destroying the uinput device releases everything kernel-side).
|
||||||
|
//!
|
||||||
|
//! Touch: forwarded as the ordinary wire touch kinds (`TouchDown/Move/Up`, normalized onto a
|
||||||
|
//! synthetic 65535² surface) through the injector service — pressure/area are dropped v1
|
||||||
|
//! (the wire touch has no field for them; `RICH_TOUCH` is the design's §8 upgrade).
|
||||||
|
|
||||||
|
use super::input::{SsPen, SsPointer, SsTouch};
|
||||||
|
use punktfunk_core::input::{InputEvent, InputKind};
|
||||||
|
use punktfunk_core::quic::{
|
||||||
|
PenSample, PenTool, PenTracker, PenTransition, PEN_ANGLE_UNKNOWN, PEN_BARREL1, PEN_BARREL2,
|
||||||
|
PEN_IN_RANGE, PEN_TILT_UNKNOWN, PEN_TOUCHING,
|
||||||
|
};
|
||||||
|
|
||||||
|
// moonlight-common-c Limelight.h event/tool/button vocabulary.
|
||||||
|
const LI_TOUCH_EVENT_HOVER: u8 = 0x00;
|
||||||
|
const LI_TOUCH_EVENT_DOWN: u8 = 0x01;
|
||||||
|
const LI_TOUCH_EVENT_UP: u8 = 0x02;
|
||||||
|
const LI_TOUCH_EVENT_MOVE: u8 = 0x03;
|
||||||
|
const LI_TOUCH_EVENT_CANCEL: u8 = 0x04;
|
||||||
|
const LI_TOUCH_EVENT_BUTTON_ONLY: u8 = 0x05;
|
||||||
|
const LI_TOUCH_EVENT_HOVER_LEAVE: u8 = 0x06;
|
||||||
|
const LI_TOUCH_EVENT_CANCEL_ALL: u8 = 0x07;
|
||||||
|
const LI_TOOL_TYPE_PEN: u8 = 0x01;
|
||||||
|
const LI_TOOL_TYPE_ERASER: u8 = 0x02;
|
||||||
|
const LI_PEN_BUTTON_PRIMARY: u8 = 0x01;
|
||||||
|
const LI_PEN_BUTTON_SECONDARY: u8 = 0x02;
|
||||||
|
const LI_ROT_UNKNOWN: u16 = 0xFFFF;
|
||||||
|
const LI_TILT_UNKNOWN: u8 = 0xFF;
|
||||||
|
|
||||||
|
/// Synthetic reference extent for forwarded touches: coordinates arrive normalized, so map
|
||||||
|
/// them onto a full-range surface and let the injector rescale to its output (the
|
||||||
|
/// `MouseMoveAbs`/touch `flags = (w << 16) | h` contract).
|
||||||
|
const TOUCH_SURFACE: u32 = 65535;
|
||||||
|
|
||||||
|
/// Most concurrently-tracked touch pointers (for `CANCEL_ALL` replay). A flood of distinct
|
||||||
|
/// ids can't grow memory — untracked ids still forward down/up verbatim, they just miss a
|
||||||
|
/// later `CANCEL_ALL` (which a real client never has more than ~10 contacts for anyway).
|
||||||
|
const MAX_TOUCH_IDS: usize = 32;
|
||||||
|
|
||||||
|
/// Per-GameStream-session pen + touch translator. Owned by the control loop next to the
|
||||||
|
/// gamepad manager; re-created on client disconnect (the dropped [`VirtualPen`] destroys the
|
||||||
|
/// uinput device, which releases any held tool/tip kernel-side).
|
||||||
|
pub struct GsPointer {
|
||||||
|
tracker: PenTracker,
|
||||||
|
dev: Option<crate::inject::pen::VirtualPen>,
|
||||||
|
create_failed: bool,
|
||||||
|
seq: u16,
|
||||||
|
out: Vec<PenTransition>,
|
||||||
|
/// Last complete pen state — the merge base for packets that omit fields
|
||||||
|
/// (`BUTTON_ONLY` ignores x/y/pressure/tilt/rotation by spec).
|
||||||
|
last: PenSample,
|
||||||
|
/// This client sends hover events, so an `UP` means "back to hover" rather than "gone" —
|
||||||
|
/// clients without hover support get proximity-out on lift instead of a pen parked
|
||||||
|
/// forever at the last point.
|
||||||
|
saw_hover: bool,
|
||||||
|
/// Active forwarded touch ids, for `CANCEL_ALL` replay as per-id ups.
|
||||||
|
touch_ids: Vec<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GsPointer {
|
||||||
|
pub fn new() -> GsPointer {
|
||||||
|
GsPointer {
|
||||||
|
tracker: PenTracker::default(),
|
||||||
|
dev: None,
|
||||||
|
create_failed: false,
|
||||||
|
seq: 0,
|
||||||
|
out: Vec::new(),
|
||||||
|
last: PenSample::default(),
|
||||||
|
saw_hover: false,
|
||||||
|
touch_ids: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apply one decoded pointer packet. Touch events forward through `sink` (the injector
|
||||||
|
/// channel); pen events drive this session's virtual tablet.
|
||||||
|
pub fn apply(&mut self, p: &SsPointer, sink: impl FnMut(InputEvent)) {
|
||||||
|
match p {
|
||||||
|
SsPointer::Pen(pen) => self.apply_pen(pen),
|
||||||
|
SsPointer::Touch(touch) => self.apply_touch(touch, sink),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_pen(&mut self, p: &SsPen) {
|
||||||
|
let Some(sample) = self.pen_sample(p) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
self.last = sample;
|
||||||
|
if self.dev.is_none() && !self.create_failed {
|
||||||
|
match crate::inject::pen::VirtualPen::create() {
|
||||||
|
Ok(d) => self.dev = Some(d),
|
||||||
|
Err(e) => {
|
||||||
|
self.create_failed = true;
|
||||||
|
tracing::warn!(
|
||||||
|
error = %format!("{e:#}"),
|
||||||
|
"gamestream pen: virtual tablet creation failed — dropping pen input"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.out.clear();
|
||||||
|
let batch = punktfunk_core::quic::PenBatch::new(self.seq, &[sample]);
|
||||||
|
self.seq = self.seq.wrapping_add(1);
|
||||||
|
self.tracker.apply(&batch, &mut self.out);
|
||||||
|
if let Some(dev) = self.dev.as_mut() {
|
||||||
|
dev.apply_batch(&self.out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Merge one edge-based packet over the last known state into a complete sample.
|
||||||
|
/// `None` = nothing to apply (a `BUTTON_ONLY` before any positioned event).
|
||||||
|
fn pen_sample(&mut self, p: &SsPen) -> Option<PenSample> {
|
||||||
|
let buttons = (if p.buttons & LI_PEN_BUTTON_PRIMARY != 0 {
|
||||||
|
PEN_BARREL1
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}) | (if p.buttons & LI_PEN_BUTTON_SECONDARY != 0 {
|
||||||
|
PEN_BARREL2
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
});
|
||||||
|
let tool = match p.tool {
|
||||||
|
LI_TOOL_TYPE_PEN => PenTool::Pen,
|
||||||
|
LI_TOOL_TYPE_ERASER => PenTool::Eraser,
|
||||||
|
_ => PenTool::Unknown,
|
||||||
|
};
|
||||||
|
// BUTTON_ONLY ignores x/y/pressure/rotation/tilt by spec — reuse the last state.
|
||||||
|
if p.event_type == LI_TOUCH_EVENT_BUTTON_ONLY {
|
||||||
|
if !self.tracker.is_active() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut s = self.last;
|
||||||
|
s.state = (s.state & !(PEN_BARREL1 | PEN_BARREL2)) | buttons;
|
||||||
|
return Some(s);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut s = PenSample {
|
||||||
|
state: buttons,
|
||||||
|
tool,
|
||||||
|
x: p.x.clamp(0.0, 1.0),
|
||||||
|
y: p.y.clamp(0.0, 1.0),
|
||||||
|
dt_us: 0,
|
||||||
|
roll_deg: PEN_ANGLE_UNKNOWN, // the GameStream wire has no barrel-roll axis
|
||||||
|
azimuth_deg: if p.rotation == LI_ROT_UNKNOWN {
|
||||||
|
PEN_ANGLE_UNKNOWN
|
||||||
|
} else {
|
||||||
|
p.rotation % 360
|
||||||
|
},
|
||||||
|
tilt_deg: if p.tilt == LI_TILT_UNKNOWN {
|
||||||
|
PEN_TILT_UNKNOWN
|
||||||
|
} else {
|
||||||
|
p.tilt.min(90)
|
||||||
|
},
|
||||||
|
..PenSample::default()
|
||||||
|
};
|
||||||
|
match p.event_type {
|
||||||
|
LI_TOUCH_EVENT_DOWN | LI_TOUCH_EVENT_MOVE => {
|
||||||
|
s.state |= PEN_IN_RANGE | PEN_TOUCHING;
|
||||||
|
// pressureOrDistance is pressure (0..1) in contact — and 0.0 means UNKNOWN
|
||||||
|
// there, which must still ink: binary-stylus clients get full pressure.
|
||||||
|
s.pressure = if p.pressure_or_distance <= 0.0 {
|
||||||
|
u16::MAX
|
||||||
|
} else {
|
||||||
|
(p.pressure_or_distance.clamp(0.0, 1.0) * 65535.0) as u16
|
||||||
|
};
|
||||||
|
s.distance = 0;
|
||||||
|
}
|
||||||
|
LI_TOUCH_EVENT_HOVER => {
|
||||||
|
self.saw_hover = true;
|
||||||
|
s.state |= PEN_IN_RANGE;
|
||||||
|
// Hovering: pressureOrDistance is distance, 1.0 = farthest.
|
||||||
|
s.distance = (p.pressure_or_distance.clamp(0.0, 1.0) * 65534.0) as u16;
|
||||||
|
}
|
||||||
|
LI_TOUCH_EVENT_UP => {
|
||||||
|
// A hover-capable client keeps proximity after lift (its HOVER/HOVER_LEAVE
|
||||||
|
// events own the exit); one that never hovers would otherwise park the pen
|
||||||
|
// in proximity forever, so lift = leave for those.
|
||||||
|
if self.saw_hover {
|
||||||
|
s.state |= PEN_IN_RANGE;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LI_TOUCH_EVENT_HOVER_LEAVE | LI_TOUCH_EVENT_CANCEL | LI_TOUCH_EVENT_CANCEL_ALL => {
|
||||||
|
s.state &= !(PEN_BARREL1 | PEN_BARREL2); // out of range releases buttons too
|
||||||
|
}
|
||||||
|
_ => return None, // unknown future event type — drop, never guess
|
||||||
|
}
|
||||||
|
Some(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_touch(&mut self, t: &SsTouch, mut sink: impl FnMut(InputEvent)) {
|
||||||
|
let ev = |kind: InputKind, id: u32, x: f32, y: f32| InputEvent {
|
||||||
|
kind,
|
||||||
|
_pad: [0; 3],
|
||||||
|
code: id,
|
||||||
|
x: (x.clamp(0.0, 1.0) * TOUCH_SURFACE as f32) as i32,
|
||||||
|
y: (y.clamp(0.0, 1.0) * TOUCH_SURFACE as f32) as i32,
|
||||||
|
flags: (TOUCH_SURFACE << 16) | TOUCH_SURFACE,
|
||||||
|
};
|
||||||
|
match t.event_type {
|
||||||
|
LI_TOUCH_EVENT_DOWN => {
|
||||||
|
if !self.touch_ids.contains(&t.pointer_id) && self.touch_ids.len() < MAX_TOUCH_IDS {
|
||||||
|
self.touch_ids.push(t.pointer_id);
|
||||||
|
}
|
||||||
|
sink(ev(InputKind::TouchDown, t.pointer_id, t.x, t.y));
|
||||||
|
}
|
||||||
|
LI_TOUCH_EVENT_MOVE => sink(ev(InputKind::TouchMove, t.pointer_id, t.x, t.y)),
|
||||||
|
LI_TOUCH_EVENT_UP | LI_TOUCH_EVENT_CANCEL => {
|
||||||
|
self.touch_ids.retain(|&id| id != t.pointer_id);
|
||||||
|
sink(ev(InputKind::TouchUp, t.pointer_id, t.x, t.y));
|
||||||
|
}
|
||||||
|
LI_TOUCH_EVENT_CANCEL_ALL => {
|
||||||
|
for id in self.touch_ids.drain(..) {
|
||||||
|
sink(ev(InputKind::TouchUp, id, 0.0, 0.0));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Touch hover has no wire representation (and BUTTON_ONLY is pen-only in
|
||||||
|
// practice) — nothing to forward.
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn pen(event_type: u8, x: f32, y: f32, pod: f32) -> SsPen {
|
||||||
|
SsPen {
|
||||||
|
event_type,
|
||||||
|
tool: LI_TOOL_TYPE_PEN,
|
||||||
|
buttons: 0,
|
||||||
|
x,
|
||||||
|
y,
|
||||||
|
pressure_or_distance: pod,
|
||||||
|
rotation: 270,
|
||||||
|
tilt: 40,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pen_down_move_up_maps_to_state_full_samples() {
|
||||||
|
let mut g = GsPointer::new();
|
||||||
|
let s = g
|
||||||
|
.pen_sample(&pen(LI_TOUCH_EVENT_DOWN, 0.25, 0.5, 0.5))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(s.state, PEN_IN_RANGE | PEN_TOUCHING);
|
||||||
|
assert_eq!(s.pressure, 32767);
|
||||||
|
assert_eq!((s.tilt_deg, s.azimuth_deg), (40, 270));
|
||||||
|
assert_eq!(s.roll_deg, PEN_ANGLE_UNKNOWN);
|
||||||
|
// Unknown contact pressure (0.0) must still ink — binary-stylus full scale.
|
||||||
|
let s = g
|
||||||
|
.pen_sample(&pen(LI_TOUCH_EVENT_MOVE, 0.3, 0.5, 0.0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(s.pressure, u16::MAX);
|
||||||
|
// No hover ever seen: UP = fully out of range (no parked proximity).
|
||||||
|
let s = g
|
||||||
|
.pen_sample(&pen(LI_TOUCH_EVENT_UP, 0.3, 0.5, 0.0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(s.state, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn hover_capable_client_keeps_proximity_on_lift() {
|
||||||
|
let mut g = GsPointer::new();
|
||||||
|
let s = g
|
||||||
|
.pen_sample(&pen(LI_TOUCH_EVENT_HOVER, 0.2, 0.2, 0.5))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(s.state, PEN_IN_RANGE);
|
||||||
|
assert_eq!(s.distance, 32767);
|
||||||
|
let s = g
|
||||||
|
.pen_sample(&pen(LI_TOUCH_EVENT_UP, 0.2, 0.2, 0.0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(s.state, PEN_IN_RANGE);
|
||||||
|
// HOVER_LEAVE exits.
|
||||||
|
let s = g
|
||||||
|
.pen_sample(&pen(LI_TOUCH_EVENT_HOVER_LEAVE, 0.2, 0.2, 0.0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(s.state, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn button_only_merges_over_last_state() {
|
||||||
|
let mut g = GsPointer::new();
|
||||||
|
// BUTTON_ONLY before any positioned event: nothing to merge over.
|
||||||
|
let mut b = pen(LI_TOUCH_EVENT_BUTTON_ONLY, 0.0, 0.0, 0.0);
|
||||||
|
b.buttons = LI_PEN_BUTTON_PRIMARY;
|
||||||
|
assert!(g.pen_sample(&b).is_none());
|
||||||
|
// After a DOWN is applied (through the real tracker path minus the device), the
|
||||||
|
// button packet reuses the held position/pressure.
|
||||||
|
g.apply_pen(&pen(LI_TOUCH_EVENT_DOWN, 0.4, 0.6, 0.8));
|
||||||
|
let s = g.pen_sample(&b).unwrap();
|
||||||
|
assert_eq!(s.state & (PEN_BARREL1 | PEN_BARREL2), PEN_BARREL1);
|
||||||
|
assert_eq!(s.state & PEN_TOUCHING, PEN_TOUCHING);
|
||||||
|
assert!((s.x - 0.4).abs() < 1e-6);
|
||||||
|
assert_eq!(s.pressure, (0.8f32 * 65535.0) as u16);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn eraser_and_unknown_tools_map() {
|
||||||
|
let mut g = GsPointer::new();
|
||||||
|
let mut p = pen(LI_TOUCH_EVENT_DOWN, 0.5, 0.5, 1.0);
|
||||||
|
p.tool = LI_TOOL_TYPE_ERASER;
|
||||||
|
assert_eq!(g.pen_sample(&p).unwrap().tool, PenTool::Eraser);
|
||||||
|
p.tool = 0x7F;
|
||||||
|
assert_eq!(g.pen_sample(&p).unwrap().tool, PenTool::Unknown);
|
||||||
|
// Unknown sentinel angles pass through as our sentinels.
|
||||||
|
p.rotation = LI_ROT_UNKNOWN;
|
||||||
|
p.tilt = LI_TILT_UNKNOWN;
|
||||||
|
let s = g.pen_sample(&p).unwrap();
|
||||||
|
assert_eq!(s.azimuth_deg, PEN_ANGLE_UNKNOWN);
|
||||||
|
assert_eq!(s.tilt_deg, PEN_TILT_UNKNOWN);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn touch_forwards_wire_touch_and_cancel_all_replays_ups() {
|
||||||
|
let mut g = GsPointer::new();
|
||||||
|
let mut got: Vec<InputEvent> = Vec::new();
|
||||||
|
let t = |event_type, pointer_id, x: f32| SsTouch {
|
||||||
|
event_type,
|
||||||
|
rotation: 0,
|
||||||
|
pointer_id,
|
||||||
|
x,
|
||||||
|
y: 0.5,
|
||||||
|
pressure_or_distance: 0.5,
|
||||||
|
};
|
||||||
|
g.apply_touch(&t(LI_TOUCH_EVENT_DOWN, 7, 0.5), |e| got.push(e));
|
||||||
|
g.apply_touch(&t(LI_TOUCH_EVENT_DOWN, 9, 0.25), |e| got.push(e));
|
||||||
|
g.apply_touch(&t(LI_TOUCH_EVENT_MOVE, 7, 0.75), |e| got.push(e));
|
||||||
|
assert_eq!(got.len(), 3);
|
||||||
|
assert_eq!(got[0].kind, InputKind::TouchDown);
|
||||||
|
assert_eq!(got[0].code, 7);
|
||||||
|
assert_eq!(got[0].x, 32767);
|
||||||
|
assert_eq!(got[0].flags, (65535 << 16) | 65535);
|
||||||
|
assert_eq!(got[2].kind, InputKind::TouchMove);
|
||||||
|
assert_eq!(got[2].x, 49151);
|
||||||
|
// CANCEL_ALL lifts every tracked contact.
|
||||||
|
got.clear();
|
||||||
|
g.apply_touch(&t(LI_TOUCH_EVENT_CANCEL_ALL, 0, 0.0), |e| got.push(e));
|
||||||
|
assert_eq!(got.len(), 2);
|
||||||
|
assert!(got.iter().all(|e| e.kind == InputKind::TouchUp));
|
||||||
|
let mut ids: Vec<u32> = got.iter().map(|e| e.code).collect();
|
||||||
|
ids.sort_unstable();
|
||||||
|
assert_eq!(ids, vec![7, 9]);
|
||||||
|
// And the set is empty now — a second CANCEL_ALL is a no-op.
|
||||||
|
got.clear();
|
||||||
|
g.apply_touch(&t(LI_TOUCH_EVENT_CANCEL_ALL, 0, 0.0), |e| got.push(e));
|
||||||
|
assert!(got.is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -309,12 +309,26 @@ fn handle_request(req: &Request, state: &AppState, peer: Option<SocketAddr>) ->
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// moonlight-common-c `LI_FF_PEN_TOUCH_EVENTS`: with this featureFlags bit set, Moonlight
|
||||||
|
/// clients send native `SS_PEN`/`SS_TOUCH` events (an iPad's Apple Pencil included) instead
|
||||||
|
/// of synthesizing mouse input client-side.
|
||||||
|
const SS_FF_PEN_TOUCH_EVENTS: u32 = 0x01;
|
||||||
|
|
||||||
/// Host capability SDP returned by DESCRIBE. Advertises HEVC + AV1 and no encryption
|
/// Host capability SDP returned by DESCRIBE. Advertises HEVC + AV1 and no encryption
|
||||||
/// (plaintext streams for now; P1.5 adds the negotiated AES paths).
|
/// (plaintext streams for now; P1.5 adds the negotiated AES paths).
|
||||||
fn describe_sdp() -> String {
|
fn describe_sdp() -> String {
|
||||||
|
// Pen/touch events are advertised only where we can actually inject them (Linux with
|
||||||
|
// uinput — the same gate as HOST_CAP_PEN; design/pen-tablet-input.md §4). Elsewhere the
|
||||||
|
// flag stays 0 so Moonlight keeps its client-side mouse emulation for touch, exactly as
|
||||||
|
// today. PUNKTFUNK_PEN=0 clears it (the operator kill-switch inside pen_supported).
|
||||||
|
let feature_flags: u32 = if crate::inject::pen_supported() {
|
||||||
|
SS_FF_PEN_TOUCH_EVENTS
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
// Line-oriented a=key:value, matching what moonlight-common-c scans for.
|
// Line-oriented a=key:value, matching what moonlight-common-c scans for.
|
||||||
let mut lines: Vec<String> = vec![
|
let mut lines: Vec<String> = vec![
|
||||||
"a=x-ss-general.featureFlags:0".into(),
|
format!("a=x-ss-general.featureFlags:{feature_flags}"),
|
||||||
"a=x-ss-general.encryptionSupported:0".into(),
|
"a=x-ss-general.encryptionSupported:0".into(),
|
||||||
"a=x-ss-general.encryptionRequested:0".into(),
|
"a=x-ss-general.encryptionRequested:0".into(),
|
||||||
"sprop-parameter-sets=AAAAAU".into(), // HEVC capability indicator
|
"sprop-parameter-sets=AAAAAU".into(), // HEVC capability indicator
|
||||||
|
|||||||
Reference in New Issue
Block a user