feat(gamestream): pen P2 — SS_PEN/SS_TOUCH ingest, featureFlags advertised on capable hosts
The Moonlight leg of design/pen-tablet-input.md §4: DESCRIBE now advertises SS_FF_PEN_TOUCH_EVENTS wherever pen injection exists (Linux+uinput, same gate as HOST_CAP_PEN — elsewhere the flag stays 0 and Moonlight keeps client-side mouse emulation exactly as today). SS_PEN packets merge over last-known state into state-full PenSamples (BUTTON_ONLY per spec carries no position; unknown contact pressure 0.0 inks at full scale; UP keeps proximity only for clients that demonstrated hover) and drive the same PenTracker→VirtualPen chain as the native plane — Moonlight iOS + Apple Pencil now exercises the full injection path end to end. SS_TOUCH forwards as wire touch on a synthetic 65535² surface with CANCEL_ALL replayed per tracked contact. No stroke timeout on this plane: ENet is ordered/reliable and Moonlight doesn't heartbeat a stationary pen. Packet layouts pinned against moonlight-common-c Input.h/Limelight.h upstream. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -71,6 +71,9 @@ pub fn spawn(state: Arc<AppState>) -> Result<()> {
|
||||
// from `seq`, so a per-message-type counter would reuse (key, nonce) pairs across
|
||||
// message types in the host direction.
|
||||
let mut pads = GamepadManager::new();
|
||||
// Pen/touch translator (SS_PEN/SS_TOUCH → virtual tablet / wire touch). Sent only
|
||||
// by clients that saw our SS_FF_PEN_TOUCH_EVENTS feature flag (rtsp.rs).
|
||||
let mut pointer = super::pen::GsPointer::new();
|
||||
let mut host_seq: u32 = 0;
|
||||
// One-shot latch for the HDR-mode control message (0x010e); re-armed on Disconnect.
|
||||
let mut hdr_sent = false;
|
||||
@@ -90,8 +93,10 @@ pub fn spawn(state: Arc<AppState>) -> Result<()> {
|
||||
peer = None;
|
||||
// Re-arm the HDR-mode signal for the next connection.
|
||||
hdr_sent = false;
|
||||
// Unplug the session's virtual pads.
|
||||
// Unplug the session's virtual pads + tablet (destroying the
|
||||
// uinput pen releases any held tool/tip kernel-side).
|
||||
pads = GamepadManager::new();
|
||||
pointer = super::pen::GsPointer::new();
|
||||
}
|
||||
Event::Receive {
|
||||
channel_id, packet, ..
|
||||
@@ -104,6 +109,7 @@ pub fn spawn(state: Arc<AppState>) -> Result<()> {
|
||||
&mut decrypt_fails,
|
||||
&inj_tx,
|
||||
&mut pads,
|
||||
&mut pointer,
|
||||
);
|
||||
}
|
||||
},
|
||||
@@ -196,6 +202,7 @@ fn decode_rfi_range(pt: &[u8]) -> Option<(i64, i64)> {
|
||||
|
||||
/// Handle one received control packet: decrypt it (learning the GCM scheme on the first one),
|
||||
/// decode any input event, and inject it into the host session.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn on_receive(
|
||||
state: &AppState,
|
||||
_channel_id: u8,
|
||||
@@ -204,6 +211,7 @@ fn on_receive(
|
||||
decrypt_fails: &mut u64,
|
||||
inj_tx: &Sender<InputEvent>,
|
||||
pads: &mut GamepadManager,
|
||||
pointer: &mut super::pen::GsPointer,
|
||||
) {
|
||||
let Some(key) = state.launch.lock().unwrap().map(|s| s.gcm_key) else {
|
||||
return; // control traffic before /launch — no key yet
|
||||
@@ -274,6 +282,15 @@ fn on_receive(
|
||||
return;
|
||||
}
|
||||
|
||||
// Pen/touch extension events (Moonlight sends them only after seeing our feature flag):
|
||||
// pen drives this session's virtual tablet; touch forwards as ordinary wire touches.
|
||||
if let Some(p) = super::input::decode_pointer(&pt) {
|
||||
pointer.apply(&p, |ev| {
|
||||
let _ = inj_tx.send(ev);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
let events = super::input::decode(&pt);
|
||||
if events.is_empty() {
|
||||
return; // keepalive / QoS / unhandled input kind
|
||||
|
||||
@@ -25,6 +25,8 @@ const MAGIC_MOUSE_BTN_UP: u32 = 0x09;
|
||||
const MAGIC_SCROLL_GEN5: u32 = 0x0A;
|
||||
const MAGIC_UTF8: u32 = 0x17;
|
||||
const MAGIC_HSCROLL: u32 = 0x5500_0001;
|
||||
const MAGIC_SS_TOUCH: u32 = 0x5500_0002;
|
||||
const MAGIC_SS_PEN: u32 = 0x5500_0003;
|
||||
|
||||
/// `code` value marking a [`InputKind::MouseScroll`] as horizontal (vs `0` = vertical).
|
||||
pub const SCROLL_HORIZONTAL: u32 = 1;
|
||||
@@ -111,6 +113,84 @@ fn decode_input_packet(p: &[u8]) -> Option<InputEvent> {
|
||||
})
|
||||
}
|
||||
|
||||
/// One decoded `SS_PEN_PACKET` body (moonlight-common-c `Input.h`; all fields little-endian,
|
||||
/// coordinates/pressure as normalized floats). Semantics — `pressure_or_distance` is pressure
|
||||
/// (0..1, 0 = unknown) for contact events and hover distance (1 = farthest) for hover;
|
||||
/// `rotation` is the tilt azimuth (0..360, `0xFFFF` unknown); `tilt` is degrees from the
|
||||
/// surface normal (0..90, `0xFF` unknown) — are interpreted in [`super::pen`].
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct SsPen {
|
||||
pub event_type: u8,
|
||||
pub tool: u8,
|
||||
pub buttons: u8,
|
||||
pub x: f32,
|
||||
pub y: f32,
|
||||
pub pressure_or_distance: f32,
|
||||
pub rotation: u16,
|
||||
pub tilt: u8,
|
||||
}
|
||||
|
||||
/// One decoded `SS_TOUCH_PACKET` body (same conventions as [`SsPen`]; contact-area fields are
|
||||
/// not carried — the wire touch kinds have nowhere to put them yet).
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct SsTouch {
|
||||
pub event_type: u8,
|
||||
pub rotation: u16,
|
||||
pub pointer_id: u32,
|
||||
pub x: f32,
|
||||
pub y: f32,
|
||||
pub pressure_or_distance: f32,
|
||||
}
|
||||
|
||||
/// A Sunshine-extension pointer event (sent only after we advertise
|
||||
/// `SS_FF_PEN_TOUCH_EVENTS` — see [`super::rtsp`]).
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub enum SsPointer {
|
||||
Pen(SsPen),
|
||||
Touch(SsTouch),
|
||||
}
|
||||
|
||||
/// Decode a control plaintext into a pen/touch pointer event, or `None` for every other
|
||||
/// message (the caller then falls through to [`decode`]). Bounds- and sanity-checked like the
|
||||
/// rest of the plane: short bodies and non-finite floats (a forged NaN must never reach the
|
||||
/// injectors' scaling) drop the packet whole.
|
||||
pub fn decode_pointer(plaintext: &[u8]) -> Option<SsPointer> {
|
||||
if plaintext.len() < 12 || u16::from_le_bytes([plaintext[0], plaintext[1]]) != INPUT_DATA_TYPE {
|
||||
return None;
|
||||
}
|
||||
let p = &plaintext[4..];
|
||||
let magic = u32::from_le_bytes([p[4], p[5], p[6], p[7]]);
|
||||
let b = &p[8..];
|
||||
let f32at = |o: usize| -> Option<f32> {
|
||||
let v = f32::from_le_bytes([*b.get(o)?, *b.get(o + 1)?, *b.get(o + 2)?, *b.get(o + 3)?]);
|
||||
v.is_finite().then_some(v)
|
||||
};
|
||||
match magic {
|
||||
// eventType, zero[1], rotation u16, pointerId u32, x, y, pressureOrDistance, areas.
|
||||
MAGIC_SS_TOUCH => Some(SsPointer::Touch(SsTouch {
|
||||
event_type: *b.first()?,
|
||||
rotation: u16::from_le_bytes([*b.get(2)?, *b.get(3)?]),
|
||||
pointer_id: u32::from_le_bytes([*b.get(4)?, *b.get(5)?, *b.get(6)?, *b.get(7)?]),
|
||||
x: f32at(8)?,
|
||||
y: f32at(12)?,
|
||||
pressure_or_distance: f32at(16)?,
|
||||
})),
|
||||
// eventType, toolType, penButtons, zero[1], x, y, pressureOrDistance, rotation u16,
|
||||
// tilt, zero2[1], areas.
|
||||
MAGIC_SS_PEN => Some(SsPointer::Pen(SsPen {
|
||||
event_type: *b.first()?,
|
||||
tool: *b.get(1)?,
|
||||
buttons: *b.get(2)?,
|
||||
x: f32at(4)?,
|
||||
y: f32at(8)?,
|
||||
pressure_or_distance: f32at(12)?,
|
||||
rotation: u16::from_le_bytes([*b.get(16)?, *b.get(17)?]),
|
||||
tilt: *b.get(18)?,
|
||||
})),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn ev(kind: InputKind, code: u32, x: i32, y: i32, flags: u32) -> InputEvent {
|
||||
InputEvent {
|
||||
kind,
|
||||
@@ -176,6 +256,68 @@ mod tests {
|
||||
assert!(decode(&bad).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_ss_pen_and_touch_golden_bytes() {
|
||||
// SS_PEN body per Input.h: DOWN, pen tool, primary button, x=0.5 y=0.25,
|
||||
// pressure=0.75, rotation=180, tilt=45, then contact areas (present but ignored).
|
||||
let mut body = vec![0x01, 0x01, 0x01, 0x00];
|
||||
for f in [0.5f32, 0.25, 0.75] {
|
||||
body.extend_from_slice(&f.to_le_bytes());
|
||||
}
|
||||
body.extend_from_slice(&180u16.to_le_bytes());
|
||||
body.extend_from_slice(&[45, 0x00]);
|
||||
for f in [0.0f32, 0.0] {
|
||||
body.extend_from_slice(&f.to_le_bytes());
|
||||
}
|
||||
let pt = wrap(0x5500_0003, &body);
|
||||
assert_eq!(
|
||||
decode_pointer(&pt),
|
||||
Some(SsPointer::Pen(SsPen {
|
||||
event_type: 0x01,
|
||||
tool: 0x01,
|
||||
buttons: 0x01,
|
||||
x: 0.5,
|
||||
y: 0.25,
|
||||
pressure_or_distance: 0.75,
|
||||
rotation: 180,
|
||||
tilt: 45,
|
||||
}))
|
||||
);
|
||||
// A pen packet is invisible to the classic decoder (no misparse as mouse/key).
|
||||
assert!(decode(&pt).is_empty());
|
||||
|
||||
// SS_TOUCH body: MOVE, rotation unknown, pointerId 42, x=1.0 y=0.0, pressure 1.0.
|
||||
let mut body = vec![0x03, 0x00];
|
||||
body.extend_from_slice(&0xFFFFu16.to_le_bytes());
|
||||
body.extend_from_slice(&42u32.to_le_bytes());
|
||||
for f in [1.0f32, 0.0, 1.0, 0.0, 0.0] {
|
||||
body.extend_from_slice(&f.to_le_bytes());
|
||||
}
|
||||
let pt = wrap(0x5500_0002, &body);
|
||||
assert_eq!(
|
||||
decode_pointer(&pt),
|
||||
Some(SsPointer::Touch(SsTouch {
|
||||
event_type: 0x03,
|
||||
rotation: 0xFFFF,
|
||||
pointer_id: 42,
|
||||
x: 1.0,
|
||||
y: 0.0,
|
||||
pressure_or_distance: 1.0,
|
||||
}))
|
||||
);
|
||||
|
||||
// Truncated bodies and forged NaN coordinates drop the packet whole.
|
||||
assert_eq!(decode_pointer(&pt[..pt.len() - 18]), None);
|
||||
let mut nan = body.clone();
|
||||
nan[8..12].copy_from_slice(&f32::NAN.to_le_bytes());
|
||||
assert_eq!(decode_pointer(&wrap(0x5500_0002, &nan)), None);
|
||||
// Non-pointer magics fall through to the classic decoder.
|
||||
assert_eq!(
|
||||
decode_pointer(&wrap(MAGIC_MOUSE_REL_GEN5, &[0, 0, 0, 0])),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_non_input_type() {
|
||||
let mut pt = vec![0x00, 0x02]; // type 0x0200 (keepalive)
|
||||
|
||||
@@ -18,6 +18,8 @@ mod input;
|
||||
mod mdns;
|
||||
mod nvhttp;
|
||||
mod pairing;
|
||||
/// Moonlight `SS_PEN`/`SS_TOUCH` → the native pen model / wire touch (design/pen-tablet-input.md §4).
|
||||
mod pen;
|
||||
mod rtsp;
|
||||
mod serverinfo;
|
||||
mod stream;
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
//! Translate Moonlight's edge-based `SS_PEN` / `SS_TOUCH` events (design/pen-tablet-input.md
|
||||
//! §4) into punktfunk's state-full pen model and wire-touch events.
|
||||
//!
|
||||
//! Pen: each packet becomes one complete [`PenSample`] (packet fields merged over the last
|
||||
//! known state, since e.g. `BUTTON_ONLY` carries no position by spec), fed through the same
|
||||
//! [`PenTracker`] → [`VirtualPen`] chain the native plane uses — so Moonlight iOS with an
|
||||
//! Apple Pencil exercises the exact injection path our own clients will.
|
||||
//!
|
||||
//! No stroke timeout here, deliberately: the native plane's 200 ms failsafe exists because
|
||||
//! datagrams are lossy and clients heartbeat; Moonlight's control stream is ordered/reliable
|
||||
//! ENet and its clients do NOT heartbeat a stationary pen — a timeout would lift a held
|
||||
//! stroke. Lost-client cleanup is the ENet disconnect (the control loop re-news this
|
||||
//! translator, and destroying the uinput device releases everything kernel-side).
|
||||
//!
|
||||
//! Touch: forwarded as the ordinary wire touch kinds (`TouchDown/Move/Up`, normalized onto a
|
||||
//! synthetic 65535² surface) through the injector service — pressure/area are dropped v1
|
||||
//! (the wire touch has no field for them; `RICH_TOUCH` is the design's §8 upgrade).
|
||||
|
||||
use super::input::{SsPen, SsPointer, SsTouch};
|
||||
use punktfunk_core::input::{InputEvent, InputKind};
|
||||
use punktfunk_core::quic::{
|
||||
PenSample, PenTool, PenTracker, PenTransition, PEN_ANGLE_UNKNOWN, PEN_BARREL1, PEN_BARREL2,
|
||||
PEN_IN_RANGE, PEN_TILT_UNKNOWN, PEN_TOUCHING,
|
||||
};
|
||||
|
||||
// moonlight-common-c Limelight.h event/tool/button vocabulary.
|
||||
const LI_TOUCH_EVENT_HOVER: u8 = 0x00;
|
||||
const LI_TOUCH_EVENT_DOWN: u8 = 0x01;
|
||||
const LI_TOUCH_EVENT_UP: u8 = 0x02;
|
||||
const LI_TOUCH_EVENT_MOVE: u8 = 0x03;
|
||||
const LI_TOUCH_EVENT_CANCEL: u8 = 0x04;
|
||||
const LI_TOUCH_EVENT_BUTTON_ONLY: u8 = 0x05;
|
||||
const LI_TOUCH_EVENT_HOVER_LEAVE: u8 = 0x06;
|
||||
const LI_TOUCH_EVENT_CANCEL_ALL: u8 = 0x07;
|
||||
const LI_TOOL_TYPE_PEN: u8 = 0x01;
|
||||
const LI_TOOL_TYPE_ERASER: u8 = 0x02;
|
||||
const LI_PEN_BUTTON_PRIMARY: u8 = 0x01;
|
||||
const LI_PEN_BUTTON_SECONDARY: u8 = 0x02;
|
||||
const LI_ROT_UNKNOWN: u16 = 0xFFFF;
|
||||
const LI_TILT_UNKNOWN: u8 = 0xFF;
|
||||
|
||||
/// Synthetic reference extent for forwarded touches: coordinates arrive normalized, so map
|
||||
/// them onto a full-range surface and let the injector rescale to its output (the
|
||||
/// `MouseMoveAbs`/touch `flags = (w << 16) | h` contract).
|
||||
const TOUCH_SURFACE: u32 = 65535;
|
||||
|
||||
/// Most concurrently-tracked touch pointers (for `CANCEL_ALL` replay). A flood of distinct
|
||||
/// ids can't grow memory — untracked ids still forward down/up verbatim, they just miss a
|
||||
/// later `CANCEL_ALL` (which a real client never has more than ~10 contacts for anyway).
|
||||
const MAX_TOUCH_IDS: usize = 32;
|
||||
|
||||
/// Per-GameStream-session pen + touch translator. Owned by the control loop next to the
|
||||
/// gamepad manager; re-created on client disconnect (the dropped [`VirtualPen`] destroys the
|
||||
/// uinput device, which releases any held tool/tip kernel-side).
|
||||
pub struct GsPointer {
|
||||
tracker: PenTracker,
|
||||
dev: Option<crate::inject::pen::VirtualPen>,
|
||||
create_failed: bool,
|
||||
seq: u16,
|
||||
out: Vec<PenTransition>,
|
||||
/// Last complete pen state — the merge base for packets that omit fields
|
||||
/// (`BUTTON_ONLY` ignores x/y/pressure/tilt/rotation by spec).
|
||||
last: PenSample,
|
||||
/// This client sends hover events, so an `UP` means "back to hover" rather than "gone" —
|
||||
/// clients without hover support get proximity-out on lift instead of a pen parked
|
||||
/// forever at the last point.
|
||||
saw_hover: bool,
|
||||
/// Active forwarded touch ids, for `CANCEL_ALL` replay as per-id ups.
|
||||
touch_ids: Vec<u32>,
|
||||
}
|
||||
|
||||
impl GsPointer {
|
||||
pub fn new() -> GsPointer {
|
||||
GsPointer {
|
||||
tracker: PenTracker::default(),
|
||||
dev: None,
|
||||
create_failed: false,
|
||||
seq: 0,
|
||||
out: Vec::new(),
|
||||
last: PenSample::default(),
|
||||
saw_hover: false,
|
||||
touch_ids: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply one decoded pointer packet. Touch events forward through `sink` (the injector
|
||||
/// channel); pen events drive this session's virtual tablet.
|
||||
pub fn apply(&mut self, p: &SsPointer, sink: impl FnMut(InputEvent)) {
|
||||
match p {
|
||||
SsPointer::Pen(pen) => self.apply_pen(pen),
|
||||
SsPointer::Touch(touch) => self.apply_touch(touch, sink),
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_pen(&mut self, p: &SsPen) {
|
||||
let Some(sample) = self.pen_sample(p) else {
|
||||
return;
|
||||
};
|
||||
self.last = sample;
|
||||
if self.dev.is_none() && !self.create_failed {
|
||||
match crate::inject::pen::VirtualPen::create() {
|
||||
Ok(d) => self.dev = Some(d),
|
||||
Err(e) => {
|
||||
self.create_failed = true;
|
||||
tracing::warn!(
|
||||
error = %format!("{e:#}"),
|
||||
"gamestream pen: virtual tablet creation failed — dropping pen input"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.out.clear();
|
||||
let batch = punktfunk_core::quic::PenBatch::new(self.seq, &[sample]);
|
||||
self.seq = self.seq.wrapping_add(1);
|
||||
self.tracker.apply(&batch, &mut self.out);
|
||||
if let Some(dev) = self.dev.as_mut() {
|
||||
dev.apply_batch(&self.out);
|
||||
}
|
||||
}
|
||||
|
||||
/// Merge one edge-based packet over the last known state into a complete sample.
|
||||
/// `None` = nothing to apply (a `BUTTON_ONLY` before any positioned event).
|
||||
fn pen_sample(&mut self, p: &SsPen) -> Option<PenSample> {
|
||||
let buttons = (if p.buttons & LI_PEN_BUTTON_PRIMARY != 0 {
|
||||
PEN_BARREL1
|
||||
} else {
|
||||
0
|
||||
}) | (if p.buttons & LI_PEN_BUTTON_SECONDARY != 0 {
|
||||
PEN_BARREL2
|
||||
} else {
|
||||
0
|
||||
});
|
||||
let tool = match p.tool {
|
||||
LI_TOOL_TYPE_PEN => PenTool::Pen,
|
||||
LI_TOOL_TYPE_ERASER => PenTool::Eraser,
|
||||
_ => PenTool::Unknown,
|
||||
};
|
||||
// BUTTON_ONLY ignores x/y/pressure/rotation/tilt by spec — reuse the last state.
|
||||
if p.event_type == LI_TOUCH_EVENT_BUTTON_ONLY {
|
||||
if !self.tracker.is_active() {
|
||||
return None;
|
||||
}
|
||||
let mut s = self.last;
|
||||
s.state = (s.state & !(PEN_BARREL1 | PEN_BARREL2)) | buttons;
|
||||
return Some(s);
|
||||
}
|
||||
|
||||
let mut s = PenSample {
|
||||
state: buttons,
|
||||
tool,
|
||||
x: p.x.clamp(0.0, 1.0),
|
||||
y: p.y.clamp(0.0, 1.0),
|
||||
dt_us: 0,
|
||||
roll_deg: PEN_ANGLE_UNKNOWN, // the GameStream wire has no barrel-roll axis
|
||||
azimuth_deg: if p.rotation == LI_ROT_UNKNOWN {
|
||||
PEN_ANGLE_UNKNOWN
|
||||
} else {
|
||||
p.rotation % 360
|
||||
},
|
||||
tilt_deg: if p.tilt == LI_TILT_UNKNOWN {
|
||||
PEN_TILT_UNKNOWN
|
||||
} else {
|
||||
p.tilt.min(90)
|
||||
},
|
||||
..PenSample::default()
|
||||
};
|
||||
match p.event_type {
|
||||
LI_TOUCH_EVENT_DOWN | LI_TOUCH_EVENT_MOVE => {
|
||||
s.state |= PEN_IN_RANGE | PEN_TOUCHING;
|
||||
// pressureOrDistance is pressure (0..1) in contact — and 0.0 means UNKNOWN
|
||||
// there, which must still ink: binary-stylus clients get full pressure.
|
||||
s.pressure = if p.pressure_or_distance <= 0.0 {
|
||||
u16::MAX
|
||||
} else {
|
||||
(p.pressure_or_distance.clamp(0.0, 1.0) * 65535.0) as u16
|
||||
};
|
||||
s.distance = 0;
|
||||
}
|
||||
LI_TOUCH_EVENT_HOVER => {
|
||||
self.saw_hover = true;
|
||||
s.state |= PEN_IN_RANGE;
|
||||
// Hovering: pressureOrDistance is distance, 1.0 = farthest.
|
||||
s.distance = (p.pressure_or_distance.clamp(0.0, 1.0) * 65534.0) as u16;
|
||||
}
|
||||
LI_TOUCH_EVENT_UP => {
|
||||
// A hover-capable client keeps proximity after lift (its HOVER/HOVER_LEAVE
|
||||
// events own the exit); one that never hovers would otherwise park the pen
|
||||
// in proximity forever, so lift = leave for those.
|
||||
if self.saw_hover {
|
||||
s.state |= PEN_IN_RANGE;
|
||||
}
|
||||
}
|
||||
LI_TOUCH_EVENT_HOVER_LEAVE | LI_TOUCH_EVENT_CANCEL | LI_TOUCH_EVENT_CANCEL_ALL => {
|
||||
s.state &= !(PEN_BARREL1 | PEN_BARREL2); // out of range releases buttons too
|
||||
}
|
||||
_ => return None, // unknown future event type — drop, never guess
|
||||
}
|
||||
Some(s)
|
||||
}
|
||||
|
||||
fn apply_touch(&mut self, t: &SsTouch, mut sink: impl FnMut(InputEvent)) {
|
||||
let ev = |kind: InputKind, id: u32, x: f32, y: f32| InputEvent {
|
||||
kind,
|
||||
_pad: [0; 3],
|
||||
code: id,
|
||||
x: (x.clamp(0.0, 1.0) * TOUCH_SURFACE as f32) as i32,
|
||||
y: (y.clamp(0.0, 1.0) * TOUCH_SURFACE as f32) as i32,
|
||||
flags: (TOUCH_SURFACE << 16) | TOUCH_SURFACE,
|
||||
};
|
||||
match t.event_type {
|
||||
LI_TOUCH_EVENT_DOWN => {
|
||||
if !self.touch_ids.contains(&t.pointer_id) && self.touch_ids.len() < MAX_TOUCH_IDS {
|
||||
self.touch_ids.push(t.pointer_id);
|
||||
}
|
||||
sink(ev(InputKind::TouchDown, t.pointer_id, t.x, t.y));
|
||||
}
|
||||
LI_TOUCH_EVENT_MOVE => sink(ev(InputKind::TouchMove, t.pointer_id, t.x, t.y)),
|
||||
LI_TOUCH_EVENT_UP | LI_TOUCH_EVENT_CANCEL => {
|
||||
self.touch_ids.retain(|&id| id != t.pointer_id);
|
||||
sink(ev(InputKind::TouchUp, t.pointer_id, t.x, t.y));
|
||||
}
|
||||
LI_TOUCH_EVENT_CANCEL_ALL => {
|
||||
for id in self.touch_ids.drain(..) {
|
||||
sink(ev(InputKind::TouchUp, id, 0.0, 0.0));
|
||||
}
|
||||
}
|
||||
// Touch hover has no wire representation (and BUTTON_ONLY is pen-only in
|
||||
// practice) — nothing to forward.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn pen(event_type: u8, x: f32, y: f32, pod: f32) -> SsPen {
|
||||
SsPen {
|
||||
event_type,
|
||||
tool: LI_TOOL_TYPE_PEN,
|
||||
buttons: 0,
|
||||
x,
|
||||
y,
|
||||
pressure_or_distance: pod,
|
||||
rotation: 270,
|
||||
tilt: 40,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pen_down_move_up_maps_to_state_full_samples() {
|
||||
let mut g = GsPointer::new();
|
||||
let s = g
|
||||
.pen_sample(&pen(LI_TOUCH_EVENT_DOWN, 0.25, 0.5, 0.5))
|
||||
.unwrap();
|
||||
assert_eq!(s.state, PEN_IN_RANGE | PEN_TOUCHING);
|
||||
assert_eq!(s.pressure, 32767);
|
||||
assert_eq!((s.tilt_deg, s.azimuth_deg), (40, 270));
|
||||
assert_eq!(s.roll_deg, PEN_ANGLE_UNKNOWN);
|
||||
// Unknown contact pressure (0.0) must still ink — binary-stylus full scale.
|
||||
let s = g
|
||||
.pen_sample(&pen(LI_TOUCH_EVENT_MOVE, 0.3, 0.5, 0.0))
|
||||
.unwrap();
|
||||
assert_eq!(s.pressure, u16::MAX);
|
||||
// No hover ever seen: UP = fully out of range (no parked proximity).
|
||||
let s = g
|
||||
.pen_sample(&pen(LI_TOUCH_EVENT_UP, 0.3, 0.5, 0.0))
|
||||
.unwrap();
|
||||
assert_eq!(s.state, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hover_capable_client_keeps_proximity_on_lift() {
|
||||
let mut g = GsPointer::new();
|
||||
let s = g
|
||||
.pen_sample(&pen(LI_TOUCH_EVENT_HOVER, 0.2, 0.2, 0.5))
|
||||
.unwrap();
|
||||
assert_eq!(s.state, PEN_IN_RANGE);
|
||||
assert_eq!(s.distance, 32767);
|
||||
let s = g
|
||||
.pen_sample(&pen(LI_TOUCH_EVENT_UP, 0.2, 0.2, 0.0))
|
||||
.unwrap();
|
||||
assert_eq!(s.state, PEN_IN_RANGE);
|
||||
// HOVER_LEAVE exits.
|
||||
let s = g
|
||||
.pen_sample(&pen(LI_TOUCH_EVENT_HOVER_LEAVE, 0.2, 0.2, 0.0))
|
||||
.unwrap();
|
||||
assert_eq!(s.state, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn button_only_merges_over_last_state() {
|
||||
let mut g = GsPointer::new();
|
||||
// BUTTON_ONLY before any positioned event: nothing to merge over.
|
||||
let mut b = pen(LI_TOUCH_EVENT_BUTTON_ONLY, 0.0, 0.0, 0.0);
|
||||
b.buttons = LI_PEN_BUTTON_PRIMARY;
|
||||
assert!(g.pen_sample(&b).is_none());
|
||||
// After a DOWN is applied (through the real tracker path minus the device), the
|
||||
// button packet reuses the held position/pressure.
|
||||
g.apply_pen(&pen(LI_TOUCH_EVENT_DOWN, 0.4, 0.6, 0.8));
|
||||
let s = g.pen_sample(&b).unwrap();
|
||||
assert_eq!(s.state & (PEN_BARREL1 | PEN_BARREL2), PEN_BARREL1);
|
||||
assert_eq!(s.state & PEN_TOUCHING, PEN_TOUCHING);
|
||||
assert!((s.x - 0.4).abs() < 1e-6);
|
||||
assert_eq!(s.pressure, (0.8f32 * 65535.0) as u16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eraser_and_unknown_tools_map() {
|
||||
let mut g = GsPointer::new();
|
||||
let mut p = pen(LI_TOUCH_EVENT_DOWN, 0.5, 0.5, 1.0);
|
||||
p.tool = LI_TOOL_TYPE_ERASER;
|
||||
assert_eq!(g.pen_sample(&p).unwrap().tool, PenTool::Eraser);
|
||||
p.tool = 0x7F;
|
||||
assert_eq!(g.pen_sample(&p).unwrap().tool, PenTool::Unknown);
|
||||
// Unknown sentinel angles pass through as our sentinels.
|
||||
p.rotation = LI_ROT_UNKNOWN;
|
||||
p.tilt = LI_TILT_UNKNOWN;
|
||||
let s = g.pen_sample(&p).unwrap();
|
||||
assert_eq!(s.azimuth_deg, PEN_ANGLE_UNKNOWN);
|
||||
assert_eq!(s.tilt_deg, PEN_TILT_UNKNOWN);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn touch_forwards_wire_touch_and_cancel_all_replays_ups() {
|
||||
let mut g = GsPointer::new();
|
||||
let mut got: Vec<InputEvent> = Vec::new();
|
||||
let t = |event_type, pointer_id, x: f32| SsTouch {
|
||||
event_type,
|
||||
rotation: 0,
|
||||
pointer_id,
|
||||
x,
|
||||
y: 0.5,
|
||||
pressure_or_distance: 0.5,
|
||||
};
|
||||
g.apply_touch(&t(LI_TOUCH_EVENT_DOWN, 7, 0.5), |e| got.push(e));
|
||||
g.apply_touch(&t(LI_TOUCH_EVENT_DOWN, 9, 0.25), |e| got.push(e));
|
||||
g.apply_touch(&t(LI_TOUCH_EVENT_MOVE, 7, 0.75), |e| got.push(e));
|
||||
assert_eq!(got.len(), 3);
|
||||
assert_eq!(got[0].kind, InputKind::TouchDown);
|
||||
assert_eq!(got[0].code, 7);
|
||||
assert_eq!(got[0].x, 32767);
|
||||
assert_eq!(got[0].flags, (65535 << 16) | 65535);
|
||||
assert_eq!(got[2].kind, InputKind::TouchMove);
|
||||
assert_eq!(got[2].x, 49151);
|
||||
// CANCEL_ALL lifts every tracked contact.
|
||||
got.clear();
|
||||
g.apply_touch(&t(LI_TOUCH_EVENT_CANCEL_ALL, 0, 0.0), |e| got.push(e));
|
||||
assert_eq!(got.len(), 2);
|
||||
assert!(got.iter().all(|e| e.kind == InputKind::TouchUp));
|
||||
let mut ids: Vec<u32> = got.iter().map(|e| e.code).collect();
|
||||
ids.sort_unstable();
|
||||
assert_eq!(ids, vec![7, 9]);
|
||||
// And the set is empty now — a second CANCEL_ALL is a no-op.
|
||||
got.clear();
|
||||
g.apply_touch(&t(LI_TOUCH_EVENT_CANCEL_ALL, 0, 0.0), |e| got.push(e));
|
||||
assert!(got.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -309,12 +309,26 @@ fn handle_request(req: &Request, state: &AppState, peer: Option<SocketAddr>) ->
|
||||
}
|
||||
}
|
||||
|
||||
/// moonlight-common-c `LI_FF_PEN_TOUCH_EVENTS`: with this featureFlags bit set, Moonlight
|
||||
/// clients send native `SS_PEN`/`SS_TOUCH` events (an iPad's Apple Pencil included) instead
|
||||
/// of synthesizing mouse input client-side.
|
||||
const SS_FF_PEN_TOUCH_EVENTS: u32 = 0x01;
|
||||
|
||||
/// Host capability SDP returned by DESCRIBE. Advertises HEVC + AV1 and no encryption
|
||||
/// (plaintext streams for now; P1.5 adds the negotiated AES paths).
|
||||
fn describe_sdp() -> String {
|
||||
// Pen/touch events are advertised only where we can actually inject them (Linux with
|
||||
// uinput — the same gate as HOST_CAP_PEN; design/pen-tablet-input.md §4). Elsewhere the
|
||||
// flag stays 0 so Moonlight keeps its client-side mouse emulation for touch, exactly as
|
||||
// today. PUNKTFUNK_PEN=0 clears it (the operator kill-switch inside pen_supported).
|
||||
let feature_flags: u32 = if crate::inject::pen_supported() {
|
||||
SS_FF_PEN_TOUCH_EVENTS
|
||||
} else {
|
||||
0
|
||||
};
|
||||
// Line-oriented a=key:value, matching what moonlight-common-c scans for.
|
||||
let mut lines: Vec<String> = vec![
|
||||
"a=x-ss-general.featureFlags:0".into(),
|
||||
format!("a=x-ss-general.featureFlags:{feature_flags}"),
|
||||
"a=x-ss-general.encryptionSupported:0".into(),
|
||||
"a=x-ss-general.encryptionRequested:0".into(),
|
||||
"sprop-parameter-sets=AAAAAU".into(), // HEVC capability indicator
|
||||
|
||||
Reference in New Issue
Block a user