fix(sdl/cursor): scale the forwarded pointer to the video fit
apple / swift (push) Successful in 1m30s
ci / web (push) Successful in 54s
ci / docs-site (push) Successful in 1m3s
android / android (push) Canceled after 1m27s
apple / screenshots (push) Canceled after 1m39s
arch / build-publish (push) Canceled after 1m29s
ci / rust (push) Canceled after 1m29s
ci / bench (push) Canceled after 1m17s
deb / build-publish (push) Canceled after 1m15s
deb / build-publish-host (push) Canceled after 27s
decky / build-publish (push) Canceled after 21s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 10s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Canceled after 0s
windows / build (aarch64-pc-windows-msvc) (push) Canceled after 0s
windows / build (x86_64-pc-windows-msvc) (push) Canceled after 0s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 15s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Canceled after 1m18s
docker / build-push (ci, ci/rust-ci-noble.Dockerfile, punktfunk-rust-ci-noble) (push) Successful in 15s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Canceled after 8s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Canceled after 4s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 18s
flatpak / build-publish (push) Canceled after 8s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 16s
docker / deploy-docs (push) Canceled after 0s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Canceled after 17s
apple / swift (push) Successful in 1m30s
ci / web (push) Successful in 54s
ci / docs-site (push) Successful in 1m3s
android / android (push) Canceled after 1m27s
apple / screenshots (push) Canceled after 1m39s
arch / build-publish (push) Canceled after 1m29s
ci / rust (push) Canceled after 1m29s
ci / bench (push) Canceled after 1m17s
deb / build-publish (push) Canceled after 1m15s
deb / build-publish-host (push) Canceled after 27s
decky / build-publish (push) Canceled after 21s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 10s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Canceled after 0s
windows / build (aarch64-pc-windows-msvc) (push) Canceled after 0s
windows / build (x86_64-pc-windows-msvc) (push) Canceled after 0s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 15s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Canceled after 1m18s
docker / build-push (ci, ci/rust-ci-noble.Dockerfile, punktfunk-rust-ci-noble) (push) Successful in 15s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Canceled after 8s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Canceled after 4s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 18s
flatpak / build-publish (push) Canceled after 8s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 16s
docker / deploy-docs (push) Canceled after 0s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Canceled after 17s
Parity with the Apple client (35c2c145): the SDL/Linux presenter built the
host-forwarded cursor as an OS cursor at its native bitmap size, so the
on-screen pointer tracked the HOST's display scaling — a 4K/high-DPI virtual
display forwards a 96 px bitmap that then rendered huge on the client.
SDL cursors are fixed-size from their surface (no draw-time scaling), so
cache the shapes RAW and resample each by the live video-fit factor
(min(window_px / mode) — the same aspect-fit `finger_to_content` uses) at
install time, rebuilding when the serial OR the fit changes (a resize). The
resample area-averages on premultiplied alpha so a transparent pixel's
colour can't bleed into the fringe. The pointer now stays sized to the
streamed desktop at any host scaling.
Not yet eyeballed on a Deck/Linux box — SDL's HiDPI cursor sizing is
version-dependent; validate on glass.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -5,10 +5,19 @@
|
||||
//! negotiated it (`HOST_CAP_CURSOR` in the Welcome — the host stopped compositing then) and
|
||||
//! only applied while the DESKTOP mouse model is engaged: under capture the pointer is
|
||||
//! relative-locked (SDL hides it) and games draw their own cursor in-frame.
|
||||
//!
|
||||
//! The host sends the bitmap in host-FRAMEBUFFER pixels, whose size tracks the host virtual
|
||||
//! display's DPI scaling (32 px at 100%, 96 px at 300%). Drawn 1:1 it balloons on a high-DPI
|
||||
//! host; instead we scale it by the SAME aspect-fit factor the video is drawn at
|
||||
//! (`min(window_px/mode)`), so the pointer stays sized to the streamed desktop at any host
|
||||
//! scaling. SDL cursors are fixed-size from their surface (no draw-time scaling), so we cache
|
||||
//! shapes RAW and resample per install — rebuilding when the serial OR the fit changes.
|
||||
|
||||
use punktfunk_core::client::NativeClient;
|
||||
use punktfunk_core::quic::{CursorState, HOST_CAP_CURSOR};
|
||||
use sdl3::mouse::{Cursor, MouseUtil, SystemCursor};
|
||||
use sdl3::pixels::PixelFormat;
|
||||
use sdl3::surface::Surface;
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -17,14 +26,27 @@ use std::time::Duration;
|
||||
/// on the reliable stream via the serial-miss path).
|
||||
const SHAPE_CACHE_MAX: usize = 64;
|
||||
|
||||
/// A forwarded cursor shape held RAW (host-framebuffer-pixel bytes + hotspot), so it can be
|
||||
/// rebuilt into a scaled OS cursor whenever the video-fit changes (a window resize). Caching a
|
||||
/// fixed-size `Cursor` instead would freeze the pointer at its build-time size.
|
||||
struct RawShape {
|
||||
rgba: Vec<u8>,
|
||||
w: u32,
|
||||
h: u32,
|
||||
hot_x: u32,
|
||||
hot_y: u32,
|
||||
}
|
||||
|
||||
pub struct CursorChannel {
|
||||
/// The Welcome carried `HOST_CAP_CURSOR` — the host forwards instead of compositing.
|
||||
negotiated: bool,
|
||||
/// Serial → built OS cursor. An SDL `Cursor` must outlive its `set()`, so the cache owns
|
||||
/// every shape ever applied this session (bounded by [`SHAPE_CACHE_MAX`]).
|
||||
shapes: HashMap<u32, Cursor>,
|
||||
/// The serial currently installed via `Cursor::set` (`None` = default/system cursor).
|
||||
installed: Option<u32>,
|
||||
/// Serial → raw forwarded shape (bounded by [`SHAPE_CACHE_MAX`]).
|
||||
shapes: HashMap<u32, RawShape>,
|
||||
/// The serial + fit scale the currently-installed OS cursor was built at (`None` =
|
||||
/// default/system cursor). A change in EITHER forces a rebuild.
|
||||
installed: Option<(u32, f32)>,
|
||||
/// Keeps the installed `Cursor` alive — SDL requires it to outlive its `set()`.
|
||||
installed_cursor: Option<Cursor>,
|
||||
/// Latest `0xD0` state (latest-wins across a drained batch).
|
||||
state: Option<CursorState>,
|
||||
}
|
||||
@@ -39,6 +61,7 @@ impl CursorChannel {
|
||||
negotiated,
|
||||
shapes: HashMap::new(),
|
||||
installed: None,
|
||||
installed_cursor: None,
|
||||
state: None,
|
||||
}
|
||||
}
|
||||
@@ -57,8 +80,16 @@ impl CursorChannel {
|
||||
/// Drain the two planes and apply the newest state — once per run-loop iteration.
|
||||
/// `desktop_active` = the desktop mouse model is engaged (captured + desktop): only then
|
||||
/// do we own the local cursor's shape/visibility; under capture SDL's relative mode owns
|
||||
/// it, and released the system cursor must look normal.
|
||||
pub fn pump(&mut self, connector: &NativeClient, mouse: &MouseUtil, desktop_active: bool) {
|
||||
/// it, and released the system cursor must look normal. `fit_scale` is host-framebuffer
|
||||
/// pixels → window pixels (the aspect-fit factor the video is drawn at); the shape is
|
||||
/// resampled by it so the pointer matches the streamed desktop at any host DPI.
|
||||
pub fn pump(
|
||||
&mut self,
|
||||
connector: &NativeClient,
|
||||
mouse: &MouseUtil,
|
||||
desktop_active: bool,
|
||||
fit_scale: f32,
|
||||
) {
|
||||
if !self.negotiated {
|
||||
return;
|
||||
}
|
||||
@@ -68,30 +99,25 @@ impl CursorChannel {
|
||||
self.shapes.clear();
|
||||
self.installed = None;
|
||||
}
|
||||
let mut data = shape.rgba;
|
||||
let built = sdl3::surface::Surface::from_data(
|
||||
&mut data,
|
||||
shape.w as u32,
|
||||
shape.h as u32,
|
||||
shape.w as u32 * 4,
|
||||
sdl3::pixels::PixelFormat::RGBA32,
|
||||
)
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|surf| {
|
||||
Cursor::from_surface(&surf, shape.hot_x as i32, shape.hot_y as i32)
|
||||
.map_err(|e| e.to_string())
|
||||
});
|
||||
match built {
|
||||
Ok(cursor) => {
|
||||
let (w, h) = (shape.w as u32, shape.h as u32);
|
||||
if w == 0 || h == 0 || shape.rgba.len() < (w * h * 4) as usize {
|
||||
tracing::warn!(w, h, "cursor shape malformed — ignored");
|
||||
continue;
|
||||
}
|
||||
// A re-sent serial replaces its entry; force re-install if it's current.
|
||||
if self.installed == Some(shape.serial) {
|
||||
if matches!(self.installed, Some((s, _)) if s == shape.serial) {
|
||||
self.installed = None;
|
||||
}
|
||||
self.shapes.insert(shape.serial, cursor);
|
||||
}
|
||||
Err(e) => tracing::warn!(error = %e, w = shape.w, h = shape.h,
|
||||
"cursor shape rejected by SDL — keeping the previous cursor"),
|
||||
}
|
||||
self.shapes.insert(
|
||||
shape.serial,
|
||||
RawShape {
|
||||
rgba: shape.rgba,
|
||||
w,
|
||||
h,
|
||||
hot_x: shape.hot_x as u32,
|
||||
hot_y: shape.hot_y as u32,
|
||||
},
|
||||
);
|
||||
}
|
||||
while let Ok(st) = connector.next_cursor_state(Duration::ZERO) {
|
||||
self.state = Some(st); // latest wins
|
||||
@@ -101,17 +127,25 @@ impl CursorChannel {
|
||||
// Capture mode / released: hand the cursor back to the system default so a
|
||||
// released pointer over the window doesn't wear the host's shape.
|
||||
if self.installed.take().is_some() {
|
||||
Cursor::from_system(SystemCursor::Arrow)
|
||||
.map(|c| c.set())
|
||||
.ok();
|
||||
if let Ok(c) = Cursor::from_system(SystemCursor::Arrow) {
|
||||
c.set();
|
||||
self.installed_cursor = Some(c); // keep it alive past set()
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
let Some(st) = self.state else { return };
|
||||
if st.visible() && self.installed != Some(st.serial) {
|
||||
if let Some(cursor) = self.shapes.get(&st.serial) {
|
||||
if st.visible() && self.installed != Some((st.serial, fit_scale)) {
|
||||
if let Some(shape) = self.shapes.get(&st.serial) {
|
||||
match build_scaled_cursor(shape, fit_scale) {
|
||||
Ok(cursor) => {
|
||||
cursor.set();
|
||||
self.installed = Some(st.serial);
|
||||
self.installed = Some((st.serial, fit_scale));
|
||||
self.installed_cursor = Some(cursor); // outlive set()
|
||||
}
|
||||
Err(e) => tracing::warn!(error = %e, w = shape.w, h = shape.h,
|
||||
"cursor shape rejected by SDL — keeping the previous cursor"),
|
||||
}
|
||||
}
|
||||
// Serial miss: the (reliable) shape hasn't landed yet — keep the previous
|
||||
// cursor for the RTT rather than flashing default.
|
||||
@@ -123,3 +157,76 @@ impl CursorChannel {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resample a raw shape by `fit_scale` and build an SDL color cursor from it. The hotspot scales
|
||||
/// with the bitmap so the click point stays true. `fit_scale <= 0` (or a degenerate result) is
|
||||
/// clamped so we always hand SDL a ≥1×1 surface.
|
||||
fn build_scaled_cursor(shape: &RawShape, fit_scale: f32) -> Result<Cursor, String> {
|
||||
let scale = if fit_scale.is_finite() && fit_scale > 0.0 {
|
||||
fit_scale
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
let dw = ((shape.w as f32 * scale).round() as u32).max(1);
|
||||
let dh = ((shape.h as f32 * scale).round() as u32).max(1);
|
||||
let hot_x = ((shape.hot_x as f32 * scale).round() as u32).min(dw - 1) as i32;
|
||||
let hot_y = ((shape.hot_y as f32 * scale).round() as u32).min(dh - 1) as i32;
|
||||
|
||||
if dw == shape.w && dh == shape.h {
|
||||
// 1:1 fit (mode == window) — no resample, feed the bytes straight through.
|
||||
let mut data = shape.rgba.clone();
|
||||
let surf = Surface::from_data(
|
||||
&mut data,
|
||||
shape.w,
|
||||
shape.h,
|
||||
shape.w * 4,
|
||||
PixelFormat::RGBA32,
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
return Cursor::from_surface(&surf, hot_x, hot_y).map_err(|e| e.to_string());
|
||||
}
|
||||
|
||||
let mut scaled = resample_rgba(&shape.rgba, shape.w, shape.h, dw, dh);
|
||||
let surf = Surface::from_data(&mut scaled, dw, dh, dw * 4, PixelFormat::RGBA32)
|
||||
.map_err(|e| e.to_string())?;
|
||||
Cursor::from_surface(&surf, hot_x, hot_y).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// Area-average resample of a straight-alpha RGBA bitmap `(sw×sh) → (dw×dh)`. Averaging is done on
|
||||
/// PREMULTIPLIED colour (weighting each source texel by its alpha) so transparent-pixel colour
|
||||
/// can't bleed into the fringe, then un-premultiplied back to straight alpha. Handles both down-
|
||||
/// and up-scale; for a cursor the common case is a downscale (high-DPI host → smaller pointer).
|
||||
fn resample_rgba(src: &[u8], sw: u32, sh: u32, dw: u32, dh: u32) -> Vec<u8> {
|
||||
let mut out = vec![0u8; (dw * dh * 4) as usize];
|
||||
let fx = sw as f32 / dw as f32;
|
||||
let fy = sh as f32 / dh as f32;
|
||||
for dy in 0..dh {
|
||||
let sy0 = (dy as f32 * fy).floor() as u32;
|
||||
let sy1 = (((dy + 1) as f32 * fy).ceil() as u32).clamp(sy0 + 1, sh);
|
||||
for dx in 0..dw {
|
||||
let sx0 = (dx as f32 * fx).floor() as u32;
|
||||
let sx1 = (((dx + 1) as f32 * fx).ceil() as u32).clamp(sx0 + 1, sw);
|
||||
let (mut r, mut g, mut b, mut a_sum, mut n) = (0f32, 0f32, 0f32, 0f32, 0f32);
|
||||
for sy in sy0..sy1 {
|
||||
for sx in sx0..sx1 {
|
||||
let i = ((sy * sw + sx) * 4) as usize;
|
||||
let a = src[i + 3] as f32 / 255.0;
|
||||
r += src[i] as f32 * a;
|
||||
g += src[i + 1] as f32 * a;
|
||||
b += src[i + 2] as f32 * a;
|
||||
a_sum += a;
|
||||
n += 1.0;
|
||||
}
|
||||
}
|
||||
let di = ((dy * dw + dx) * 4) as usize;
|
||||
if a_sum > 0.0 {
|
||||
out[di] = (r / a_sum).round().clamp(0.0, 255.0) as u8;
|
||||
out[di + 1] = (g / a_sum).round().clamp(0.0, 255.0) as u8;
|
||||
out[di + 2] = (b / a_sum).round().clamp(0.0, 255.0) as u8;
|
||||
out[di + 3] = (a_sum / n * 255.0).round().clamp(0.0, 255.0) as u8;
|
||||
}
|
||||
// else fully transparent — already zero-filled.
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
@@ -773,12 +773,20 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
|
||||
// Cursor channel (M2): drain forwarded shape/state and drive the local OS cursor —
|
||||
// only meaningful in the desktop mouse model (capture's relative lock hides it).
|
||||
if let Some(st) = stream.as_mut() {
|
||||
// Host-framebuffer px → window px: the aspect-fit factor the video is drawn at
|
||||
// (same `min(surface/content)` as `finger_to_content`). The forwarded pointer is
|
||||
// resampled by it so a high-DPI host's oversized bitmap lands sized to the streamed
|
||||
// desktop rather than ballooning. 1:1 until the first frame gives `last_video`.
|
||||
let fit_scale = st.last_video.map_or(1.0, |(vw, vh)| {
|
||||
let (pw, ph) = window.size_in_pixels();
|
||||
(pw as f32 / vw.max(1) as f32).min(ph as f32 / vh.max(1) as f32)
|
||||
});
|
||||
if let (Some(chan), Some(c)) = (st.cursor_chan.as_mut(), st.connector.as_ref()) {
|
||||
let desktop_active = st
|
||||
.capture
|
||||
.as_ref()
|
||||
.is_some_and(|cap| cap.captured() && cap.desktop());
|
||||
chan.pump(c, &mouse, desktop_active);
|
||||
chan.pump(c, &mouse, desktop_active, fit_scale);
|
||||
// §8 mid-stream render flip: tell the host who renders the pointer whenever
|
||||
// the local model changes. Desktop-active = we draw it (host excludes +
|
||||
// forwards); anything else — the capture model OR a released pointer — the
|
||||
|
||||
Reference in New Issue
Block a user