Merge remote-tracking branch 'origin/main'
apple / swift (push) Successful in 55s
ci / rust (push) Failing after 1m4s
ci / web (push) Successful in 36s
ci / docs-site (push) Successful in 30s
android / android (push) Successful in 2m27s
deb / build-publish (push) Successful in 2m24s
decky / build-publish (push) Successful in 25s
ci / bench (push) Successful in 4m31s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 17s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 2m47s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 2m55s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 22s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 2m14s
flatpak / build-publish (push) Failing after 2m41s
rpm / build-publish (fedora-44, punktfunk-fedora44-rpm) (push) Failing after 4m15s
docker / deploy-docs (push) Successful in 21s
rpm / build-publish (bazzite, punktfunk-fedora-rpm) (push) Has been cancelled

This commit is contained in:
2026-06-17 07:24:27 +00:00
11 changed files with 503 additions and 135 deletions
+1
View File
@@ -182,6 +182,7 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo
CompositorPref::from_u8(compositor_pref.clamp(0, u8::MAX as jint) as u8),
GamepadPref::from_u8(gamepad_pref.clamp(0, u8::MAX as jint) as u8),
bitrate_kbps.max(0) as u32, // 0 = host default
0, // video_caps: 8-bit only on Android for now
None, // launch: default app
pin, // Some → Crypto on host-fp mismatch
identity, // owned (cert, key) PEM, or None (anonymous)
+2 -1
View File
@@ -308,7 +308,8 @@ fn speed_test(app: Rc<App>, req: ConnectRequest) {
},
CompositorPref::Auto,
GamepadPref::Auto,
0,
0, // bitrate_kbps (host default)
0, // video_caps: the Linux client has no 10-bit/HDR present path yet
None, // launch: speed-test probe connect, no game
pin,
Some(identity),
@@ -96,6 +96,7 @@ fn pump(
params.compositor,
params.gamepad,
params.bitrate_kbps,
0, // video_caps: the Linux client has no 10-bit/HDR present path yet
None, // launch: the Linux client has no library picker yet
params.pin,
Some(params.identity),
+147 -22
View File
@@ -14,7 +14,7 @@
use crate::discovery::{self, DiscoveredHost};
use crate::gamepad::GamepadService;
use crate::present::Presenter;
use crate::session::{self, SessionEvent, SessionParams};
use crate::session::{self, SessionEvent, SessionParams, Stats};
use crate::trust::{self, KnownHost, KnownHosts, Settings};
use crate::video::DecodedFrame;
use punktfunk_core::client::NativeClient;
@@ -51,6 +51,56 @@ struct Target {
pair_optional: bool,
}
/// Stable app services handed to the page components as props. Each routed screen that uses
/// hooks (`hosts_page`/`pair_page`/`stream_page`) is mounted as its own `component(...)`, so
/// its hooks live in an isolated slot list — calling them on the shared parent `cx` would
/// change the hook order whenever the screen changes (reactor's Rules-of-Hooks guard aborts).
///
/// `Svc` compares equal by `ctx` identity (it never meaningfully changes across renders), so a
/// page whose props are just `Svc` re-renders only via its own state hooks, never spuriously
/// from the parent.
#[derive(Clone)]
struct Svc {
ctx: Arc<AppCtx>,
set_screen: AsyncSetState<Screen>,
set_status: AsyncSetState<String>,
}
impl PartialEq for Svc {
fn eq(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.ctx, &other.ctx)
}
}
/// Props for the hosts page: the services plus the changing discovery/status data that must
/// drive its re-render (compared by value, so a new host list or error refreshes the page).
#[derive(Clone)]
struct HostsProps {
svc: Svc,
hosts: Vec<DiscoveredHost>,
status: String,
}
impl PartialEq for HostsProps {
fn eq(&self, other: &Self) -> bool {
self.svc == other.svc && self.hosts == other.hosts && self.status == other.status
}
}
/// Props for the stream page: the services plus the live stats that drive the HUD overlay
/// (compared by value, so each new sample re-renders the overlay).
#[derive(Clone)]
struct StreamProps {
svc: Svc,
stats: Stats,
}
impl PartialEq for StreamProps {
fn eq(&self, other: &Self) -> bool {
self.svc == other.svc && self.stats == other.stats
}
}
/// UI-thread-only present context: the D3D11 presenter plus the decoded-frame receiver.
struct PresentCtx {
presenter: Presenter,
@@ -68,6 +118,9 @@ thread_local! {
struct Shared {
handoff: Mutex<Option<(Arc<NativeClient>, async_channel::Receiver<DecodedFrame>)>>,
target: Mutex<Target>,
/// Latest stream stats, written by the session's event loop and mirrored into reactor state
/// by the stream page's HUD poll thread to drive the overlay.
stats: Mutex<Stats>,
}
pub struct AppCtx {
@@ -173,6 +226,7 @@ fn root(cx: &mut RenderCx, ctx: &Arc<AppCtx>) -> Element {
let (screen, set_screen) = cx.use_async_state(Screen::Hosts);
let (hosts, set_hosts) = cx.use_async_state(Vec::<DiscoveredHost>::new());
let (status, set_status) = cx.use_async_state(String::new());
let (stats, set_stats) = cx.use_async_state(Stats::default());
// Continuous LAN discovery (spawned once).
cx.use_effect((), {
@@ -193,8 +247,40 @@ fn root(cx: &mut RenderCx, ctx: &Arc<AppCtx>) -> Element {
}
});
// HUD stats: the session event loop writes `shared.stats`; this poll thread mirrors it into
// root state so the stream page gets it as a *prop*. (A child component's own async-state
// update is pruned when its props are unchanged — only a prop change re-renders it, exactly
// like discovery → hosts above.)
cx.use_effect((), {
let shared = ctx.shared.clone();
let set_stats = set_stats.clone();
move || {
std::thread::Builder::new()
.name("pf-hud".into())
.spawn(move || {
let mut last = Stats::default();
loop {
std::thread::sleep(std::time::Duration::from_millis(400));
let s = *shared.stats.lock().unwrap();
if s != last {
last = s;
set_stats.call(s);
}
}
})
.ok();
}
});
// Each hook-using screen is mounted as its own component so its hooks are isolated from
// root's (root's own hooks above stay a stable prefix regardless of which screen renders).
let svc = Svc {
ctx: ctx.clone(),
set_screen: set_screen.clone(),
set_status: set_status.clone(),
};
match screen {
Screen::Hosts => hosts_page(cx, ctx, &hosts, &status, &set_screen, &set_status),
Screen::Hosts => component(hosts_page, HostsProps { svc, hosts, status }),
Screen::Connecting => vstack((
ProgressRing::indeterminate()
.width(48.0)
@@ -211,20 +297,19 @@ fn root(cx: &mut RenderCx, ctx: &Arc<AppCtx>) -> Element {
.horizontal_alignment(HorizontalAlignment::Center)
.vertical_alignment(VerticalAlignment::Center)
.into(),
// settings_page uses no hooks (it never touches `cx`), so calling it inline is sound.
Screen::Settings => settings_page(ctx, &set_screen),
Screen::Pair => pair_page(cx, ctx, &set_screen, &set_status),
Screen::Stream => stream_page(cx, ctx),
Screen::Pair => component(pair_page, svc),
Screen::Stream => component(stream_page, StreamProps { svc, stats }),
}
}
fn hosts_page(
cx: &mut RenderCx,
ctx: &Arc<AppCtx>,
hosts: &[DiscoveredHost],
status: &str,
set_screen: &AsyncSetState<Screen>,
set_status: &AsyncSetState<String>,
) -> Element {
fn hosts_page(props: &HostsProps, cx: &mut RenderCx) -> Element {
let ctx = &props.svc.ctx;
let hosts = props.hosts.as_slice();
let status = props.status.as_str();
let set_screen = &props.svc.set_screen;
let set_status = &props.svc.set_status;
let (manual, set_manual) = cx.use_state(String::new());
let known = KnownHosts::load();
@@ -459,6 +544,7 @@ fn connect(
let _ = k.save();
}
gamepad.attach(connector.clone());
*shared.stats.lock().unwrap() = Stats::default(); // clear any prior session's numbers
*shared.handoff.lock().unwrap() = Some((connector, handle.frames.clone()));
ss.call(Screen::Stream);
}
@@ -483,7 +569,7 @@ fn connect(
ss.call(Screen::Hosts);
break;
}
Ok(SessionEvent::Stats(_)) => {}
Ok(SessionEvent::Stats(s)) => *shared.stats.lock().unwrap() = s,
Err(_) => {
gamepad.detach();
ss.call(Screen::Hosts);
@@ -493,12 +579,10 @@ fn connect(
});
}
fn pair_page(
cx: &mut RenderCx,
ctx: &Arc<AppCtx>,
set_screen: &AsyncSetState<Screen>,
set_status: &AsyncSetState<String>,
) -> Element {
fn pair_page(props: &Svc, cx: &mut RenderCx) -> Element {
let ctx = &props.ctx;
let set_screen = &props.set_screen;
let set_status = &props.set_status;
let (code, set_code) = cx.use_state(String::new());
let target = ctx.shared.target.lock().unwrap().clone();
@@ -688,7 +772,8 @@ fn present_newest(ctx: &mut PresentCtx) {
ctx.presenter.present(cpu);
}
fn stream_page(cx: &mut RenderCx, ctx: &Arc<AppCtx>) -> Element {
fn stream_page(props: &StreamProps, cx: &mut RenderCx) -> Element {
let ctx = &props.svc.ctx;
// Take the connector + frames handoff once on mount; keep the connector alive (and for input)
// in a use_ref, stash frames for `on_ready`, install the input hooks (and remove on unmount).
let connector_ref = cx.use_ref::<Option<Arc<NativeClient>>>(None);
@@ -710,7 +795,7 @@ fn stream_page(cx: &mut RenderCx, ctx: &Arc<AppCtx>) -> Element {
cx.use_effect((), {
let rendering = rendering.clone();
move || {
if let Ok(r) = on_rendering(|| {
if let Ok(r) = on_rendering(move || {
PRESENT.with(|cell| {
if let Some(ctx) = cell.borrow_mut().as_mut() {
present_newest(ctx);
@@ -722,6 +807,8 @@ fn stream_page(cx: &mut RenderCx, ctx: &Arc<AppCtx>) -> Element {
}
});
let mode = connector_ref.borrow().as_ref().map(|c| c.mode());
grid((
swap_chain_panel()
.on_ready(|panel| match Presenter::new(1280, 720) {
Ok(p) => {
@@ -746,6 +833,44 @@ fn stream_page(cx: &mut RenderCx, ctx: &Arc<AppCtx>) -> Element {
ctx.presenter.resize(w as u32, h as u32);
}
});
})
}),
hud_overlay(&props.stats, mode),
))
.into()
}
/// The streaming HUD overlay (top-right), mirroring the Apple client: mode + fps/throughput, the
/// capture→client latency + decode time, and the release-cursor hint. Layered over the
/// `SwapChainPanel` in the same grid cell.
fn hud_overlay(stats: &Stats, mode: Option<Mode>) -> Element {
let res = mode
.map(|m| format!("{}\u{00D7}{}@{}", m.width, m.height, m.refresh_hz))
.unwrap_or_else(|| "\u{2014}".into());
let line1 = format!("{res} {:.0} fps {:.1} Mb/s", stats.fps, stats.mbps);
let line2 = format!(
"capture\u{2192}client {:.1} ms p50 \u{00B7} decode {:.1} ms",
stats.latency_ms, stats.decode_ms
);
border(
vstack((
text_block(line1)
.font_size(12.0)
.foreground(Color::rgb(255, 255, 255)),
text_block(line2)
.font_size(11.0)
.foreground(Color::rgb(200, 200, 200)),
text_block("Ctrl+Alt+Shift+Q releases the mouse")
.font_size(11.0)
.foreground(Color::rgb(160, 160, 160)),
))
.spacing(2.0),
)
.background(Color::rgb(0, 0, 0))
.corner_radius(8.0)
.padding(uniform(10.0))
.opacity(0.82)
.horizontal_alignment(HorizontalAlignment::Right)
.vertical_alignment(VerticalAlignment::Top)
.margin(uniform(12.0))
.into()
}
+167 -63
View File
@@ -1,14 +1,23 @@
//! Stream input: Win32 low-level keyboard + mouse hooks forwarding to the host while the WinUI
//! window is focused and capture is engaged.
//! window is focused and the pointer is captured.
//!
//! windows-reactor exposes no raw key-down/up or pointer-position/wheel events (only keyboard
//! *accelerators* and pointer button-state), which is insufficient for a game stream. So this
//! drops below XAML to `WH_KEYBOARD_LL` / `WH_MOUSE_LL`, installed on the UI thread when the
//! stream page mounts and removed when it unmounts. The `SwapChainPanel` fills the window, so the
//! pointer maps through the window's client rect (Contain-fit into the negotiated mode), and
//! keys carry the native Windows VK directly (the wire contract). While captured, events inside
//! the video area are swallowed (so Alt+Tab / Win etc. reach the host); Ctrl+Alt+Shift+Q toggles
//! capture; clicks outside the client area (the title bar) pass through so the window stays usable.
//! stream page mounts and removed when it unmounts.
//!
//! **Pointer lock.** While captured the cursor is *locked* the way a game-streaming client locks
//! it (Moonlight/Parsec): the OS cursor is hidden + confined to the window (`ClipCursor`), and
//! every physical move is turned into a **relative** delta (`InputKind::MouseMove`) — we read the
//! offset from the window centre, ship it (scaled screen→host through the Contain-fit factor, with
//! sub-pixel remainder carried so slow drags aren't lost), then warp the cursor back to centre so
//! it never reaches a screen edge. This is why the old absolute path froze: swallowing
//! `WM_MOUSEMOVE` pinned the OS cursor, so `pt` never travelled and the absolute coordinate
//! snapped to one point. Keys carry the native Windows VK directly (the wire contract).
//!
//! **Ctrl+Alt+Shift+Q** toggles capture — releasing the lock hands the cursor back to the local
//! desktop (and re-grabs on the next toggle). Losing foreground also releases the lock so the
//! cursor is never stranded.
use punktfunk_core::client::NativeClient;
use punktfunk_core::config::Mode;
@@ -17,16 +26,15 @@ use std::collections::HashSet;
use std::sync::atomic::{AtomicIsize, Ordering};
use std::sync::{Arc, Mutex};
use windows::Win32::Foundation::{HWND, LPARAM, LRESULT, POINT, RECT, WPARAM};
use windows::Win32::Graphics::Gdi::ScreenToClient;
use windows::Win32::Graphics::Gdi::ClientToScreen;
use windows::Win32::System::LibraryLoader::GetModuleHandleW;
use windows::Win32::UI::Input::KeyboardAndMouse::{
GetAsyncKeyState, VIRTUAL_KEY, VK_CONTROL, VK_MENU, VK_Q, VK_SHIFT,
};
use windows::Win32::UI::Input::KeyboardAndMouse::VK_Q;
use windows::Win32::UI::WindowsAndMessaging::{
CallNextHookEx, GetClientRect, GetForegroundWindow, SetWindowsHookExW, UnhookWindowsHookEx,
HC_ACTION, HHOOK, KBDLLHOOKSTRUCT, MSLLHOOKSTRUCT, WH_KEYBOARD_LL, WH_MOUSE_LL, WM_KEYUP,
WM_LBUTTONDOWN, WM_LBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP, WM_MOUSEHWHEEL, WM_MOUSEMOVE,
WM_MOUSEWHEEL, WM_RBUTTONDOWN, WM_RBUTTONUP, WM_SYSKEYUP, WM_XBUTTONDOWN, WM_XBUTTONUP,
CallNextHookEx, ClipCursor, GetClientRect, GetForegroundWindow, SetCursorPos,
SetWindowsHookExW, ShowCursor, UnhookWindowsHookEx, HC_ACTION, HHOOK, KBDLLHOOKSTRUCT,
LLMHF_INJECTED, MSLLHOOKSTRUCT, WH_KEYBOARD_LL, WH_MOUSE_LL, WM_KEYUP, WM_LBUTTONDOWN,
WM_LBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP, WM_MOUSEHWHEEL, WM_MOUSEMOVE, WM_MOUSEWHEEL,
WM_RBUTTONDOWN, WM_RBUTTONUP, WM_SYSKEYUP, WM_XBUTTONDOWN, WM_XBUTTONUP,
};
struct State {
@@ -34,7 +42,21 @@ struct State {
mode: Mode,
/// Our window handle, stored as the raw `isize` so `State` is `Send` (`HWND` is not).
hwnd: isize,
/// User intent: forward input to the host (toggled by Ctrl+Alt+Shift+Q).
captured: bool,
/// The OS pointer is currently locked (hidden + confined + recentering). Tracks the real
/// `ClipCursor`/`ShowCursor` state so we engage/disengage exactly once per transition.
locked: bool,
/// Lock centre in screen coordinates (the cursor is warped here after every move).
center_x: i32,
center_y: i32,
/// Sub-pixel remainder of the screen→host scale, carried so slow drags aren't truncated away.
acc_x: f32,
acc_y: f32,
/// Modifier state, tracked from the hook's own event stream (see `kbd_proc`).
ctrl: bool,
alt: bool,
shift: bool,
held_keys: HashSet<u8>,
held_buttons: HashSet<u32>,
}
@@ -48,14 +70,25 @@ static MOUSE_HOOK: AtomicIsize = AtomicIsize::new(0);
/// Install the hooks for a streaming session. Call from the UI thread once the window is shown.
pub fn install(connector: Arc<NativeClient>, mode: Mode) {
let hwnd = unsafe { GetForegroundWindow() };
*STATE.lock().unwrap() = Some(State {
let mut st = State {
connector,
mode,
hwnd: hwnd.0 as isize,
captured: true,
locked: false,
center_x: 0,
center_y: 0,
acc_x: 0.0,
acc_y: 0.0,
ctrl: false,
alt: false,
shift: false,
held_keys: HashSet::new(),
held_buttons: HashSet::new(),
});
};
// Lock immediately (the window is foreground at mount, like Moonlight grabbing on stream start).
set_locked(&mut st, true);
*STATE.lock().unwrap() = Some(st);
unsafe {
let hinst = GetModuleHandleW(None).ok();
if let Ok(h) = SetWindowsHookExW(WH_KEYBOARD_LL, Some(kbd_proc), hinst.map(Into::into), 0) {
@@ -65,10 +98,13 @@ pub fn install(connector: Arc<NativeClient>, mode: Mode) {
MOUSE_HOOK.store(h.0 as isize, Ordering::SeqCst);
}
}
tracing::info!("stream input hooks installed (Ctrl+Alt+Shift+Q toggles capture)");
tracing::info!(
"stream input hooks installed — pointer locked (Ctrl+Alt+Shift+Q toggles capture)"
);
}
/// Remove the hooks and flush any held keys/buttons (so nothing sticks down on the host).
/// Remove the hooks, release the pointer lock, and flush any held keys/buttons (so nothing
/// sticks down on the host).
pub fn uninstall() {
unsafe {
let k = KBD_HOOK.swap(0, Ordering::SeqCst);
@@ -80,14 +116,65 @@ pub fn uninstall() {
let _ = UnhookWindowsHookEx(HHOOK(m as *mut _));
}
}
if let Some(st) = STATE.lock().unwrap().take() {
for vk in &st.held_keys {
send(&st.connector, InputKind::KeyUp, *vk as u32, 0, 0, 0);
if let Some(mut st) = STATE.lock().unwrap().take() {
set_locked(&mut st, false); // hand the cursor back to the desktop
flush_held(&mut st);
}
for b in &st.held_buttons {
send(&st.connector, InputKind::MouseButtonUp, *b, 0, 0, 0);
}
/// Release every held key/button on the host, so nothing sticks down when capture is dropped
/// (toggled off) or the session ends.
fn flush_held(st: &mut State) {
let c = st.connector.clone();
for vk in st.held_keys.drain() {
send(&c, InputKind::KeyUp, vk as u32, 0, 0, 0);
}
for b in st.held_buttons.drain() {
send(&c, InputKind::MouseButtonUp, b, 0, 0, 0);
}
}
/// Engage or release the pointer lock: confine + hide + recentre on, free + show on off.
/// Guarded so the `ClipCursor`/`ShowCursor` calls stay balanced (one each per transition).
fn set_locked(st: &mut State, on: bool) {
if on == st.locked {
return;
}
let hwnd = HWND(st.hwnd as *mut _);
unsafe {
if on {
let mut rc = RECT::default();
if GetClientRect(hwnd, &mut rc).is_ok() {
let mut tl = POINT {
x: rc.left,
y: rc.top,
};
let mut br = POINT {
x: rc.right,
y: rc.bottom,
};
let _ = ClientToScreen(hwnd, &mut tl);
let _ = ClientToScreen(hwnd, &mut br);
let clip = RECT {
left: tl.x,
top: tl.y,
right: br.x,
bottom: br.y,
};
let _ = ClipCursor(Some(&clip as *const RECT));
st.center_x = (tl.x + br.x) / 2;
st.center_y = (tl.y + br.y) / 2;
let _ = SetCursorPos(st.center_x, st.center_y);
}
let _ = ShowCursor(false);
st.acc_x = 0.0;
st.acc_y = 0.0;
} else {
let _ = ClipCursor(None);
let _ = ShowCursor(true);
}
}
st.locked = on;
}
fn send(c: &NativeClient, kind: InputKind, code: u32, x: i32, y: i32, flags: u32) {
@@ -101,10 +188,6 @@ fn send(c: &NativeClient, kind: InputKind, code: u32, x: i32, y: i32, flags: u32
});
}
fn key_down(vk: VIRTUAL_KEY) -> bool {
(unsafe { GetAsyncKeyState(vk.0 as i32) } as u16 & 0x8000) != 0
}
unsafe extern "system" fn kbd_proc(code: i32, wparam: WPARAM, lparam: LPARAM) -> LRESULT {
if code == HC_ACTION as i32 {
let kb = unsafe { &*(lparam.0 as *const KBDLLHOOKSTRUCT) };
@@ -113,16 +196,26 @@ unsafe extern "system" fn kbd_proc(code: i32, wparam: WPARAM, lparam: LPARAM) ->
let vk = kb.vkCode as u16;
let mut guard = STATE.lock().unwrap();
if let Some(st) = guard.as_mut() {
// Track modifier state from the hook's own event stream — reliable even while we
// swallow these keys (GetAsyncKeyState doesn't reflect keys suppressed by our own LL
// hook, which is why the shortcut never fired). Handles the generic + L/R vk codes.
match kb.vkCode {
0x11 | 0xA2 | 0xA3 => st.ctrl = !up, // (L/R)CONTROL
0x12 | 0xA4 | 0xA5 => st.alt = !up, // (L/R)MENU (Alt)
0x10 | 0xA0 | 0xA1 => st.shift = !up, // (L/R)SHIFT
_ => {}
}
let foreground = unsafe { GetForegroundWindow() }.0 as isize == st.hwnd;
if foreground {
// Capture toggle: Ctrl+Alt+Shift+Q (consumed locally, never forwarded).
if !up
&& vk == VK_Q.0
&& key_down(VK_CONTROL)
&& key_down(VK_MENU)
&& key_down(VK_SHIFT)
{
st.captured = !st.captured;
if !up && vk == VK_Q.0 && st.ctrl && st.alt && st.shift {
let on = !st.captured;
st.captured = on;
set_locked(st, on); // grab/release the cursor immediately
if !on {
flush_held(st); // release held keys/buttons so nothing sticks on the host
}
tracing::info!(captured = on, "capture toggled (Ctrl+Alt+Shift+Q)");
return LRESULT(1);
}
if st.captured {
@@ -143,48 +236,59 @@ unsafe extern "system" fn kbd_proc(code: i32, wparam: WPARAM, lparam: LPARAM) ->
unsafe { CallNextHookEx(None, code, wparam, lparam) }
}
/// Map a screen point to video pixels through the client-rect Contain-fit letterbox. Returns
/// `None` when the point is outside the video area (so the title bar / borders stay interactive).
fn map_abs(st: &State, screen: POINT) -> Option<(i32, i32, u32)> {
let hwnd = HWND(st.hwnd as *mut _);
let mut p = screen;
unsafe {
let _ = ScreenToClient(hwnd, &mut p);
}
/// Client-area size in pixels (for the screen→host relative-motion scale).
fn client_size(hwnd: isize) -> (f32, f32) {
let mut rc = RECT::default();
if unsafe { GetClientRect(hwnd, &mut rc) }.is_err() {
return None;
if unsafe { GetClientRect(HWND(hwnd as *mut _), &mut rc) }.is_ok() {
(
(rc.right - rc.left).max(1) as f32,
(rc.bottom - rc.top).max(1) as f32,
)
} else {
(1.0, 1.0)
}
let (ww, wh) = (
(rc.right - rc.left).max(1) as f64,
(rc.bottom - rc.top).max(1) as f64,
);
if (p.x as f64) < 0.0 || (p.y as f64) < 0.0 || p.x as f64 > ww || p.y as f64 > wh {
return None;
}
let (vw, vh) = (st.mode.width.max(1) as f64, st.mode.height.max(1) as f64);
let scale = (ww / vw).min(wh / vh);
let (ox, oy) = ((ww - vw * scale) / 2.0, (wh - vh * scale) / 2.0);
let px = (((p.x as f64 - ox) / scale).round()).clamp(0.0, vw - 1.0) as i32;
let py = (((p.y as f64 - oy) / scale).round()).clamp(0.0, vh - 1.0) as i32;
let flags = (st.mode.width << 16) | (st.mode.height & 0xffff);
Some((px, py, flags))
}
unsafe extern "system" fn mouse_proc(code: i32, wparam: WPARAM, lparam: LPARAM) -> LRESULT {
if code == HC_ACTION as i32 {
let ms = unsafe { &*(lparam.0 as *const MSLLHOOKSTRUCT) };
let msg = wparam.0 as u32;
let injected = (ms.flags & LLMHF_INJECTED) != 0;
let mut guard = STATE.lock().unwrap();
if let Some(st) = guard.as_mut() {
let foreground = unsafe { GetForegroundWindow() }.0 as isize == st.hwnd;
if st.captured && foreground {
let Some((px, py, flags)) = map_abs(st, ms.pt) else {
let want_lock = st.captured && foreground;
if want_lock != st.locked {
set_locked(st, want_lock); // sync to focus changes (e.g. lost foreground)
}
if st.locked {
// Skip the synthetic move our own SetCursorPos recentre generates.
if injected {
return unsafe { CallNextHookEx(None, code, wparam, lparam) };
};
}
let c = st.connector.clone();
match msg {
WM_MOUSEMOVE => send(&c, InputKind::MouseMoveAbs, 0, px, py, flags),
WM_MOUSEMOVE => {
let dx = (ms.pt.x - st.center_x) as f32;
let dy = (ms.pt.y - st.center_y) as f32;
if dx != 0.0 || dy != 0.0 {
// screen px → host px: the Contain-fit display scale's inverse, so the
// host cursor tracks the physical mouse 1:1 on screen at any window size.
let (ww, wh) = client_size(st.hwnd);
let (vw, vh) =
(st.mode.width.max(1) as f32, st.mode.height.max(1) as f32);
let s = (ww / vw).min(wh / vh).max(0.01);
st.acc_x += dx / s;
st.acc_y += dy / s;
let (hx, hy) = (st.acc_x.trunc() as i32, st.acc_y.trunc() as i32);
st.acc_x -= hx as f32;
st.acc_y -= hy as f32;
if hx != 0 || hy != 0 {
send(&c, InputKind::MouseMove, 0, hx, hy, 0);
}
}
let _ = unsafe { SetCursorPos(st.center_x, st.center_y) };
}
WM_LBUTTONDOWN => button(st, 1, true),
WM_LBUTTONUP => button(st, 1, false),
WM_RBUTTONDOWN => button(st, 3, true),
@@ -211,7 +315,7 @@ unsafe extern "system" fn mouse_proc(code: i32, wparam: WPARAM, lparam: LPARAM)
),
_ => {}
}
return LRESULT(1); // swallow inside the video area
return LRESULT(1); // swallow inside the locked window
}
}
}
+93 -5
View File
@@ -5,8 +5,12 @@
//! The device prefers a hardware adapter and falls back to **WARP** (the GPU-less dev box runs
//! the whole present path in software). The draw is a single full-screen triangle sampling the
//! video texture; a letterbox is produced by clearing the back buffer black and setting the
//! viewport to the Contain-fit rect (no per-frame vertex buffer). SDR 8-bit path; the
//! 10-bit/HDR present (`R10G10B10A2` + `SetColorSpace1`) is a follow-up alongside P010 decode.
//! viewport to the Contain-fit rect (no per-frame vertex buffer).
//!
//! **HDR10**: when a frame is BT.2020 PQ (`CpuFrame::hdr`), the swapchain flips to
//! `R10G10B10A2` + `DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020` (+ HDR10 metadata) via
//! `ResizeBuffers`/`SetColorSpace1`; the decoded samples are already PQ-encoded so the shader is a
//! plain passthrough and the compositor maps PQ→display. SDR stays 8-bit B8G8R8A8.
//!
//! All `windows` types here come from the same windows-rs commit as `windows-reactor`, so the
//! `IDXGISwapChain1` handed to `set_swap_chain` satisfies reactor's `windows_core::Interface`.
@@ -50,6 +54,9 @@ pub struct Presenter {
/// Panel (swapchain) size in pixels, updated on resize.
panel_w: u32,
panel_h: u32,
/// Whether the swapchain is currently in 10-bit HDR10 (R10G10B10A2 + ST.2084) mode; flipped
/// to match each frame's `hdr` flag.
hdr: bool,
}
impl Presenter {
@@ -69,6 +76,7 @@ impl Presenter {
tex: None,
panel_w: width.max(1),
panel_h: height.max(1),
hdr: false,
})
}
@@ -100,6 +108,9 @@ impl Presenter {
/// last texture (or black). Called from the reactor `on_rendering` per-frame callback.
pub fn present(&mut self, frame: Option<&CpuFrame>) {
if let Some(f) = frame {
if f.hdr != self.hdr {
self.set_hdr(f.hdr);
}
if let Err(e) = self.upload(f) {
tracing::warn!(error = %e, "frame upload failed");
}
@@ -144,16 +155,74 @@ impl Presenter {
}
}
/// Switch the swapchain between 8-bit SDR (B8G8R8A8, sRGB/BT.709) and 10-bit HDR10
/// (R10G10B10A2, ST.2084 PQ BT.2020). `ResizeBuffers` can change the back-buffer format in
/// place, so the panel binding (`set_swap_chain`) stays valid — no rebind needed. The decoded
/// samples are already PQ-encoded BT.2020 (see `video::convert`), so the colour space is all the
/// compositor needs to map them to the display.
fn set_hdr(&mut self, on: bool) {
self.rtv = None; // release back-buffer refs before ResizeBuffers
self.tex = None; // texture format changes (R10G10B10A2 vs R8G8B8A8)
let format = if on {
DXGI_FORMAT_R10G10B10A2_UNORM
} else {
DXGI_FORMAT_B8G8R8A8_UNORM
};
unsafe {
if let Err(e) = self.swap.ResizeBuffers(
0,
self.panel_w,
self.panel_h,
format,
DXGI_SWAP_CHAIN_FLAG(0),
) {
tracing::warn!(error = %e, "ResizeBuffers for HDR switch failed");
return;
}
let colorspace = if on {
DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020
} else {
DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709
};
if let Ok(sc3) = self.swap.cast::<IDXGISwapChain3>() {
// Only set a colour space the swapchain accepts for present (on an SDR desktop the
// DWM still tone-maps HDR10 → SDR, so leaving the default there is fine).
if let Ok(support) = sc3.CheckColorSpaceSupport(colorspace) {
if support & DXGI_SWAP_CHAIN_COLOR_SPACE_SUPPORT_FLAG_PRESENT.0 as u32 != 0 {
let _ = sc3.SetColorSpace1(colorspace);
}
}
}
if on {
if let Ok(sc4) = self.swap.cast::<IDXGISwapChain4>() {
let md = hdr10_metadata();
let bytes = std::slice::from_raw_parts(
&md as *const DXGI_HDR_METADATA_HDR10 as *const u8,
std::mem::size_of::<DXGI_HDR_METADATA_HDR10>(),
);
let _ = sc4.SetHDRMetaData(DXGI_HDR_METADATA_TYPE_HDR10, Some(bytes));
}
}
}
self.hdr = on;
tracing::info!(hdr = on, "swapchain colour mode switched");
}
fn upload(&mut self, frame: &CpuFrame) -> Result<()> {
let (w, h) = (frame.width, frame.height);
let need_new = !matches!(&self.tex, Some((_, _, tw, th)) if *tw == w && *th == h);
if need_new {
let format = if self.hdr {
DXGI_FORMAT_R10G10B10A2_UNORM
} else {
DXGI_FORMAT_R8G8B8A8_UNORM
};
let desc = D3D11_TEXTURE2D_DESC {
Width: w,
Height: h,
MipLevels: 1,
ArraySize: 1,
Format: DXGI_FORMAT_R8G8B8A8_UNORM,
Format: format,
SampleDesc: DXGI_SAMPLE_DESC {
Count: 1,
Quality: 0,
@@ -191,7 +260,7 @@ impl Presenter {
let row_bytes = (w as usize) * 4;
for y in 0..h as usize {
std::ptr::copy_nonoverlapping(
frame.rgba.as_ptr().add(y * src_pitch),
frame.pixels.as_ptr().add(y * src_pitch),
dst.add(y * dst_pitch),
row_bytes.min(src_pitch),
);
@@ -273,7 +342,10 @@ fn create_composition_swapchain(
BufferCount: 2,
Scaling: DXGI_SCALING_STRETCH,
SwapEffect: DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL,
AlphaMode: DXGI_ALPHA_MODE_PREMULTIPLIED,
// IGNORE (opaque), not PREMULTIPLIED: the video fills the panel and the HDR `X2BGR10`
// upload leaves the 2 padding/alpha bits 0 — premultiplied alpha would then make HDR frames
// transparent. Opaque is correct for a full-frame video surface either way.
AlphaMode: DXGI_ALPHA_MODE_IGNORE,
Flags: 0,
};
unsafe {
@@ -354,3 +426,19 @@ fn blob_bytes(blob: &ID3DBlob) -> &[u8] {
std::slice::from_raw_parts(p, n)
}
}
/// Generic HDR10 mastering metadata: BT.2020 primaries + D65 white (0.00002 units), a 1000-nit
/// mastering display, MaxCLL 1000 / MaxFALL 400. The protocol doesn't carry the stream's real
/// mastering metadata yet (host follow-up), so these are sane defaults the display tone-maps from.
fn hdr10_metadata() -> DXGI_HDR_METADATA_HDR10 {
DXGI_HDR_METADATA_HDR10 {
RedPrimary: [35400, 14600],
GreenPrimary: [8500, 39850],
BluePrimary: [6550, 2300],
WhitePoint: [15635, 16450],
MaxMasteringLuminance: 1000,
MinMasteringLuminance: 1, // 0.0001-nit units → 0.0001 nits
MaxContentLightLevel: 1000,
MaxFrameAverageLightLevel: 400,
}
}
@@ -30,7 +30,7 @@ pub struct SessionParams {
pub identity: (String, String),
}
#[derive(Clone, Copy, Default)]
#[derive(Clone, Copy, Default, PartialEq)]
pub struct Stats {
pub fps: f32,
pub mbps: f32,
@@ -99,6 +99,10 @@ fn pump(
params.compositor,
params.gamepad,
params.bitrate_kbps,
// Advertise 10-bit + HDR10: the presenter handles BT.2020 PQ (R10G10B10A2) frames, so the
// host may upgrade HDR content to a Main10/PQ stream (it still only does so for actual HDR
// content with its own 10-bit gate). 8-bit SDR is unaffected.
punktfunk_core::quic::VIDEO_CAP_10BIT | punktfunk_core::quic::VIDEO_CAP_HDR,
None, // launch: the Windows client has no library picker yet
params.pin,
Some(params.identity),
+45 -15
View File
@@ -20,13 +20,17 @@ pub enum DecodedFrame {
Cpu(CpuFrame),
}
/// RGBA pixels for a D3D11 `R8G8B8A8_UNORM` texture upload (which takes a row pitch).
/// Packed 4-byte-per-pixel frame for a D3D11 texture upload (which takes a row pitch). The bytes
/// are `R8G8B8A8` for SDR and `X2BGR10` (== DXGI `R10G10B10A2`, R in the low 10 bits) for HDR.
pub struct CpuFrame {
pub width: u32,
pub height: u32,
/// RGBA row stride in bytes (≥ width*4 — swscale pads rows for SIMD).
/// Row stride in bytes (≥ width*4 — swscale pads rows for SIMD).
pub stride: usize,
pub rgba: Vec<u8>,
pub pixels: Vec<u8>,
/// BT.2020 PQ HDR10 frame: `pixels` is `X2BGR10` and the presenter switches to a 10-bit
/// R10G10B10A2 + ST.2084 swapchain. `false` = ordinary 8-bit BT.709 SDR.
pub hdr: bool,
}
pub struct Decoder {
@@ -51,8 +55,9 @@ impl Decoder {
struct SoftwareDecoder {
decoder: ffmpeg::decoder::Video,
/// Rebuilt whenever the decoded format/size changes (mid-stream `Reconfigure`).
sws: Option<(scaling::Context, Pixel, u32, u32)>,
/// Rebuilt whenever the decoded format/size **or output format** changes (mid-stream
/// `Reconfigure`, or an SDR↔HDR flip): `(ctx, src_fmt, w, h, dst_fmt)`.
sws: Option<(scaling::Context, Pixel, u32, u32, Pixel)>,
}
impl SoftwareDecoder {
@@ -79,28 +84,53 @@ impl SoftwareDecoder {
let mut frame = AvFrame::empty();
let mut out = None;
while self.decoder.receive_frame(&mut frame).is_ok() {
out = Some(self.convert_rgba(&frame)?);
out = Some(self.convert(&frame)?);
}
Ok(out)
}
fn convert_rgba(&mut self, frame: &AvFrame) -> Result<CpuFrame> {
/// Convert the decoded YUV frame to a packed 4-byte format the presenter uploads directly:
/// SDR → `RGBA` (BT.709), HDR (SMPTE ST.2084 / PQ transfer) → `X2BGR10` (10-bit, == DXGI
/// R10G10B10A2) using the BT.2020 matrix. For HDR the PQ-encoded values pass through unchanged
/// (swscale only applies the YUV→RGB matrix + range, never the transfer) — exactly what an
/// HDR10/ST.2084 swapchain wants.
fn convert(&mut self, frame: &AvFrame) -> Result<CpuFrame> {
use ffmpeg::color::TransferCharacteristic;
let (fmt, w, h) = (frame.format(), frame.width(), frame.height());
let rebuild =
!matches!(&self.sws, Some((_, f, sw, sh)) if *f == fmt && *sw == w && *sh == h);
let hdr = frame.color_transfer_characteristic() == TransferCharacteristic::SMPTE2084;
let dst = if hdr { Pixel::X2BGR10LE } else { Pixel::RGBA };
let rebuild = !matches!(&self.sws, Some((_, f, sw, sh, d)) if *f == fmt && *sw == w && *sh == h && *d == dst);
if rebuild {
let ctx = scaling::Context::get(fmt, w, h, Pixel::RGBA, w, h, scaling::Flags::POINT)
let mut ctx = scaling::Context::get(fmt, w, h, dst, w, h, scaling::Flags::POINT)
.context("swscale context")?;
self.sws = Some((ctx, fmt, w, h));
if hdr {
// BT.2020 non-constant-luminance YUV (limited range) → full-range RGB. swscale
// applies only the matrix + range here, so the samples stay PQ-encoded.
unsafe {
let coef = ffmpeg::ffi::sws_getCoefficients(ffmpeg::ffi::SWS_CS_BT2020);
ffmpeg::ffi::sws_setColorspaceDetails(
ctx.as_mut_ptr(),
coef,
0, // src range: limited (video)
coef,
1, // dst range: full
0,
1 << 16,
1 << 16, // brightness / contrast / saturation defaults (16.16)
);
}
}
self.sws = Some((ctx, fmt, w, h, dst));
}
let (sws, ..) = self.sws.as_mut().unwrap();
let mut rgba = AvFrame::empty();
sws.run(frame, &mut rgba).map_err(|e| anyhow!("sws: {e}"))?;
let mut conv = AvFrame::empty();
sws.run(frame, &mut conv).map_err(|e| anyhow!("sws: {e}"))?;
Ok(CpuFrame {
width: w,
height: h,
stride: rgba.stride(0),
rgba: rgba.data(0).to_vec(),
stride: conv.stride(0),
pixels: conv.data(0).to_vec(),
hdr,
})
}
}
+3
View File
@@ -891,6 +891,9 @@ pub unsafe extern "C" fn punktfunk_connect_ex4(
pref,
gamepad,
bitrate_kbps,
// 8-bit only over the C ABI for now — the ABI doesn't yet carry the embedder's video
// caps (Apple/Android decode 8-bit). The native Windows client advertises 10-bit/HDR.
0,
launch,
pin,
identity,
+11 -4
View File
@@ -196,6 +196,10 @@ impl NativeClient {
compositor: CompositorPref,
gamepad: GamepadPref,
bitrate_kbps: u32,
// Client video capabilities advertised to the host (bitfield of quic::VIDEO_CAP_10BIT /
// VIDEO_CAP_HDR) — the host upgrades to a 10-bit / HDR encode only when the matching bit is
// set. 0 = the 8-bit BT.709 stream every client understands.
video_caps: u8,
launch: Option<String>,
pin: Option<[u8; 32]>,
identity: Option<(String, String)>,
@@ -245,6 +249,7 @@ impl NativeClient {
compositor,
gamepad,
bitrate_kbps,
video_caps,
launch,
pin,
identity,
@@ -569,6 +574,7 @@ struct WorkerArgs {
compositor: CompositorPref,
gamepad: GamepadPref,
bitrate_kbps: u32,
video_caps: u8,
launch: Option<String>,
pin: Option<[u8; 32]>,
identity: Option<(String, String)>,
@@ -597,6 +603,7 @@ async fn worker_main(args: WorkerArgs) {
compositor,
gamepad,
bitrate_kbps,
video_caps,
launch,
pin,
identity,
@@ -657,10 +664,10 @@ async fn worker_main(args: WorkerArgs) {
name: None,
// Library id to launch this session, if the embedder asked for one.
launch: launch.clone(),
// TODO(hdr): advertise the embedder's real decode caps once the ABI carries them
// and the Apple/Linux clients decode 10-bit. 0 = 8-bit only — the host then never
// upgrades this connector's session to a stream it can't yet present.
video_caps: 0,
// The embedder's decode/present caps (e.g. the Windows client advertises
// VIDEO_CAP_10BIT | VIDEO_CAP_HDR). The host only upgrades to a 10-bit / HDR encode
// when the matching bit is set, so `0` stays an 8-bit BT.709 stream.
video_caps,
}
.encode(),
)
+4
View File
@@ -3180,6 +3180,7 @@ mod tests {
CompositorPref::Auto,
GamepadPref::Auto,
0,
0, // video_caps
None, // launch
None,
Some((cert.clone(), key.clone())),
@@ -3211,6 +3212,7 @@ mod tests {
CompositorPref::Auto,
GamepadPref::Auto,
0,
0, // video_caps
None, // launch
None,
Some((cert, key)),
@@ -3271,6 +3273,7 @@ mod tests {
CompositorPref::Auto,
GamepadPref::Auto,
0,
0, // video_caps
None, // launch
None,
None,
@@ -3297,6 +3300,7 @@ mod tests {
CompositorPref::Auto,
GamepadPref::Auto,
0,
0, // video_caps
None, // launch
Some(host_fp),
Some((cert.clone(), key.clone())),