From f4b52d0bf5fa2a5885b29805f6de99e2a07b40fd Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 17 Jul 2026 19:32:03 +0200 Subject: [PATCH] feat(clipboard): pass original JPEG/GIF through verbatim beside the PNG floor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transcoding every image to PNG was the Phase-1 floor, but it bloats lossy originals (a copied JPEG re-encoded lossless is 5-10x the bytes for zero quality gain — feeding the render-timeout on constrained links) and flattens GIFs to one frame. The wire already carries kind LISTS, so offer the original format beside the portable fallback and let the destination pick: - New wire kinds image/jpeg + image/gif (§3.5 extension; strings only, no protocol change). image/png stays the universal floor every peer accepts. - macOS: public.jpeg / com.compuserve.gif rows pass through verbatim both directions; the PNG floor is announced whenever any image is present. - Windows: registered JFIF/GIF formats read + promised verbatim; every image offer still promises CF_DIB, whose delayed render now fetches the richest offered kind (png > jpeg > gif) through the generalized image_to_dib. - Linux maps gain the matching rows (native MIME pass-through). Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 99 +++++++++++++++++++ .../Clipboard/ClipboardSync.swift | 18 +++- crates/pf-clipboard/Cargo.toml | 2 +- crates/pf-clipboard/src/host.rs | 12 +++ crates/pf-clipboard/src/host/windows.rs | 77 ++++++++++++--- crates/pf-clipboard/src/host/winfmt.rs | 11 ++- 6 files changed, 198 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4202bf9..ee48e669 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -560,6 +560,12 @@ dependencies = [ "syn", ] +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.0" @@ -756,6 +762,12 @@ dependencies = [ "cc", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.5" @@ -1141,6 +1153,15 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "fec-rs" version = "0.1.0" @@ -1495,6 +1516,16 @@ dependencies = [ "polyval", ] +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gio" version = "0.22.6" @@ -1988,6 +2019,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "gif", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -2386,6 +2434,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "nasm-rs" version = "0.3.2" @@ -2805,6 +2863,7 @@ dependencies = [ "anyhow", "ashpd", "futures-util", + "image", "libc", "punktfunk-core", "quinn", @@ -3075,6 +3134,19 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "polling" version = "3.11.0" @@ -3382,6 +3454,12 @@ dependencies = [ "winresource", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "pyrowave-sys" version = "0.13.0" @@ -5202,6 +5280,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "wide" version = "0.7.33" @@ -6014,6 +6098,21 @@ version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] + [[package]] name = "zvariant" version = "5.12.0" diff --git a/clients/apple/Sources/PunktfunkKit/Clipboard/ClipboardSync.swift b/clients/apple/Sources/PunktfunkKit/Clipboard/ClipboardSync.swift index 454891de..5083b8b1 100644 --- a/clients/apple/Sources/PunktfunkKit/Clipboard/ClipboardSync.swift +++ b/clients/apple/Sources/PunktfunkKit/Clipboard/ClipboardSync.swift @@ -29,6 +29,11 @@ public final class ClipboardSync: NSObject { ("text/rtf", .rtf), ("text/html", .html), ("image/png", .png), + // Original image formats pass through VERBATIM beside the PNG floor — a copied JPEG + // never balloons into PNG, a GIF keeps its animation; the destination picks the richest + // kind it can place. + ("image/jpeg", NSPasteboard.PasteboardType("public.jpeg")), + ("image/gif", NSPasteboard.PasteboardType("com.compuserve.gif")), ] /// Pasteboard marker types that must never cross the wire (password managers mark secrets /// with these — see nspasteboard.org). @@ -205,11 +210,14 @@ public final class ClipboardSync: NSObject { var kinds = Self.wireToPasteboard .filter { types.contains($0.type) } .map { PunktfunkConnection.ClipKind(mime: $0.wire) } - // Images: macOS image copies usually carry TIFF (browsers add WebP/AVIF/GIF, screenshots - // TIFF) and only sometimes PNG — announce the portable `image/png` whenever ANY - // convertible image type is present; `serveFetch` converts at fetch time (lazy, §3.5). + // PNG floor: announce the portable `image/png` whenever ANY convertible image is present + // — native PNG, TIFF/HEIC (screenshots, Preview), or a JPEG/GIF original already being + // offered verbatim above. `readWireData` converts at fetch time (lazy, §3.5), so the + // fallback costs nothing unless a peer actually pastes it. if !kinds.contains(where: { $0.mime == "image/png" }), - types.contains(.tiff) || types.contains(NSPasteboard.PasteboardType("public.heic")) + types.contains(.tiff) + || types.contains(NSPasteboard.PasteboardType("public.heic")) + || kinds.contains(where: { $0.mime.hasPrefix("image/") }) { kinds.append(PunktfunkConnection.ClipKind(mime: "image/png")) } @@ -385,6 +393,8 @@ private final class RemoteOfferProvider: NSObject, NSPasteboardItemDataProvider case .rtf: return "text/rtf" case .html: return "text/html" case .png: return "image/png" + case NSPasteboard.PasteboardType("public.jpeg"): return "image/jpeg" + case NSPasteboard.PasteboardType("com.compuserve.gif"): return "image/gif" default: return nil } } diff --git a/crates/pf-clipboard/Cargo.toml b/crates/pf-clipboard/Cargo.toml index 8eaade85..e9331753 100644 --- a/crates/pf-clipboard/Cargo.toml +++ b/crates/pf-clipboard/Cargo.toml @@ -22,7 +22,7 @@ quinn = "0.11" tokio = { version = "1", features = ["rt", "rt-multi-thread", "sync", "time", "macros"] } # CF_DIB <-> PNG conversion (winfmt) - most Windows apps paste bitmaps, not the "PNG" format. # Unconditional (not windows-gated) so winfmt's pure-conversion unit tests run on every host. -image = { version = "0.25", default-features = false, features = ["png", "bmp"] } +image = { version = "0.25", default-features = false, features = ["png", "bmp", "jpeg", "gif"] } [target.'cfg(target_os = "linux")'.dependencies] # Mutter's direct RemoteDesktop clipboard is raw D-Bus via `ashpd::zbus` — NOT the xdg diff --git a/crates/pf-clipboard/src/host.rs b/crates/pf-clipboard/src/host.rs index 87f40649..8dbf8bbf 100644 --- a/crates/pf-clipboard/src/host.rs +++ b/crates/pf-clipboard/src/host.rs @@ -235,6 +235,12 @@ pub const WIRE_HTML: &str = "text/html"; pub const WIRE_RTF: &str = "text/rtf"; /// Wire MIME for a PNG image. pub const WIRE_PNG: &str = "image/png"; +/// Wire MIME for a JPEG image — passed through VERBATIM when the source clipboard carries one +/// (no PNG transcode: a lossy original re-encoded lossless is pure bloat). [`WIRE_PNG`] remains +/// the universal fallback every peer must accept; JPEG/GIF are richer options beside it. +pub const WIRE_JPEG: &str = "image/jpeg"; +/// Wire MIME for a GIF image — verbatim pass-through preserves animation end to end. +pub const WIRE_GIF: &str = "image/gif"; /// Map a Wayland selection MIME to its canonical wire MIME, or `None` to drop it (internal targets /// like `TARGETS`/`TIMESTAMP`/`SAVE_TARGETS`, and formats we don't sync in Phase 1). Aliases @@ -248,6 +254,8 @@ pub fn wayland_to_wire(wl: &str) -> Option<&'static str> { "text/html" => Some(WIRE_HTML), "text/rtf" | "application/rtf" | "text/richtext" => Some(WIRE_RTF), "image/png" => Some(WIRE_PNG), + "image/jpeg" => Some(WIRE_JPEG), + "image/gif" => Some(WIRE_GIF), _ => match base { "text/plain" | "UTF8_STRING" | "STRING" | "TEXT" => Some(WIRE_TEXT), _ => None, @@ -270,6 +278,8 @@ pub fn wayland_candidates(wire: &str) -> &'static [&'static str] { WIRE_HTML => &["text/html"], WIRE_RTF => &["text/rtf", "application/rtf", "text/richtext"], WIRE_PNG => &["image/png"], + WIRE_JPEG => &["image/jpeg"], + WIRE_GIF => &["image/gif"], _ => &[], } } @@ -330,6 +340,8 @@ pub fn wayland_offers_for(wire_mimes: &[String]) -> Vec { push("text/rtf"); } WIRE_PNG => push("image/png"), + WIRE_JPEG => push("image/jpeg"), + WIRE_GIF => push("image/gif"), other => push(other), } } diff --git a/crates/pf-clipboard/src/host/windows.rs b/crates/pf-clipboard/src/host/windows.rs index 9efc2901..e24b28ac 100644 --- a/crates/pf-clipboard/src/host/windows.rs +++ b/crates/pf-clipboard/src/host/windows.rs @@ -48,7 +48,9 @@ use ::windows::Win32::UI::WindowsAndMessaging::{ }; use super::winfmt; -use super::{ClipEvent, PasteResponder, WIRE_HTML, WIRE_PNG, WIRE_RTF, WIRE_TEXT}; +use super::{ + ClipEvent, PasteResponder, WIRE_GIF, WIRE_HTML, WIRE_JPEG, WIRE_PNG, WIRE_RTF, WIRE_TEXT, +}; /// Custom app message that wakes the pump to drain the [`Cmd`] channel. const WM_APP_CMD: u32 = WM_APP + 1; @@ -98,6 +100,13 @@ struct WinClip { fmt_html: u32, fmt_rtf: u32, fmt_png: u32, + /// Registered `"JFIF"` — the conventional raw-JPEG clipboard format (Office/browsers). + fmt_jfif: u32, + /// Registered `"GIF"` — raw GIF bytes (animation preserved by verbatim pass-through). + fmt_gif: u32, + /// The wire MIMEs of the offer currently promised via delayed rendering — `WM_RENDERFORMAT` + /// for the synthesized `CF_DIB` picks the richest image kind among them to fetch + convert. + offered_wires: RefCell>, /// Our own message window — used for the owner-check and clipboard opens. own_hwnd: HWND, } @@ -137,6 +146,15 @@ impl WinClip { if avail(self.fmt_png) || avail(CF_DIB.0 as u32) { out.push(WIRE_PNG.to_string()); } + // Original lossy/animated formats offered VERBATIM beside the PNG floor — the client + // picks the richest kind it can place, so a copied JPEG never balloons into PNG and a + // GIF keeps its animation. + if avail(self.fmt_jfif) { + out.push(WIRE_JPEG.to_string()); + } + if avail(self.fmt_gif) { + out.push(WIRE_GIF.to_string()); + } out } @@ -192,6 +210,7 @@ impl WinClip { } } *self.offered.borrow_mut() = fmts; + *self.offered_wires.borrow_mut() = wire.to_vec(); } /// Drop the selection we own (empty the clipboard iff we're still its owner). @@ -265,8 +284,23 @@ impl WinClip { /// `WM_RENDERFORMAT`: a host app is pasting a format we promised. Fetch the bytes from the client /// (blocking this thread, bounded) and `SetClipboardData` them for the paster. fn on_render_format(&self, fmt: u32) { - let Some(wire) = self.wire_for_format(fmt) else { - return; + // The synthesized CF_DIB promise has no wire kind of its own: fetch the richest image + // kind the client offered (PNG first — lossless with alpha — then JPEG, then GIF's first + // frame) and convert. Every other format maps 1:1. + let wire: &str = if fmt == CF_DIB.0 as u32 { + let offered = self.offered_wires.borrow(); + match [WIRE_PNG, WIRE_JPEG, WIRE_GIF] + .into_iter() + .find(|w| offered.iter().any(|o| o == w)) + { + Some(w) => w, + None => return, + } + } else { + match self.wire_for_format(fmt) { + Some(w) => w, + None => return, + } }; let (tx, rx) = std::sync::mpsc::channel::>(); let ev = ClipEvent::Paste { @@ -281,9 +315,9 @@ impl WinClip { Err(_) => return, // timeout / dropped → leave the format unrendered (empty paste) }; let win_bytes = if fmt == CF_DIB.0 as u32 { - // The app asked for a bitmap: the client served PNG — convert. A PNG that doesn't - // decode leaves the format unrendered (empty paste), matching the timeout path. - match winfmt::png_to_dib(&bytes) { + // The app asked for a bitmap: convert whatever image kind the client served. Bytes + // that don't decode leave the format unrendered (empty paste), like the timeout path. + match winfmt::image_to_dib(&bytes) { Some(d) => d, None => return, } @@ -310,6 +344,8 @@ impl WinClip { WIRE_HTML => Some(self.fmt_html), WIRE_RTF => Some(self.fmt_rtf), WIRE_PNG => Some(self.fmt_png), + WIRE_JPEG => Some(self.fmt_jfif), + WIRE_GIF => Some(self.fmt_gif), _ => None, } } @@ -322,8 +358,12 @@ impl WinClip { Some(WIRE_HTML) } else if fmt == self.fmt_rtf { Some(WIRE_RTF) - } else if fmt == self.fmt_png || fmt == CF_DIB.0 as u32 { + } else if fmt == self.fmt_png { Some(WIRE_PNG) + } else if fmt == self.fmt_jfif { + Some(WIRE_JPEG) + } else if fmt == self.fmt_gif { + Some(WIRE_GIF) } else { None } @@ -340,9 +380,12 @@ impl WinClip { } } // Image offers also promise CF_DIB — most pasting apps (Paint, Office, chat clients) - // ask for the bitmap family, not the registered "PNG"; Windows synthesizes - // CF_BITMAP/CF_DIBV5 from the promised CF_DIB. `on_render_format` converts on demand. - if w == WIRE_PNG && !out.contains(&(CF_DIB.0 as u32)) { + // ask for the bitmap family, not the registered image formats; Windows synthesizes + // CF_BITMAP/CF_DIBV5 from the promised CF_DIB. `on_render_format` fetches the richest + // offered image kind and converts on demand. + if matches!(w.as_str(), WIRE_PNG | WIRE_JPEG | WIRE_GIF) + && !out.contains(&(CF_DIB.0 as u32)) + { out.push(CF_DIB.0 as u32); } } @@ -376,12 +419,18 @@ impl WindowsClipboard { let fmt_html = register_format(w!("HTML Format"))?; let fmt_rtf = register_format(w!("Rich Text Format"))?; let fmt_png = register_format(w!("PNG"))?; + let fmt_jfif = register_format(w!("JFIF"))?; + let fmt_gif = register_format(w!("GIF"))?; let (ready_tx, ready_rx) = tokio::sync::oneshot::channel::>(); let cw = Arc::clone(¤t_wire); let join = std::thread::Builder::new() .name("punktfunk-clipboard-win".into()) - .spawn(move || pump_thread(clip_tx, cmd_rx, cw, fmt_html, fmt_rtf, fmt_png, ready_tx)) + .spawn(move || { + pump_thread( + clip_tx, cmd_rx, cw, fmt_html, fmt_rtf, fmt_png, fmt_jfif, fmt_gif, ready_tx, + ) + }) .context("spawn windows clipboard thread")?; let hwnd = match tokio::time::timeout(Duration::from_secs(3), ready_rx).await { @@ -573,6 +622,7 @@ fn create_window() -> anyhow::Result { } /// The message-loop thread body: build the window, wire up state, then pump until `WM_QUIT`. +#[allow(clippy::too_many_arguments)] fn pump_thread( clip_tx: ClipTx, cmd_rx: tokio::sync::mpsc::UnboundedReceiver, @@ -580,6 +630,8 @@ fn pump_thread( fmt_html: u32, fmt_rtf: u32, fmt_png: u32, + fmt_jfif: u32, + fmt_gif: u32, ready_tx: tokio::sync::oneshot::Sender>, ) { let hwnd = match create_window() { @@ -598,9 +650,12 @@ fn pump_thread( current_wire, cmd_rx: RefCell::new(cmd_rx), offered: RefCell::new(Vec::new()), + offered_wires: RefCell::new(Vec::new()), fmt_html, fmt_rtf, fmt_png, + fmt_jfif, + fmt_gif, own_hwnd: hwnd, }); let ptr = Box::into_raw(state); diff --git a/crates/pf-clipboard/src/host/winfmt.rs b/crates/pf-clipboard/src/host/winfmt.rs index e284bf8d..0eb6d8dd 100644 --- a/crates/pf-clipboard/src/host/winfmt.rs +++ b/crates/pf-clipboard/src/host/winfmt.rs @@ -265,7 +265,7 @@ mod tests { image::DynamicImage::ImageRgba8(img.clone()) .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) .unwrap(); - let dib = png_to_dib(&png).expect("png -> dib"); + let dib = image_to_dib(&png).expect("png -> dib"); // BITMAPINFOHEADER sanity: 40-byte header, 3x2, 32bpp. assert_eq!(u32::from_le_bytes(dib[0..4].try_into().unwrap()), 40); assert_eq!(i32::from_le_bytes(dib[4..8].try_into().unwrap()), 3); @@ -290,10 +290,11 @@ mod tests { // the backend converts. A CF_DIB HGLOBAL is a BMP file minus its 14-byte BITMAPFILEHEADER: // BITMAPINFOHEADER (or V4/V5) + optional palette/masks + pixel rows. -/// PNG wire bytes → `CF_DIB` HGLOBAL bytes (BITMAPINFOHEADER, 32bpp BGRA, BI_RGB, bottom-up). -/// `None` when the PNG doesn't decode — the caller leaves the format unrendered (empty paste). -pub fn png_to_dib(png: &[u8]) -> Option> { - let img = image::load_from_memory_with_format(png, image::ImageFormat::Png).ok()?; +/// Image wire bytes (PNG / JPEG / GIF — any format the `image` crate sniffs) → `CF_DIB` HGLOBAL +/// bytes (BITMAPINFOHEADER, 32bpp BGRA, BI_RGB, bottom-up). GIFs contribute their first frame. +/// `None` when the bytes don't decode — the caller leaves the format unrendered (empty paste). +pub fn image_to_dib(bytes: &[u8]) -> Option> { + let img = image::load_from_memory(bytes).ok()?; let rgba = img.to_rgba8(); let (w, h) = (rgba.width() as usize, rgba.height() as usize); if w == 0 || h == 0 || w > 32767 || h > 32767 {