feat(apple): cursor channel on the Mac client — ABI v11 + NSCursor rendering

The Mac client now exercises the full cursor channel against capable
hosts: desktop-mode sessions advertise CLIENT_CAP_CURSOR, the host
stops compositing the pointer, and StreamView draws the forwarded
shapes as the real NSCursor — plus the M3 host-driven model flip.

- ABI v11: punktfunk_connect_ex9 (adds client_caps; ex7/ex8 pass 0 —
  Android untouched), PunktfunkCursorShape/State + the two
  next_cursor_* poll fns (audio-style borrow contract), and
  PUNKTFUNK_CLIENT_CAP_CURSOR. Fixed in passing: the first insertion
  split next_host_timing's cfg/no_mangle attributes off the fn, which
  silently dropped the symbol from the header AND dylib — caught by
  the Swift build, reattached with its docs.
- PunktfunkConnection: clientCaps connect param, hostSupportsCursor,
  CursorShapeEvent/CursorStateEvent + nextCursorShape/nextCursorState
  (one cursor thread drains both planes; cursorLock joins the close()
  ladder).
- SessionModel: desktop-mode sessions (DefaultsKey.mouseMode) connect
  with the cursor cap on macOS.
- StreamView: shape cache (serial → NSCursor, straight-alpha RGBA via
  CGImage), resetCursorRects wears the HOST shape while the desktop
  model is engaged (hidden host pointer ⇒ invisible), latest-wins
  state pump, and the M3 auto-flip — edge-triggered on relative_hint,
  ⌃⌥⇧M sets an override latch cleared by the next host edge, leaving
  relative warps the pointer to the host position via the inverse
  letterbox mapping (CGWarpMouseCursorPosition, CursorCapture's
  coordinate convention).

Verified: swift build + full test suites green (xcframework rebuilt at
ABI v11, signed); core 218 tests + clippy -D warnings on Linux (.21).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-22 23:43:23 +02:00
co-authored by Claude Fable 5
parent 3a34440a6b
commit ec8ca9a535
6 changed files with 531 additions and 18 deletions
@@ -302,13 +302,26 @@ final class SessionModel: ObservableObject {
if preferredCodec == PunktfunkConnection.codecPyroWave, MetalWaveletDecoder.supported { if preferredCodec == PunktfunkConnection.codecPyroWave, MetalWaveletDecoder.supported {
videoCodecs |= PunktfunkConnection.codecPyroWave videoCodecs |= PunktfunkConnection.codecPyroWave
} }
// Cursor channel (remote-desktop-sweep M2, macOS): sessions STARTING in the desktop
// mouse model advertise local cursor rendering the host then stops compositing
// the pointer and forwards shape/state, which StreamView draws as the real
// NSCursor. Capture-mode sessions keep today's composited pointer.
#if os(macOS)
let clientCaps: UInt8 =
(MouseInputMode(
rawValue: UserDefaults.standard.string(forKey: DefaultsKey.mouseMode) ?? "")
?? .capture) == .desktop ? 0x01 : 0
#else
let clientCaps: UInt8 = 0
#endif
let result = Result { try PunktfunkConnection( let result = Result { try PunktfunkConnection(
host: host.address, port: host.port, host: host.address, port: host.port,
width: width, height: height, refreshHz: hz, width: width, height: height, refreshHz: hz,
pinSHA256: pin, identity: identity, compositor: compositor, pinSHA256: pin, identity: identity, compositor: compositor,
gamepad: gamepad, bitrateKbps: bitrateKbps, videoCaps: videoCaps, gamepad: gamepad, bitrateKbps: bitrateKbps, videoCaps: videoCaps,
audioChannels: audioChannels, audioChannels: audioChannels,
videoCodecs: videoCodecs, preferredCodec: preferredCodec, launchID: launchID, videoCodecs: videoCodecs, preferredCodec: preferredCodec,
clientCaps: clientCaps, launchID: launchID,
// Delegated approval: the host holds this connect open until the operator approves // Delegated approval: the host holds this connect open until the operator approves
// it (~180 s) outwait that window so a slow approval still lands here. Normal // it (~180 s) outwait that window so a slow approval still lands here. Normal
// connects keep the snappy default. // connects keep the snappy default.
@@ -221,6 +221,9 @@ public final class PunktfunkConnection {
/// core). The clip *sends* (`clipControl`/`clipOffer`/`clipServe`) share this lock too: /// core). The clip *sends* (`clipControl`/`clipOffer`/`clipServe`) share this lock too:
/// they're quick non-blocking enqueues, and a single lock keeps close() ordering simple. /// they're quick non-blocking enqueues, and a single lock keeps close() ordering simple.
private let clipboardLock = NSLock() private let clipboardLock = NSLock()
/// Serializes the (single) cursor pull thread against close() both cursor planes are
/// drained by ONE thread, so one lock covers them.
private let cursorLock = NSLock()
/// Negotiated session mode (host-confirmed). /// Negotiated session mode (host-confirmed).
public private(set) var width: UInt32 = 0 public private(set) var width: UInt32 = 0
@@ -381,6 +384,84 @@ public final class PunktfunkConnection {
public var hostSupportsClipboard: Bool { public var hostSupportsClipboard: Bool {
hostCaps & UInt8(PUNKTFUNK_HOST_CAP_CLIPBOARD) != 0 hostCaps & UInt8(PUNKTFUNK_HOST_CAP_CLIPBOARD) != 0
} }
/// The host answered `HOST_CAP_CURSOR`: it stopped compositing the pointer and forwards
/// shape/state on the cursor planes the client MUST draw the cursor locally.
public var hostSupportsCursor: Bool {
hostCaps & 0x04 != 0
}
/// One forwarded host-cursor shape (the cursor channel, ABI v11): straight-alpha RGBA,
/// `rgba.count == width * height * 4`, hotspot within the bitmap. Cache by `serial`
/// states reference shapes by it and a re-shown serial never resends pixels.
public struct CursorShapeEvent: Sendable {
public let serial: UInt32
public let width: Int
public let height: Int
public let hotX: Int
public let hotY: Int
public let rgba: Data
}
/// Per-host-tick cursor state: position (host video px, the pointer/hotspot point),
/// visibility, and the host-driven relative-mode hint (an app grabbed/hid the pointer
/// run captured relative; clear absolute, reappearing at `x`/`y`). Latest-wins.
public struct CursorStateEvent: Sendable {
public let serial: UInt32
public let visible: Bool
public let relativeHint: Bool
public let x: Int32
public let y: Int32
}
/// Pull the next forwarded cursor SHAPE (nil = timeout). Only a session connected with
/// `clientCaps` cursor bit against a `hostSupportsCursor` host receives any. Drain shape
/// AND state from ONE dedicated cursor thread (they share a lock).
public func nextCursorShape(timeoutMs: UInt32 = 0) throws -> CursorShapeEvent? {
cursorLock.lock()
defer { cursorLock.unlock() }
guard let h = liveHandle() else { throw PunktfunkClientError.closed }
var out = PunktfunkCursorShape()
let rc = punktfunk_connection_next_cursor_shape(h, &out, timeoutMs)
switch rc {
case statusOK:
// Copy out of the ABI borrow (valid until the next shape call) immediately.
let bytes = out.rgba.map { Data(bytes: $0, count: Int(out.len)) } ?? Data()
return CursorShapeEvent(
serial: out.serial, width: Int(out.w), height: Int(out.h),
hotX: Int(out.hot_x), hotY: Int(out.hot_y), rgba: bytes)
case statusNoFrame:
return nil
case statusClosed:
throw PunktfunkClientError.closed
default:
throw PunktfunkClientError.status(rc)
}
}
/// Pull the next cursor STATE (nil = timeout). Latest-wins drain the queue and apply
/// only the newest. Same thread + gate as [`nextCursorShape`].
public func nextCursorState(timeoutMs: UInt32 = 0) throws -> CursorStateEvent? {
cursorLock.lock()
defer { cursorLock.unlock() }
guard let h = liveHandle() else { throw PunktfunkClientError.closed }
var out = PunktfunkCursorState()
let rc = punktfunk_connection_next_cursor_state(h, &out, timeoutMs)
switch rc {
case statusOK:
return CursorStateEvent(
serial: out.serial,
visible: out.flags & 0x01 != 0,
relativeHint: out.flags & 0x02 != 0,
x: out.x, y: out.y)
case statusNoFrame:
return nil
case statusClosed:
throw PunktfunkClientError.closed
default:
throw PunktfunkClientError.status(rc)
}
}
/// The resolved codec as a `VideoCodec` (H.264 / HEVC / AV1) drives the bitstream framing /// The resolved codec as a `VideoCodec` (H.264 / HEVC / AV1) drives the bitstream framing
/// (Annex-B NAL parsing vs the AV1 OBU repack). /// (Annex-B NAL parsing vs the AV1 OBU repack).
public var videoCodec: VideoCodec { VideoCodec(wire: resolvedCodec) } public var videoCodec: VideoCodec { VideoCodec(wire: resolvedCodec) }
@@ -417,6 +498,7 @@ public final class PunktfunkConnection {
audioChannels: UInt8 = 2, audioChannels: UInt8 = 2,
videoCodecs: UInt8 = 0x02, // PUNKTFUNK_CODEC_HEVC the codecs this client can decode videoCodecs: UInt8 = 0x02, // PUNKTFUNK_CODEC_HEVC the codecs this client can decode
preferredCodec: UInt8 = 0, // 0 = auto; else PUNKTFUNK_CODEC_* soft preference preferredCodec: UInt8 = 0, // 0 = auto; else PUNKTFUNK_CODEC_* soft preference
clientCaps: UInt8 = 0, // ABI v11: PUNKTFUNK_CLIENT_CAP_CURSOR = render the host cursor locally
launchID: String? = nil, launchID: String? = nil,
timeoutMs: UInt32 = 10_000 timeoutMs: UInt32 = 10_000
) throws { ) throws {
@@ -436,18 +518,18 @@ public final class PunktfunkConnection {
withOptionalCString(launchID) { launch in withOptionalCString(launchID) { launch in
if let pin = pinSHA256 { if let pin = pinSHA256 {
return pin.withUnsafeBytes { p in return pin.withUnsafeBytes { p in
punktfunk_connect_ex8( punktfunk_connect_ex9(
cs, port, width, height, refreshHz, compositor.rawValue, cs, port, width, height, refreshHz, compositor.rawValue,
gamepad.rawValue, bitrateKbps, videoCaps, audioChannels, gamepad.rawValue, bitrateKbps, videoCaps, audioChannels,
videoCodecs, preferredCodec, launch, videoCodecs, preferredCodec, clientCaps, launch,
p.bindMemory(to: UInt8.self).baseAddress, &observed, p.bindMemory(to: UInt8.self).baseAddress, &observed,
cert, key, timeoutMs, &connectStatus) cert, key, timeoutMs, &connectStatus)
} }
} }
return punktfunk_connect_ex8( return punktfunk_connect_ex9(
cs, port, width, height, refreshHz, compositor.rawValue, cs, port, width, height, refreshHz, compositor.rawValue,
gamepad.rawValue, bitrateKbps, videoCaps, audioChannels, gamepad.rawValue, bitrateKbps, videoCaps, audioChannels,
videoCodecs, preferredCodec, launch, videoCodecs, preferredCodec, clientCaps, launch,
nil, &observed, cert, key, timeoutMs, &connectStatus) nil, &observed, cert, key, timeoutMs, &connectStatus)
} }
} }
@@ -1059,10 +1141,12 @@ public final class PunktfunkConnection {
feedbackLock.lock() feedbackLock.lock()
statsLock.lock() statsLock.lock()
clipboardLock.lock() clipboardLock.lock()
cursorLock.lock()
abiLock.lock() abiLock.lock()
let h = handle let h = handle
handle = nil handle = nil
abiLock.unlock() abiLock.unlock()
cursorLock.unlock()
clipboardLock.unlock() clipboardLock.unlock()
statsLock.unlock() statsLock.unlock()
feedbackLock.unlock() feedbackLock.unlock()
@@ -219,6 +219,16 @@ public final class StreamLayerView: NSView {
/// flipped live by M. A live flip re-engages capture in the new model so /// flipped live by M. A live flip re-engages capture in the new model so
/// disassociation + the abs/rel choice swap atomically. Main-thread only. /// disassociation + the abs/rel choice swap atomically. Main-thread only.
private var desktopMouse = false private var desktopMouse = false
/// Cursor channel (M2): the host forwards shape/state and WE draw the pointer. Active
/// when the Welcome carried `HOST_CAP_CURSOR` (only sessions that advertised the client
/// cap get it). Shapes cache by serial; state is latest-wins. Main-thread only.
private var cursorChannelActive = false
private var hostCursors: [UInt32: NSCursor] = [:]
private var cursorState: PunktfunkConnection.CursorStateEvent?
/// M3 hint tracking: edge-triggered so a manual M isn't fought the override latch
/// holds until the HOST's intent next changes.
private var lastHint: Bool?
private var hintOverride = false
/// One-shot auto-engage request (stream start, trust confirmed) attempted as soon /// One-shot auto-engage request (stream start, trust confirmed) attempted as soon
/// as the view is in a window with real bounds, then dropped, so it can never fire /// as the view is in a window with real bounds, then dropped, so it can never fire
/// surprisingly later (e.g. on a resize). /// surprisingly later (e.g. on a resize).
@@ -480,12 +490,136 @@ public final class StreamLayerView: NSView {
/// globally via `CursorCapture` (the pointer can't leave the view there). /// globally via `CursorCapture` (the pointer can't leave the view there).
override public func resetCursorRects() { override public func resetCursorRects() {
if captured && desktopMouse { if captured && desktopMouse {
addCursorRect(bounds, cursor: Self.invisibleCursor) // Cursor channel active: wear the HOST's pointer shape (it is no longer in the
// video); hidden host pointer (or no shape yet) = invisible. Without the channel,
// M1 behavior: invisible local cursor, the composited host cursor is the visible one.
if cursorChannelActive, let st = cursorState, st.visible,
let host = hostCursors[st.serial] {
addCursorRect(bounds, cursor: host)
} else {
addCursorRect(bounds, cursor: Self.invisibleCursor)
}
} else { } else {
super.resetCursorRects() super.resetCursorRects()
} }
} }
/// Flip the mouse model with the atomic release/re-engage swap; `reappearAt` (host video
/// px the M3 hand-back position) warps the local pointer so leaving relative lands the
/// cursor exactly where the host last had it.
private func setDesktopMouse(_ on: Bool, reappearAt: (x: Int32, y: Int32)?) {
guard desktopMouse != on else { return }
let wasCaptured = captured
if wasCaptured { releaseCapture() }
desktopMouse = on
if wasCaptured { engageCapture(fromClick: false) }
window?.invalidateCursorRects(for: self)
if on, let p = reappearAt, let sp = cgScreenPoint(forHostX: p.x, p.y) {
CGWarpMouseCursorPosition(sp)
}
}
/// The single cursor pull thread (both planes share the connection's cursor lock):
/// latest-wins state at a short timeout + a non-blocking shape poll per iteration.
/// Exits when the connection closes; events hop to main where all cursor state lives.
private func startCursorPump(_ connection: PunktfunkConnection) {
let thread = Thread { [weak self] in
while true {
do {
var newest: PunktfunkConnection.CursorStateEvent?
if let st = try connection.nextCursorState(timeoutMs: 100) {
newest = st
while let more = try connection.nextCursorState(timeoutMs: 0) {
newest = more // drain latest wins
}
}
while let shape = try connection.nextCursorShape(timeoutMs: 0) {
DispatchQueue.main.async { self?.applyCursorShape(shape) }
}
if let st = newest {
DispatchQueue.main.async { self?.applyCursorState(st) }
}
} catch {
return // connection closed the session is over
}
if self == nil { return }
}
}
thread.name = "pf-cursor-pump"
thread.start()
}
private func applyCursorShape(_ ev: PunktfunkConnection.CursorShapeEvent) {
guard let cursor = Self.makeCursor(ev) else {
streamInputLog.warning("cursor shape rejected (\(ev.width)x\(ev.height)) — keeping the previous cursor")
return
}
if hostCursors.count >= 64 { hostCursors.removeAll() } // degenerate host: reset
hostCursors[ev.serial] = cursor
if cursorState?.serial == ev.serial {
window?.invalidateCursorRects(for: self)
}
}
private func applyCursorState(_ ev: PunktfunkConnection.CursorStateEvent) {
let prev = cursorState
cursorState = ev
if prev?.visible != ev.visible || prev?.serial != ev.serial {
window?.invalidateCursorRects(for: self)
}
// M3 host-driven mode flip, edge-triggered (a fresh host intent clears a manual
// override): hint captured relative; clear absolute, reappearing at the host's
// last pointer position.
if lastHint != ev.relativeHint {
lastHint = ev.relativeHint
hintOverride = false
}
if !hintOverride, captured, desktopMouse == ev.relativeHint {
setDesktopMouse(!ev.relativeHint, reappearAt: ev.relativeHint ? nil : (ev.x, ev.y))
streamInputLog.info("host cursor hint: mouse model flipped to \(self.desktopMouse ? "desktop" : "capture", privacy: .public)")
}
}
/// Build an `NSCursor` from a forwarded straight-alpha RGBA shape.
private static func makeCursor(_ ev: PunktfunkConnection.CursorShapeEvent) -> NSCursor? {
let (w, h) = (ev.width, ev.height)
guard w > 0, h > 0, ev.rgba.count >= w * h * 4,
let provider = CGDataProvider(data: ev.rgba as CFData),
let cg = CGImage(
width: w, height: h, bitsPerComponent: 8, bitsPerPixel: 32,
bytesPerRow: w * 4, space: CGColorSpaceCreateDeviceRGB(),
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.last.rawValue),
provider: provider, decode: nil, shouldInterpolate: false,
intent: .defaultIntent)
else { return nil }
let image = NSImage(cgImage: cg, size: NSSize(width: w, height: h))
return NSCursor(
image: image,
hotSpot: NSPoint(x: min(ev.hotX, w - 1), y: min(ev.hotY, h - 1)))
}
/// Host video px CG GLOBAL screen coordinates (top-left origin, the
/// `CGWarpMouseCursorPosition` convention `CursorCapture` established) through the
/// aspect-fit letterbox the inverse direction of `hostPoint(from:)`.
private func cgScreenPoint(forHostX hx: Int32, _ hy: Int32) -> CGPoint? {
guard let connection, let window else { return nil }
let mode = connection.currentMode()
guard mode.width > 0, mode.height > 0 else { return nil }
let fit = AVMakeRect(
aspectRatio: CGSize(width: Int(mode.width), height: Int(mode.height)),
insideRect: bounds)
guard fit.width > 0, fit.height > 0 else { return nil }
let u = (CGFloat(hx) / CGFloat(mode.width)).clamped(to: 0...1)
let v = (CGFloat(hy) / CGFloat(mode.height)).clamped(to: 0...1)
let videoMinYTop = bounds.height - fit.maxY
let pTop = CGPoint(x: fit.minX + u * fit.width, y: videoMinYTop + v * fit.height)
let inView = CGPoint(x: pTop.x, y: bounds.height - pTop.y)
let inWindow = convert(inView, to: nil)
let onScreen = window.convertPoint(toScreen: inWindow)
let primaryHeight = NSScreen.screens.first?.frame.height ?? 0
return CGPoint(x: onScreen.x, y: primaryHeight - onScreen.y)
}
/// A single local monitor for motion + buttons, installed only while captured. A local /// A single local monitor for motion + buttons, installed only while captured. A local
/// monitor is more robust than view overrides for relative motion: it sidesteps the /// monitor is more robust than view overrides for relative motion: it sidesteps the
/// `window.acceptsMouseMovedEvents`/tracking-area/responder-chain requirements, and /// `window.acceptsMouseMovedEvents`/tracking-area/responder-chain requirements, and
@@ -647,12 +781,10 @@ public final class StreamLayerView: NSView {
streamInputLog.info("mouse-mode chord ignored: gamescope host is relative-only") streamInputLog.info("mouse-mode chord ignored: gamescope host is relative-only")
return return
} }
let wasCaptured = self.captured // A manual flip outranks the standing host hint until the hint next CHANGES.
if wasCaptured { self.releaseCapture() } self.hintOverride = true
self.desktopMouse.toggle() self.setDesktopMouse(!self.desktopMouse, reappearAt: nil)
if wasCaptured { self.engageCapture(fromClick: false) } streamInputLog.info("chord: mouse mode \(self.desktopMouse ? "desktop" : "capture", privacy: .public)")
self.window?.invalidateCursorRects(for: self)
streamInputLog.info("mouse mode: \(self.desktopMouse ? "desktop" : "capture", privacy: .public)")
} }
// The cross-client combos (Q/D/S Ctrl+Alt+Shift on the other clients), delivered by // The cross-client combos (Q/D/S Ctrl+Alt+Shift on the other clients), delivered by
// the monitor only while captured; the same key-window ownership rule as throughout. // the monitor only while captured; the same key-window ownership rule as throughout.
@@ -692,6 +824,13 @@ public final class StreamLayerView: NSView {
if mode == .desktop && !absOK { if mode == .desktop && !absOK {
streamInputLog.info("desktop mouse mode unavailable on a gamescope host (relative-only) — using capture") streamInputLog.info("desktop mouse mode unavailable on a gamescope host (relative-only) — using capture")
} }
// Cursor channel (M2): the host stopped compositing the pointer drain its shape/
// state planes and draw the pointer as the real NSCursor (plus the M3 auto-flip).
if connection.hostSupportsCursor {
cursorChannelActive = true
streamInputLog.info("cursor channel negotiated — host cursor renders locally")
startCursorPump(connection)
}
// Presenter choice + lifecycle live in SessionPresenter (shared with iOS/tvOS): stage-2 // Presenter choice + lifecycle live in SessionPresenter (shared with iOS/tvOS): stage-2
// (explicit VTDecompressionSession decode + a CAMetalLayer/display-link present) by // (explicit VTDecompressionSession decode + a CAMetalLayer/display-link present) by
+193 -4
View File
@@ -555,6 +555,9 @@ pub struct PunktfunkConnection {
/// (a fetched payload, an offer's format list, or a fetch-request's MIME) — /// (a fetched payload, an offer's format list, or a fetch-request's MIME) —
/// borrow-until-next-call, same contract as `last`. /// borrow-until-next-call, same contract as `last`.
last_clip: std::sync::Mutex<Option<Vec<u8>>>, last_clip: std::sync::Mutex<Option<Vec<u8>>>,
/// The last cursor shape handed out — `next_cursor_shape`'s `rgba` pointer borrows it
/// until the next cursor-shape call (the `last_audio` contract).
last_cursor_shape: std::sync::Mutex<Option<crate::quic::CursorShape>>,
} }
/// Lazily-initialized in-core Opus decode state. A coupled-1-stream multistream decoder is /// Lazily-initialized in-core Opus decode state. A coupled-1-stream multistream decoder is
@@ -1400,6 +1403,7 @@ pub unsafe extern "C" fn punktfunk_connect_ex7(
connect_ex_impl( connect_ex_impl(
host, host,
port, port,
0, // pre-v11 variant: no client caps
width, width,
height, height,
refresh_hz, refresh_hz,
@@ -1459,6 +1463,7 @@ pub unsafe extern "C" fn punktfunk_connect_ex8(
connect_ex_impl( connect_ex_impl(
host, host,
port, port,
0, // pre-v11 variant: no client caps
width, width,
height, height,
refresh_hz, refresh_hz,
@@ -1480,6 +1485,70 @@ pub unsafe extern "C" fn punktfunk_connect_ex8(
} }
} }
/// Like [`punktfunk_connect_ex8`], plus `client_caps` (ABI v11): a bitfield of
/// `PUNKTFUNK_CLIENT_CAP_CURSOR` (0x01). Setting the cursor bit asks the host to STOP
/// compositing the pointer into the video and forward it out-of-band instead — the embedder
/// MUST then drain [`punktfunk_connection_next_cursor_shape`] /
/// [`punktfunk_connection_next_cursor_state`] and draw the pointer itself, or the session has
/// no visible cursor at all. Pass 0 for the composited behavior of every earlier variant.
///
/// # Safety
/// Same as [`punktfunk_connect_ex8`].
#[cfg(feature = "quic")]
#[no_mangle]
#[allow(clippy::too_many_arguments)]
pub unsafe extern "C" fn punktfunk_connect_ex9(
host: *const std::os::raw::c_char,
port: u16,
width: u32,
height: u32,
refresh_hz: u32,
compositor: u32,
gamepad: u32,
bitrate_kbps: u32,
video_caps: u8,
audio_channels: u8,
video_codecs: u8,
preferred_codec: u8,
client_caps: u8,
launch_id: *const std::os::raw::c_char,
pin_sha256: *const u8,
observed_sha256_out: *mut u8,
client_cert_pem: *const std::os::raw::c_char,
client_key_pem: *const std::os::raw::c_char,
timeout_ms: u32,
status_out: *mut i32,
) -> *mut PunktfunkConnection {
unsafe {
connect_ex_impl(
host,
port,
client_caps,
width,
height,
refresh_hz,
compositor,
gamepad,
bitrate_kbps,
video_caps,
audio_channels,
video_codecs,
preferred_codec,
launch_id,
pin_sha256,
observed_sha256_out,
client_cert_pem,
client_key_pem,
timeout_ms,
status_out,
)
}
}
/// [`punktfunk_connect_ex9`] `client_caps` bit: render the host cursor locally (the cursor
/// channel, `design/remote-desktop-sweep.md` M2).
pub const PUNKTFUNK_CLIENT_CAP_CURSOR: u8 = 0x01;
/// Shared body of [`punktfunk_connect_ex7`] / [`punktfunk_connect_ex8`]: `status_out` /// Shared body of [`punktfunk_connect_ex7`] / [`punktfunk_connect_ex8`]: `status_out`
/// (nullable) is written on EVERY path — `Ok`, the mapped [`PunktfunkError`], /// (nullable) is written on EVERY path — `Ok`, the mapped [`PunktfunkError`],
/// `InvalidArg` for bad arguments, `Panic` if the connect panicked. /// `InvalidArg` for bad arguments, `Panic` if the connect panicked.
@@ -1488,6 +1557,7 @@ pub unsafe extern "C" fn punktfunk_connect_ex8(
unsafe fn connect_ex_impl( unsafe fn connect_ex_impl(
host: *const std::os::raw::c_char, host: *const std::os::raw::c_char,
port: u16, port: u16,
client_caps: u8,
width: u32, width: u32,
height: u32, height: u32,
refresh_hz: u32, refresh_hz: u32,
@@ -1574,10 +1644,10 @@ unsafe fn connect_ex_impl(
// themselves (EDR / MediaCodec), so the host's EDID defaults are fine there. An `ex8` // themselves (EDR / MediaCodec), so the host's EDID defaults are fine there. An `ex8`
// variant can carry it if a passthrough embedder ever needs it. // variant can carry it if a passthrough embedder ever needs it.
None, None,
// No client_caps in the C ABI yet either: cursor-channel opt-in for Apple/Android // ABI v11 ([`punktfunk_connect_ex9`]): CLIENT_CAP_CURSOR here asks the host to STOP
// arrives with the ABI v11 cursor poll fns — until an embedder can RENDER the // compositing the pointer — only an embedder that renders the cursor planes
// forwarded cursor it must not ask the host to stop compositing it. // ([`punktfunk_connection_next_cursor_shape`]/`_state`) may set it. ex7/ex8 pass 0.
0, client_caps,
launch, launch,
pin, pin,
identity, identity,
@@ -1597,6 +1667,7 @@ unsafe fn connect_ex_impl(
last_audio: std::sync::Mutex::new(None), last_audio: std::sync::Mutex::new(None),
audio_pcm: std::sync::Mutex::new(AudioPcmState::default()), audio_pcm: std::sync::Mutex::new(AudioPcmState::default()),
last_clip: std::sync::Mutex::new(None), last_clip: std::sync::Mutex::new(None),
last_cursor_shape: std::sync::Mutex::new(None),
})) }))
} }
Err(e) => { Err(e) => {
@@ -2246,6 +2317,124 @@ pub unsafe extern "C" fn punktfunk_connection_next_hdr_meta(
}) })
} }
/// One forwarded host-cursor shape (ABI v11, the cursor channel): straight-alpha RGBA8, no
/// padding, `len == w * h * 4`, hotspot within `w`×`h`. `serial` is the identity
/// [`PunktfunkCursorState`] refers to — cache the built OS cursor by it.
#[repr(C)]
pub struct PunktfunkCursorShape {
pub serial: u32,
pub w: u16,
pub h: u16,
pub hot_x: u16,
pub hot_y: u16,
/// Borrows connection memory until the NEXT cursor-shape call (the audio contract).
pub rgba: *const u8,
pub len: usize,
}
/// Per-frame host-cursor state (ABI v11): position (the pointer/hotspot point in the host
/// video's pixel space), visibility, and the host-driven relative-mode hint. `flags` bit 0 =
/// visible, bit 1 = relative hint (a host app grabbed/hid the pointer — run captured
/// relative; clear = return to absolute, reappearing at `x`/`y`).
#[repr(C)]
pub struct PunktfunkCursorState {
pub serial: u32,
pub flags: u8,
pub x: i32,
pub y: i32,
}
/// Pull the next forwarded cursor SHAPE (sent on pointer-bitmap change over the reliable
/// control stream; only a session connected with `PUNKTFUNK_CLIENT_CAP_CURSOR` against a
/// capable host receives any). On `Ok`, `out->rgba` borrows connection memory until the next
/// cursor-shape call on this handle. Drain from a dedicated thread (one thread per plane).
///
/// # Safety
/// `c` is a valid connection handle; `out` is writable. At most one thread pulls cursor
/// shapes; it may run concurrently with every other plane's puller.
#[cfg(feature = "quic")]
#[no_mangle]
pub unsafe extern "C" fn punktfunk_connection_next_cursor_shape(
c: *mut PunktfunkConnection,
out: *mut PunktfunkCursorShape,
timeout_ms: u32,
) -> PunktfunkStatus {
guard(|| {
let c = match unsafe { c.as_ref() } {
Some(c) => c,
None => return PunktfunkStatus::NullPointer,
};
if out.is_null() {
return PunktfunkStatus::NullPointer;
}
match c
.inner
.next_cursor_shape(std::time::Duration::from_millis(timeout_ms as u64))
{
Ok(shape) => {
let mut slot = c.last_cursor_shape.lock().unwrap();
*slot = Some(shape);
let sh = slot.as_ref().unwrap();
unsafe {
*out = PunktfunkCursorShape {
serial: sh.serial,
w: sh.w,
h: sh.h,
hot_x: sh.hot_x,
hot_y: sh.hot_y,
rgba: sh.rgba.as_ptr(),
len: sh.rgba.len(),
};
}
PunktfunkStatus::Ok
}
Err(e) => e.status(),
}
})
}
/// Pull the next cursor STATE (a `0xD0` datagram per host encode tick — latest-wins; drain
/// the queue and apply only the newest). Same negotiation gate as
/// [`punktfunk_connection_next_cursor_shape`].
///
/// # Safety
/// `c` is a valid connection handle; `out` is writable. At most one thread pulls cursor
/// state; it may run concurrently with every other plane's puller.
#[cfg(feature = "quic")]
#[no_mangle]
pub unsafe extern "C" fn punktfunk_connection_next_cursor_state(
c: *mut PunktfunkConnection,
out: *mut PunktfunkCursorState,
timeout_ms: u32,
) -> PunktfunkStatus {
guard(|| {
let c = match unsafe { c.as_ref() } {
Some(c) => c,
None => return PunktfunkStatus::NullPointer,
};
if out.is_null() {
return PunktfunkStatus::NullPointer;
}
match c
.inner
.next_cursor_state(std::time::Duration::from_millis(timeout_ms as u64))
{
Ok(st) => {
unsafe {
*out = PunktfunkCursorState {
serial: st.serial,
flags: st.flags,
x: st.x,
y: st.y,
};
}
PunktfunkStatus::Ok
}
Err(e) => e.status(),
}
})
}
/// Pull the next per-AU host timing (0xCF) into `*out`: the host's capture→sent duration for one /// Pull the next per-AU host timing (0xCF) into `*out`: the host's capture→sent duration for one
/// access unit, correlated to the AU by `pts_ns` (see [`PunktfunkHostTiming`]). /// access unit, correlated to the AU by `pts_ns` (see [`PunktfunkHostTiming`]).
/// [`PunktfunkStatus::NoFrame`] on timeout, [`PunktfunkStatus::Closed`] once the session ended. /// [`PunktfunkStatus::NoFrame`] on timeout, [`PunktfunkStatus::Closed`] once the session ended.
+1 -1
View File
@@ -102,7 +102,7 @@ pub use stats::Stats;
/// v10: added `punktfunk_connection_clock_offset_now_ns` — the LIVE (mid-stream re-synced) /// v10: added `punktfunk_connection_clock_offset_now_ns` — the LIVE (mid-stream re-synced)
/// clock offset ongoing latency math must use; the connect-time getter stays frozen by /// clock offset ongoing latency math must use; the connect-time getter stays frozen by
/// contract. Additive, client-local — no wire change, so [`WIRE_VERSION`] is unchanged. /// contract. Additive, client-local — no wire change, so [`WIRE_VERSION`] is unchanged.
pub const ABI_VERSION: u32 = 10; pub const ABI_VERSION: u32 = 11;
/// The punktfunk/1 **wire** version — what `Hello`/`Welcome` carry and hosts equality-check. /// The punktfunk/1 **wire** version — what `Hello`/`Welcome` carry and hosts equality-check.
/// Deliberately its own constant: [`ABI_VERSION`] tracks the embeddable **C surface** /// Deliberately its own constant: [`ABI_VERSION`] tracks the embeddable **C surface**
+89 -1
View File
@@ -45,7 +45,7 @@
// v10: added `punktfunk_connection_clock_offset_now_ns` — the LIVE (mid-stream re-synced) // v10: added `punktfunk_connection_clock_offset_now_ns` — the LIVE (mid-stream re-synced)
// clock offset ongoing latency math must use; the connect-time getter stays frozen by // clock offset ongoing latency math must use; the connect-time getter stays frozen by
// contract. Additive, client-local — no wire change, so [`WIRE_VERSION`] is unchanged. // contract. Additive, client-local — no wire change, so [`WIRE_VERSION`] is unchanged.
#define ABI_VERSION 10 #define ABI_VERSION 11
// The punktfunk/1 **wire** version — what `Hello`/`Welcome` carry and hosts equality-check. // The punktfunk/1 **wire** version — what `Hello`/`Welcome` carry and hosts equality-check.
// Deliberately its own constant: [`ABI_VERSION`] tracks the embeddable **C surface** // Deliberately its own constant: [`ABI_VERSION`] tracks the embeddable **C surface**
@@ -215,6 +215,10 @@
// clipboard, so a client may offer the toggle. (Mirrors `quic::HOST_CAP_CLIPBOARD`.) // clipboard, so a client may offer the toggle. (Mirrors `quic::HOST_CAP_CLIPBOARD`.)
#define PUNKTFUNK_HOST_CAP_CLIPBOARD 2 #define PUNKTFUNK_HOST_CAP_CLIPBOARD 2
// [`punktfunk_connect_ex9`] `client_caps` bit: render the host cursor locally (the cursor
// channel, `design/remote-desktop-sweep.md` M2).
#define PUNKTFUNK_CLIENT_CAP_CURSOR 1
// `*ttl_ms` sentinel written by [`punktfunk_connection_next_rumble2`] for a legacy (v1) rumble // `*ttl_ms` sentinel written by [`punktfunk_connection_next_rumble2`] for a legacy (v1) rumble
// datagram — an old host that sent no self-termination lease. The client then falls back to its // datagram — an old host that sent no self-termination lease. The client then falls back to its
// own staleness heuristic for that update instead of a host-supplied deadline. // own staleness heuristic for that update instead of a host-supplied deadline.
@@ -1373,6 +1377,31 @@ typedef struct {
} PunktfunkHdrMeta; } PunktfunkHdrMeta;
#endif #endif
// One forwarded host-cursor shape (ABI v11, the cursor channel): straight-alpha RGBA8, no
// padding, `len == w * h * 4`, hotspot within `w`×`h`. `serial` is the identity
// [`PunktfunkCursorState`] refers to — cache the built OS cursor by it.
typedef struct {
uint32_t serial;
uint16_t w;
uint16_t h;
uint16_t hot_x;
uint16_t hot_y;
// Borrows connection memory until the NEXT cursor-shape call (the audio contract).
const uint8_t *rgba;
uintptr_t len;
} PunktfunkCursorShape;
// Per-frame host-cursor state (ABI v11): position (the pointer/hotspot point in the host
// video's pixel space), visibility, and the host-driven relative-mode hint. `flags` bit 0 =
// visible, bit 1 = relative hint (a host app grabbed/hid the pointer — run captured
// relative; clear = return to absolute, reappearing at `x`/`y`).
typedef struct {
uint32_t serial;
uint8_t flags;
int32_t x;
int32_t y;
} PunktfunkCursorState;
#if defined(PUNKTFUNK_FEATURE_QUIC) #if defined(PUNKTFUNK_FEATURE_QUIC)
// One access unit's host-side processing time ([`punktfunk_connection_next_host_timing`]): // One access unit's host-side processing time ([`punktfunk_connection_next_host_timing`]):
// capture → fully sent, i.e. the whole host pipeline (capture read/convert, encode, FEC+seal, // capture → fully sent, i.e. the whole host pipeline (capture read/convert, encode, FEC+seal,
@@ -1859,6 +1888,38 @@ PunktfunkConnection *punktfunk_connect_ex8(const char *host,
int32_t *status_out); int32_t *status_out);
#endif #endif
#if defined(PUNKTFUNK_FEATURE_QUIC)
// Like [`punktfunk_connect_ex8`], plus `client_caps` (ABI v11): a bitfield of
// `PUNKTFUNK_CLIENT_CAP_CURSOR` (0x01). Setting the cursor bit asks the host to STOP
// compositing the pointer into the video and forward it out-of-band instead — the embedder
// MUST then drain [`punktfunk_connection_next_cursor_shape`] /
// [`punktfunk_connection_next_cursor_state`] and draw the pointer itself, or the session has
// no visible cursor at all. Pass 0 for the composited behavior of every earlier variant.
//
// # Safety
// Same as [`punktfunk_connect_ex8`].
PunktfunkConnection *punktfunk_connect_ex9(const char *host,
uint16_t port,
uint32_t width,
uint32_t height,
uint32_t refresh_hz,
uint32_t compositor,
uint32_t gamepad,
uint32_t bitrate_kbps,
uint8_t video_caps,
uint8_t audio_channels,
uint8_t video_codecs,
uint8_t preferred_codec,
uint8_t client_caps,
const char *launch_id,
const uint8_t *pin_sha256,
uint8_t *observed_sha256_out,
const char *client_cert_pem,
const char *client_key_pem,
uint32_t timeout_ms,
int32_t *status_out);
#endif
#if defined(PUNKTFUNK_FEATURE_QUIC) #if defined(PUNKTFUNK_FEATURE_QUIC)
// Generate a persistent client identity: a self-signed certificate + private key, both // Generate a persistent client identity: a self-signed certificate + private key, both
// PEM, NUL-terminated, written into the caller's buffers. Generate ONCE, store both // PEM, NUL-terminated, written into the caller's buffers. Generate ONCE, store both
@@ -2080,6 +2141,33 @@ PunktfunkStatus punktfunk_connection_next_hdr_meta(PunktfunkConnection *c,
uint32_t timeout_ms); uint32_t timeout_ms);
#endif #endif
#if defined(PUNKTFUNK_FEATURE_QUIC)
// Pull the next forwarded cursor SHAPE (sent on pointer-bitmap change over the reliable
// control stream; only a session connected with `PUNKTFUNK_CLIENT_CAP_CURSOR` against a
// capable host receives any). On `Ok`, `out->rgba` borrows connection memory until the next
// cursor-shape call on this handle. Drain from a dedicated thread (one thread per plane).
//
// # Safety
// `c` is a valid connection handle; `out` is writable. At most one thread pulls cursor
// shapes; it may run concurrently with every other plane's puller.
PunktfunkStatus punktfunk_connection_next_cursor_shape(PunktfunkConnection *c,
PunktfunkCursorShape *out,
uint32_t timeout_ms);
#endif
#if defined(PUNKTFUNK_FEATURE_QUIC)
// Pull the next cursor STATE (a `0xD0` datagram per host encode tick — latest-wins; drain
// the queue and apply only the newest). Same negotiation gate as
// [`punktfunk_connection_next_cursor_shape`].
//
// # Safety
// `c` is a valid connection handle; `out` is writable. At most one thread pulls cursor
// state; it may run concurrently with every other plane's puller.
PunktfunkStatus punktfunk_connection_next_cursor_state(PunktfunkConnection *c,
PunktfunkCursorState *out,
uint32_t timeout_ms);
#endif
#if defined(PUNKTFUNK_FEATURE_QUIC) #if defined(PUNKTFUNK_FEATURE_QUIC)
// Pull the next per-AU host timing (0xCF) into `*out`: the host's capture→sent duration for one // Pull the next per-AU host timing (0xCF) into `*out`: the host's capture→sent duration for one
// access unit, correlated to the AU by `pts_ns` (see [`PunktfunkHostTiming`]). // access unit, correlated to the AU by `pts_ns` (see [`PunktfunkHostTiming`]).