From 1cd5e0e3752692232b725824fab991067fcba1ee Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Thu, 18 Jun 2026 22:09:54 +0000 Subject: [PATCH] feat(android): HDR (Main10 / BT.2020 PQ) + fix ndk feature gating Mirrors the Apple client's HDR path so the Android client can display HDR from a Windows HDR host: - nativeConnect now advertises VIDEO_CAP_10BIT | VIDEO_CAP_HDR (was 0), so the host upgrades to a Main10 / BT.2020 PQ encode. - decode.rs detects HDR reactively from the decoder's reported output colour (color-transfer ST2084=6 / HLG=7, color-range) -- the AMediaCodec analogue of VideoToolbox's format description on Apple -- and signals the Surface dataspace (Bt2020[Itu]Pq / Bt2020[Itu]Hlg) so the compositor/display switch to HDR. AMediaCodec decodes Main10 from the in-band SPS; no profile override needed. Also fixes the Android build: set_frame_rate (added in 5262e28) is gated on the ndk `nativewindow` + `api-level-30` features, which weren't enabled -- so that commit could not compile under cargo-ndk. Enable features = ["media","audio","nativewindow","api-level-31"] (minSdk 31): covers set_frame_rate (api-30), set_buffers_data_space + the DataSpace module (api-28), and ANativeWindow (nativewindow). Verified host-side: fmt --all + clippy --workspace (the caps advertise + JNI surface). The android-gated decode + NDK gating verified against the ndk 0.9 sources; android.yml (cargo-ndk) is the compile gate, and real HDR display needs an HDR device + Windows HDR host. Co-Authored-By: Claude Opus 4.8 (1M context) --- clients/android/native/Cargo.toml | 2 +- clients/android/native/src/decode.rs | 53 +++++++++++++++++++++++++-- clients/android/native/src/session.rs | 11 ++++-- 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/clients/android/native/Cargo.toml b/clients/android/native/Cargo.toml index b6ed6f8..6ae9eba 100644 --- a/clients/android/native/Cargo.toml +++ b/clients/android/native/Cargo.toml @@ -28,7 +28,7 @@ android_logger = "0.14" # NDK bindings. "media" = AMediaCodec/ANativeWindow (video); "audio" = AAudio (audio playback). # Pure-Rust FFI to libmediandk/libnativewindow/libaaudio — no C++/libc++_shared to bundle. Decode + # audio run entirely in Rust on native threads (the "no async on the hot path" invariant). -ndk = { version = "0.9", features = ["media", "audio"] } +ndk = { version = "0.9", features = ["media", "audio", "nativewindow", "api-level-31"] } # setpriority/gettid to raise the decode thread toward URGENT_DISPLAY (see decode::boost_thread_priority). libc = "0.2" # Opus decode for the host→client audio plane (0xC9: 48 kHz stereo, 5 ms frames). Same crate the diff --git a/clients/android/native/src/decode.rs b/clients/android/native/src/decode.rs index 00cee6e..b785c51 100644 --- a/clients/android/native/src/decode.rs +++ b/clients/android/native/src/decode.rs @@ -6,6 +6,7 @@ //! WxH (from [`NativeClient::mode`]) and feed each access unit as it arrives. The decode thread owns //! the codec + window for its whole life; [`crate::session`] signals it to stop via the shared flag. +use ndk::data_space::DataSpace; use ndk::media::media_codec::{ DequeuedInputBufferResult, DequeuedOutputBufferInfoResult, MediaCodec, MediaCodecDirection, }; @@ -83,6 +84,9 @@ pub fn run( // Capture→client-receipt latency uses the negotiated host-minus-client clock offset (0 if the // host didn't answer the skew handshake — then the HUD flags it "same-host"). let clock_offset = client.clock_offset_ns; + // The dataspace we've signalled on the Surface so far (None = default/SDR). Set reactively once + // the decoder reports an HDR stream (see `drain`); avoids re-applying every format event. + let mut applied_ds: Option = None; while !shutdown.load(Ordering::Relaxed) { match client.next_frame(Duration::from_millis(5)) { Ok(frame) => { @@ -105,7 +109,7 @@ pub fn run( Err(PunktfunkError::NoFrame) => {} // timeout — still drain output below Err(_) => break, // session closed } - rendered += drain(&codec); + rendered += drain(&codec, &window, &mut applied_ds); // Loss recovery: under infinite GOP the only recovery keyframe is one we request. The // reassembler drops unrecoverable AUs (frames_dropped); the decoder then conceals the @@ -191,8 +195,8 @@ fn feed(codec: &MediaCodec, au: &[u8], pts_us: u64) { } /// Release any ready output buffers to the surface (render = true), latency-first. Returns the -/// number of frames presented. -fn drain(codec: &MediaCodec) -> u64 { +/// number of frames presented. Also reacts to `OutputFormatChanged` to signal HDR on the Surface. +fn drain(codec: &MediaCodec, window: &NativeWindow, applied_ds: &mut Option) -> u64 { let mut n = 0; loop { match codec.dequeue_output_buffer(Duration::from_millis(0)) { @@ -203,7 +207,27 @@ fn drain(codec: &MediaCodec) -> u64 { } n += 1; } - // TryAgainLater / OutputFormatChanged / OutputBuffersChanged — nothing to render now. + Ok(DequeuedOutputBufferInfoResult::OutputFormatChanged) => { + // The decoder has parsed the SPS and now reports the stream's real colour signalling + // (the AMediaCodec analogue of VideoToolbox's format description on the Apple client). + // If it's HDR (BT.2020 PQ/HLG), tell the Surface so the compositor/display switch to + // HDR; SDR streams leave the default dataspace alone. The decoder itself picks a + // Main10 path from the SPS — no profile override needed. Keep looping (buffers follow). + if let Some(ds) = hdr_dataspace(codec) { + if *applied_ds != Some(ds) { + match window.set_buffers_data_space(ds) { + Ok(()) => { + *applied_ds = Some(ds); + log::info!("decode: HDR stream → Surface dataspace {ds:?}"); + } + Err(e) => log::warn!( + "decode: set_buffers_data_space({ds:?}) failed (non-fatal): {e}" + ), + } + } + } + } + // TryAgainLater / OutputBuffersChanged — nothing to render now. Ok(_) => break, Err(e) => { log::warn!("decode: dequeue_output_buffer: {e}"); @@ -213,3 +237,24 @@ fn drain(codec: &MediaCodec) -> u64 { } n } + +/// Map the decoder's reported output colour to a BT.2020 HDR dataspace, or `None` for SDR. The +/// integer values are the Android MediaFormat colour constants the NDK shares: COLOR_TRANSFER +/// ST2084 = 6 (PQ/HDR10), HLG = 7; COLOR_RANGE FULL = 1, LIMITED = 2 (the host encodes limited). +fn hdr_dataspace(codec: &MediaCodec) -> Option { + let fmt = codec.output_format(); + let full_range = fmt.i32("color-range") == Some(1); + match fmt.i32("color-transfer") { + Some(6) => Some(if full_range { + DataSpace::Bt2020Pq + } else { + DataSpace::Bt2020ItuPq + }), + Some(7) => Some(if full_range { + DataSpace::Bt2020Hlg + } else { + DataSpace::Bt2020ItuHlg + }), + _ => None, // SDR (BT.709 / SDR_VIDEO) or unspecified + } +} diff --git a/clients/android/native/src/session.rs b/clients/android/native/src/session.rs index 8847090..2246e1e 100644 --- a/clients/android/native/src/session.rs +++ b/clients/android/native/src/session.rs @@ -184,10 +184,13 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeConnect<'lo CompositorPref::from_u8(compositor_pref.clamp(0, u8::MAX as jint) as u8), GamepadPref::from_u8(gamepad_pref.clamp(0, u8::MAX as jint) as u8), bitrate_kbps.max(0) as u32, // 0 = host default - 0, // video_caps: 8-bit only on Android for now - None, // launch: default app - pin, // Some → Crypto on host-fp mismatch - identity, // owned (cert, key) PEM, or None (anonymous) + // Advertise 10-bit + HDR: the host (e.g. Windows) only upgrades to a Main10 / BT.2020 PQ + // encode when the client sets these. AMediaCodec decodes Main10 from the SPS and the decode + // loop signals the Surface's HDR dataspace from the reported colour (see crate::decode). + punktfunk_core::quic::VIDEO_CAP_10BIT | punktfunk_core::quic::VIDEO_CAP_HDR, + None, // launch: default app + pin, // Some → Crypto on host-fp mismatch + identity, // owned (cert, key) PEM, or None (anonymous) Duration::from_secs(10), ) { Ok(client) => {